hdfs/src/test/hdfs/org/apache/hadoop/hdfs/BenchmarkThroughput.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.ChecksumFileSystem;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import org.apache.log4j.Level;

/**
 * This class benchmarks the performance of the local file system, raw local
 * file system and HDFS at reading and writing files. The user should invoke
 * the main of this class and optionally include a repetition count.
 */
public class BenchmarkThroughput extends Configured implements Tool {

  // the property in the config that specifies a working directory
  private LocalDirAllocator dir;
  private long startTime;
  // the size of the buffer to use
  private int BUFFER_SIZE;

  private void resetMeasurements() {
    startTime = System.currentTimeMillis();
  }

  private void printMeasurements() {
    System.out.println(" time: " +
                       ((System.currentTimeMillis() - startTime)/1000));
  }

  private Path writeLocalFile(String name, Configuration conf,
                                     long total) throws IOException {
    Path path = dir.getLocalPathForWrite(name, total, conf);
    System.out.print("Writing " + name);
    resetMeasurements();
    OutputStream out = new FileOutputStream(new File(path.toString()));
    byte[] data = new byte[BUFFER_SIZE];
    for(long size=0; size < total; size += BUFFER_SIZE) {
      out.write(data);
    }
    out.close();
    printMeasurements();
    return path;
  }

  private void readLocalFile(Path path,
                                    String name,
                                    Configuration conf) throws IOException {
    System.out.print("Reading " + name);
    resetMeasurements();
    InputStream in = new FileInputStream(new File(path.toString()));
    byte[] data = new byte[BUFFER_SIZE];
    long size = 0;
    while (size >= 0) {
      size = in.read(data);
    }
    in.close();
    printMeasurements();
  }

  private void writeAndReadLocalFile(String name,
                                            Configuration conf,
                                            long size
                                           ) throws IOException {
    Path f = null;
    try {
      f = writeLocalFile(name, conf, size);
      readLocalFile(f, name, conf);
    } finally {
      if (f != null) {
        new File(f.toString()).delete();
      }
    }
  }

  private Path writeFile(FileSystem fs,
                                String name,
                                Configuration conf,
                                long total
                                ) throws IOException {
    Path f = dir.getLocalPathForWrite(name, total, conf);
    System.out.print("Writing " + name);
    resetMeasurements();
    OutputStream out = fs.create(f);
    byte[] data = new byte[BUFFER_SIZE];
    for(long size = 0; size < total; size += BUFFER_SIZE) {
      out.write(data);
    }
    out.close();
    printMeasurements();
    return f;
  }

  private void readFile(FileSystem fs,
                               Path f,
                               String name,
                               Configuration conf
                               ) throws IOException {
    System.out.print("Reading " + name);
    resetMeasurements();
    InputStream in = fs.open(f);
    byte[] data = new byte[BUFFER_SIZE];
    long val = 0;
    while (val >= 0) {
      val = in.read(data);
    }
    in.close();
    printMeasurements();
  }

  private void writeAndReadFile(FileSystem fs,
                                       String name,
                                       Configuration conf,
                                       long size
                                       ) throws IOException {
    Path f = null;
    try {
      f = writeFile(fs, name, conf, size);
      readFile(fs, f, name, conf);
    } finally {
      try {
        if (f != null) {
          fs.delete(f, true);
        }
      } catch (IOException ie) {
        // IGNORE
      }
    }
  }

  private static void printUsage() {
    ToolRunner.printGenericCommandUsage(System.err);
    System.err.println("Usage: dfsthroughput [#reps]");
    System.err.println("Config properties:\n" +
      "  dfsthroughput.file.size:\tsize of each write/read (10GB)\n" +
      "  dfsthroughput.buffer.size:\tbuffer size for write/read (4k)\n");
  }

  public int run(String[] args) throws IOException {
    // silence the minidfs cluster
    Log hadoopLog = LogFactory.getLog("org");
    if (hadoopLog instanceof Log4JLogger) {
      ((Log4JLogger) hadoopLog).getLogger().setLevel(Level.WARN);
    }
    int reps = 1;
    if (args.length == 1) {
      try {
        reps = Integer.parseInt(args[0]);
      } catch (NumberFormatException e) {
        printUsage();
        return -1;
      }
    } else if (args.length > 1) {
      printUsage();
      return -1;
    }
    Configuration conf = getConf();
    // the size of the file to write
    long SIZE = conf.getLong("dfsthroughput.file.size",
        10L * 1024 * 1024 * 1024);
    BUFFER_SIZE = conf.getInt("dfsthroughput.buffer.size", 4 * 1024);

    String localDir = conf.get("mapred.temp.dir");
    dir = new LocalDirAllocator("mapred.temp.dir");

    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }
    MiniDFSCluster cluster = null;
    try {
      cluster = new MiniDFSCluster(conf, 1, true, new String[]{"/foo"});
      cluster.waitActive();
      FileSystem dfs = cluster.getFileSystem();
      for(int i=0; i < reps; ++i) {
        writeAndReadFile(dfs, "dfs", conf, SIZE);
      }
    } finally {
      if (cluster != null) {
        cluster.shutdown();
        // clean up minidfs junk
        rawLocal.delete(new Path(localDir, "dfs"), true);
      }
    }
    return 0;
  }

  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new HdfsConfiguration(),
        new BenchmarkThroughput(), args);
    System.exit(res);
  }

}