hadoop-mapreduce/src/test/mapred/org/apache/hadoop/fs/slive/CreateOp.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.fs.slive;

import java.io.IOException;
import java.util.List;
import java.util.Random;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.slive.DataWriter.GenerateOutput;
import org.apache.hadoop.fs.slive.OperationOutput.OutputType;

/**
 * Operation which selects a random file and a random number of bytes to create
 * that file with (from the write size option) and selects a random block size
 * (from the block size option) and a random replication amount (from the
 * replication option) and attempts to create a file with those options.
 * 
 * This operation will capture statistics on success for bytes written, time
 * taken (milliseconds), and success count and on failure it will capture the
 * number of failures and the time taken (milliseconds) to fail.
 */
class CreateOp extends Operation {

  private static final Log LOG = LogFactory.getLog(CreateOp.class);

  private static int DEF_IO_BUFFER_SIZE = 4096;

  private static final String IO_BUF_CONFIG = ("io.file.buffer.size");

  CreateOp(ConfigExtractor cfg, Random rnd) {
    super(CreateOp.class.getSimpleName(), cfg, rnd);
  }

  /**
   * Returns the block size to use (aligned to nearest BYTES_PER_CHECKSUM if
   * configuration says a value exists) - this will avoid the warnings caused by
   * this not occurring and the file will not be created if it is not correct...
   * 
   * @return long
   */
  private long determineBlockSize() {
    Range<Long> blockSizeRange = getConfig().getBlockSize();
    long blockSize = Range.betweenPositive(getRandom(), blockSizeRange);
    Long byteChecksum = getConfig().getByteCheckSum();
    if (byteChecksum == null) {
      return blockSize;
    }
    // adjust to nearest multiple
    long full = (blockSize / byteChecksum) * byteChecksum;
    long toFull = blockSize - full;
    if (toFull >= (byteChecksum / 2)) {
      full += byteChecksum;
    }
    // adjust if over extended
    if (full > blockSizeRange.getUpper()) {
      full = blockSizeRange.getUpper();
    }
    if (full < blockSizeRange.getLower()) {
      full = blockSizeRange.getLower();
    }
    return full;
  }

  /**
   * Gets the replication amount
   * 
   * @return short
   */
  private short determineReplication() {
    Range<Short> replicationAmountRange = getConfig().getReplication();
    Range<Long> repRange = new Range<Long>(replicationAmountRange.getLower()
        .longValue(), replicationAmountRange.getUpper().longValue());
    short replicationAmount = (short) Range.betweenPositive(getRandom(),
        repRange);
    return replicationAmount;
  }

  /**
   * Gets the output buffering size to use
   * 
   * @return int
   */
  private int getBufferSize() {
    return getConfig().getConfig().getInt(IO_BUF_CONFIG, DEF_IO_BUFFER_SIZE);
  }

  /**
   * Gets the file to create
   * 
   * @return Path
   */
  protected Path getCreateFile() {
    Path fn = getFinder().getFile();
    return fn;
  }

  @Override // Operation
  List<OperationOutput> run(FileSystem fs) {
    List<OperationOutput> out = super.run(fs);
    FSDataOutputStream os = null;
    try {
      Path fn = getCreateFile();
      Range<Long> writeSizeRange = getConfig().getWriteSize();
      long writeSize = 0;
      long blockSize = determineBlockSize();
      short replicationAmount = determineReplication();
      if (getConfig().shouldWriteUseBlockSize()) {
        writeSizeRange = getConfig().getBlockSize();
      }
      writeSize = Range.betweenPositive(getRandom(), writeSizeRange);
      long bytesWritten = 0;
      long timeTaken = 0;
      int bufSize = getBufferSize();
      boolean overWrite = false;
      DataWriter writer = new DataWriter(getRandom());
      LOG.info("Attempting to create file at " + fn + " of size "
          + Helper.toByteInfo(writeSize) + " using blocksize "
          + Helper.toByteInfo(blockSize) + " and replication amount "
          + replicationAmount);
      {
        // open & create
        long startTime = Timer.now();
        os = fs.create(fn, overWrite, bufSize, replicationAmount, blockSize);
        timeTaken += Timer.elapsed(startTime);
        // write the given length
        GenerateOutput stats = writer.writeSegment(writeSize, os);
        bytesWritten += stats.getBytesWritten();
        timeTaken += stats.getTimeTaken();
        // capture close time
        startTime = Timer.now();
        os.close();
        os = null;
        timeTaken += Timer.elapsed(startTime);
      }
      LOG.info("Created file at " + fn + " of size "
          + Helper.toByteInfo(bytesWritten) + " bytes using blocksize "
          + Helper.toByteInfo(blockSize) + " and replication amount "
          + replicationAmount + " in " + timeTaken + " milliseconds");
      // collect all the stats
      out.add(new OperationOutput(OutputType.LONG, getType(),
          ReportWriter.OK_TIME_TAKEN, timeTaken));
      out.add(new OperationOutput(OutputType.LONG, getType(),
          ReportWriter.BYTES_WRITTEN, bytesWritten));
      out.add(new OperationOutput(OutputType.LONG, getType(),
          ReportWriter.SUCCESSES, 1L));
    } catch (IOException e) {
      out.add(new OperationOutput(OutputType.LONG, getType(),
          ReportWriter.FAILURES, 1L));
      LOG.warn("Error with creating", e);
    } finally {
      if (os != null) {
        try {
          os.close();
        } catch (IOException e) {
          LOG.warn("Error closing create stream", e);
        }
      }
    }
    return out;
  }
}