common/src/test/core/org/apache/hadoop/io/file/tfile/TestTFileUnsortedByteArrays.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with this
 * work for additional information regarding copyright ownership. The ASF
 * licenses this file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package org.apache.hadoop.io.file.tfile;

import java.io.IOException;

import junit.framework.Assert;
import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.file.tfile.TFile.Reader;
import org.apache.hadoop.io.file.tfile.TFile.Writer;
import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner;

public class TestTFileUnsortedByteArrays extends TestCase {
  private static String ROOT =
      System.getProperty("test.build.data", "/tmp/tfile-test");


  private final static int BLOCK_SIZE = 512;
  private final static int BUF_SIZE = 64;

  private FileSystem fs;
  private Configuration conf;
  private Path path;
  private FSDataOutputStream out;
  private Writer writer;

  private String compression = Compression.Algorithm.GZ.getName();
  private String outputFile = "TFileTestUnsorted";
  /*
   * pre-sampled numbers of records in one block, based on the given the
   * generated key and value strings
   */
  private int records1stBlock = 4314;
  private int records2ndBlock = 4108;

  public void init(String compression, String outputFile,
      int numRecords1stBlock, int numRecords2ndBlock) {
    this.compression = compression;
    this.outputFile = outputFile;
    this.records1stBlock = numRecords1stBlock;
    this.records2ndBlock = numRecords2ndBlock;
  }

  @Override
  public void setUp() throws IOException {
    conf = new Configuration();
    path = new Path(ROOT, outputFile);
    fs = path.getFileSystem(conf);
    out = fs.create(path);
    writer = new Writer(out, BLOCK_SIZE, compression, null, conf);
    writer.append("keyZ".getBytes(), "valueZ".getBytes());
    writer.append("keyM".getBytes(), "valueM".getBytes());
    writer.append("keyN".getBytes(), "valueN".getBytes());
    writer.append("keyA".getBytes(), "valueA".getBytes());
    closeOutput();
  }

  @Override
  public void tearDown() throws IOException {
    fs.delete(path, true);
  }

  // we still can scan records in an unsorted TFile
  public void testFailureScannerWithKeys() throws IOException {
    Reader reader =
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Assert.assertFalse(reader.isSorted());
    Assert.assertEquals((int) reader.getEntryCount(), 4);

    try {
      Scanner scanner =
          reader.createScannerByKey("aaa".getBytes(), "zzz".getBytes());
      Assert
          .fail("Failed to catch creating scanner with keys on unsorted file.");
    }
    catch (RuntimeException e) {
    }
    finally {
      reader.close();
    }
  }

  // we still can scan records in an unsorted TFile
  public void testScan() throws IOException {
    Reader reader =
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Assert.assertFalse(reader.isSorted());
    Assert.assertEquals((int) reader.getEntryCount(), 4);

    Scanner scanner = reader.createScanner();

    try {

      // read key and value
      byte[] kbuf = new byte[BUF_SIZE];
      int klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

      byte[] vbuf = new byte[BUF_SIZE];
      int vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

      scanner.advance();

      // now try get value first
      vbuf = new byte[BUF_SIZE];
      vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

      kbuf = new byte[BUF_SIZE];
      klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
    }
    finally {
      scanner.close();
      reader.close();
    }
  }

  // we still can scan records in an unsorted TFile
  public void testScanRange() throws IOException {
    Reader reader =
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Assert.assertFalse(reader.isSorted());
    Assert.assertEquals((int) reader.getEntryCount(), 4);

    Scanner scanner = reader.createScanner();

    try {

      // read key and value
      byte[] kbuf = new byte[BUF_SIZE];
      int klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

      byte[] vbuf = new byte[BUF_SIZE];
      int vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

      scanner.advance();

      // now try get value first
      vbuf = new byte[BUF_SIZE];
      vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

      kbuf = new byte[BUF_SIZE];
      klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
    }
    finally {
      scanner.close();
      reader.close();
    }
  }

  public void testFailureSeek() throws IOException {
    Reader reader =
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Scanner scanner = reader.createScanner();

    try {
      // can't find ceil
      try {
        scanner.lowerBound("keyN".getBytes());
        Assert.fail("Cannot search in a unsorted TFile!");
      }
      catch (Exception e) {
        // noop, expecting excetions
      }
      finally {
      }

      // can't find higher
      try {
        scanner.upperBound("keyA".getBytes());
        Assert.fail("Cannot search higher in a unsorted TFile!");
      }
      catch (Exception e) {
        // noop, expecting excetions
      }
      finally {
      }

      // can't seek
      try {
        scanner.seekTo("keyM".getBytes());
        Assert.fail("Cannot search a unsorted TFile!");
      }
      catch (Exception e) {
        // noop, expecting excetions
      }
      finally {
      }
    }
    finally {
      scanner.close();
      reader.close();
    }
  }

  private void closeOutput() throws IOException {
    if (writer != null) {
      writer.close();
      writer = null;
      out.close();
      out = null;
    }
  }
}