hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestArrayFile.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.io;

import java.io.*;


import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.conf.*;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;

/** Support for flat files of binary key/value pairs. */
public class TestArrayFile {
  private static final Logger LOG =
      LoggerFactory.getLogger(TestArrayFile.class);
  
  private static final Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
      TestMapFile.class.getSimpleName()));
  private static String TEST_FILE = new Path(TEST_DIR, "test.array").toString();

  @Test
  public void testArrayFile() throws Exception {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.getLocal(conf);
    RandomDatum[] data = generate(10000);
    writeTest(fs, data, TEST_FILE);
    readTest(fs, data, TEST_FILE, conf);
  }

  @Test
  public void testEmptyFile() throws Exception {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.getLocal(conf);
    writeTest(fs, new RandomDatum[0], TEST_FILE);
    ArrayFile.Reader reader = new ArrayFile.Reader(fs, TEST_FILE, conf);
    assertNull(reader.get(0, new RandomDatum()));
    reader.close();
  }

  private static RandomDatum[] generate(int count) {
    if(LOG.isDebugEnabled()) {
      LOG.debug("generating " + count + " records in debug");
    }
    RandomDatum[] data = new RandomDatum[count];
    RandomDatum.Generator generator = new RandomDatum.Generator();
    for (int i = 0; i < count; i++) {
      generator.next();
      data[i] = generator.getValue();
    }
    return data;
  }

  private static void writeTest(FileSystem fs, RandomDatum[] data, String file)
    throws IOException {
    Configuration conf = new Configuration();
    MapFile.delete(fs, file);
    if(LOG.isDebugEnabled()) {
      LOG.debug("creating with " + data.length + " debug");
    }
    ArrayFile.Writer writer = new ArrayFile.Writer(conf, fs, file, RandomDatum.class);
    writer.setIndexInterval(100);
    for (int i = 0; i < data.length; i++)
      writer.append(data[i]);
    writer.close();
  }

  private static void readTest(FileSystem fs, RandomDatum[] data, String file, Configuration conf)
    throws IOException {
    RandomDatum v = new RandomDatum();
    if(LOG.isDebugEnabled()) {
      LOG.debug("reading " + data.length + " debug");
    }
    ArrayFile.Reader reader = new ArrayFile.Reader(fs, file, conf);
    try {
      for (int i = 0; i < data.length; i++) {       // try forwards
        reader.get(i, v);
        if (!v.equals(data[i])) {
          throw new RuntimeException("wrong value at " + i);
        }
      }
      for (int i = data.length-1; i >= 0; i--) {    // then backwards
        reader.get(i, v);
        if (!v.equals(data[i])) {
          throw new RuntimeException("wrong value at " + i);
        }
      }
      if(LOG.isDebugEnabled()) {
        LOG.debug("done reading " + data.length + " debug");
      }
    } finally {
      reader.close();
    }
  }

  /** 
   * test on {@link ArrayFile.Reader} iteration methods
   * <pre> 
   * {@code next(), seek()} in and out of range.
   * </pre>
   */
  @Test
  public void testArrayFileIteration() {
    int SIZE = 10;
    Configuration conf = new Configuration();    
    try {
      FileSystem fs = FileSystem.get(conf);
      ArrayFile.Writer writer = new ArrayFile.Writer(conf, fs, TEST_FILE, 
          LongWritable.class, CompressionType.RECORD, defaultProgressable);
      assertNotNull("testArrayFileIteration error !!!", writer);
      
      for (int i = 0; i < SIZE; i++)
        writer.append(new LongWritable(i));
      
      writer.close();
      
      ArrayFile.Reader reader = new ArrayFile.Reader(fs, TEST_FILE, conf);
      LongWritable nextWritable = new LongWritable(0);
      
      for (int i = 0; i < SIZE; i++) {
        nextWritable = (LongWritable)reader.next(nextWritable);
        assertEquals(nextWritable.get(), i);
      }
        
      assertTrue("testArrayFileIteration seek error !!!",
          reader.seek(new LongWritable(6)));
      nextWritable = (LongWritable) reader.next(nextWritable);
      assertTrue("testArrayFileIteration error !!!", reader.key() == 7);
      assertTrue("testArrayFileIteration error !!!",
          nextWritable.equals(new LongWritable(7)));
      assertFalse("testArrayFileIteration error !!!",
          reader.seek(new LongWritable(SIZE + 5)));
      reader.close();
    } catch (Exception ex) {
      fail("testArrayFileWriterConstruction error !!!");
    }
  }
 
  /** For debugging and testing. */
  public static void main(String[] args) throws Exception {
    int count = 1024 * 1024;
    boolean create = true;
    boolean check = true;
    String file = TEST_FILE;
    String usage = "Usage: TestArrayFile [-count N] [-nocreate] [-nocheck] file";
      
    if (args.length == 0) {
      System.err.println(usage);
      System.exit(-1);
    }

    Configuration conf = new Configuration();
    int i = 0;
    Path fpath = null;
    FileSystem fs = null;
    try {
      for (; i < args.length; i++) {       // parse command line
        if (args[i] == null) {
          continue;
        } else if (args[i].equals("-count")) {
          count = Integer.parseInt(args[++i]);
        } else if (args[i].equals("-nocreate")) {
          create = false;
        } else if (args[i].equals("-nocheck")) {
          check = false;
        } else {                                       
          // file is required parameter
          file = args[i];
          fpath=new Path(file);
        }
      }
        
      fs = fpath.getFileSystem(conf);
        
      LOG.info("count = " + count);
      LOG.info("create = " + create);
      LOG.info("check = " + check);
      LOG.info("file = " + file);

      RandomDatum[] data = generate(count);

      if (create) {
        writeTest(fs, data, file);
      }

      if (check) {
        readTest(fs, data, file, conf);
      }
    } finally {
      fs.close();
    }
  }
  
  private static final Progressable defaultProgressable = new Progressable() {
    @Override
    public void progress() {      
    }
  };
  
}