hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.tools;

import java.io.ByteArrayOutputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.StringTokenizer;

import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.HarFileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import static org.junit.Assert.*;
import org.junit.Before;
import org.junit.Test;

/**
 * test {@link HadoopArchives}
 */
public class TestHadoopArchives {

  public static final String HADOOP_ARCHIVES_JAR = JarFinder
      .getJar(HadoopArchives.class);

  {
    ((Log4JLogger) LogFactory.getLog(org.apache.hadoop.security.Groups.class))
        .getLogger().setLevel(Level.ERROR);

  }

  private static final String inputDir = "input";

  private Path inputPath;
  private Path archivePath;
  private final List<String> fileList = new ArrayList<String>();
  private MiniDFSCluster dfscluster;

  private Configuration conf;
  private FileSystem fs;

  private static String createFile(Path root, FileSystem fs, String... dirsAndFile
      ) throws IOException {
    String fileBaseName = dirsAndFile[dirsAndFile.length - 1]; 
    return createFile(root, fs, fileBaseName.getBytes("UTF-8"), dirsAndFile);
  }
  
  private static String createFile(Path root, FileSystem fs, byte[] fileContent, String... dirsAndFile
    ) throws IOException {
    StringBuilder sb = new StringBuilder();
    for (String segment: dirsAndFile) {
      if (sb.length() > 0) {
        sb.append(Path.SEPARATOR);  
      }
      sb.append(segment);
    }
    final Path f = new Path(root, sb.toString());
    final FSDataOutputStream out = fs.create(f);
    try {
         out.write(fileContent);
    } finally {
      out.close();
    }
    return sb.toString();
  }

  @Before
  public void setUp() throws Exception {
    conf = new Configuration();
    conf.set(CapacitySchedulerConfiguration.PREFIX
        + CapacitySchedulerConfiguration.ROOT + "."
        + CapacitySchedulerConfiguration.QUEUES, "default");
    conf.set(CapacitySchedulerConfiguration.PREFIX
        + CapacitySchedulerConfiguration.ROOT + ".default."
        + CapacitySchedulerConfiguration.CAPACITY, "100");
    dfscluster = new MiniDFSCluster
      .Builder(conf)
      .checkExitOnShutdown(true)
      .numDataNodes(2)
      .format(true)
      .racks(null)
      .build();

    fs = dfscluster.getFileSystem();
    
    // prepare archive path:
    archivePath = new Path(fs.getHomeDirectory(), "archive");
    fs.delete(archivePath, true);
    
    // prepare input path:
    inputPath = new Path(fs.getHomeDirectory(), inputDir);
    fs.delete(inputPath, true);
    fs.mkdirs(inputPath);
    // create basic input files:
    fileList.add(createFile(inputPath, fs, "a"));
    fileList.add(createFile(inputPath, fs, "b"));
    fileList.add(createFile(inputPath, fs, "c"));
  }

  @After
  public void tearDown() throws Exception {
    if (dfscluster != null) {
      dfscluster.shutdown();
    }
  }

  @Test
  public void testRelativePath() throws Exception {
    final Path sub1 = new Path(inputPath, "dir1");
    fs.mkdirs(sub1);
    createFile(inputPath, fs, sub1.getName(), "a");
    final FsShell shell = new FsShell(conf);

    final List<String> originalPaths = lsr(shell, "input");
    System.out.println("originalPaths: " + originalPaths);

    // make the archive:
    final String fullHarPathStr = makeArchive();

    // compare results:
    final List<String> harPaths = lsr(shell, fullHarPathStr);
    Assert.assertEquals(originalPaths, harPaths);
  }
  
@Test
  public void testPathWithSpaces() throws Exception {
    // create files/directories with spaces
    createFile(inputPath, fs, "c c");
    final Path sub1 = new Path(inputPath, "sub 1");
    fs.mkdirs(sub1);
    createFile(sub1, fs, "file x y z");
    createFile(sub1, fs, "file");
    createFile(sub1, fs, "x");
    createFile(sub1, fs, "y");
    createFile(sub1, fs, "z");
    final Path sub2 = new Path(inputPath, "sub 1 with suffix");
    fs.mkdirs(sub2);
    createFile(sub2, fs, "z");

    final FsShell shell = new FsShell(conf);
    final String inputPathStr = inputPath.toUri().getPath();
    final List<String> originalPaths = lsr(shell, inputPathStr);

    // make the archive:
    final String fullHarPathStr = makeArchive();

    // compare results
    final List<String> harPaths = lsr(shell, fullHarPathStr);
    Assert.assertEquals(originalPaths, harPaths);
  }

  private static List<String> lsr(final FsShell shell, String dir)
      throws Exception {
    System.out.println("lsr root=" + dir);
    final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
    final PrintStream out = new PrintStream(bytes);
    final PrintStream oldOut = System.out;
    final PrintStream oldErr = System.err;
    System.setOut(out);
    System.setErr(out);
    final String results;
    try {
      Assert.assertEquals(0, shell.run(new String[] { "-lsr", dir }));
      results = bytes.toString();
    } finally {
      IOUtils.closeStream(out);
      System.setOut(oldOut);
      System.setErr(oldErr);
    }
    System.out.println("lsr results:\n" + results);
    String dirname = dir;
    if (dir.lastIndexOf(Path.SEPARATOR) != -1) {
      dirname = dir.substring(dir.lastIndexOf(Path.SEPARATOR));
    }

    final List<String> paths = new ArrayList<String>();
    for (StringTokenizer t = new StringTokenizer(results, "\n"); t
        .hasMoreTokens();) {
      final String s = t.nextToken();
      final int i = s.indexOf(dirname);
      if (i >= 0) {
        paths.add(s.substring(i + dirname.length()));
      }
    }
    Collections.sort(paths);
    System.out
        .println("lsr paths = " + paths.toString().replace(", ", ",\n  "));
    return paths;
  }
  
  @Test
  public void testReadFileContent() throws Exception {
    fileList.add(createFile(inputPath, fs, "c c"));
    final Path sub1 = new Path(inputPath, "sub 1");
    fs.mkdirs(sub1);
    fileList.add(createFile(inputPath, fs, sub1.getName(), "file x y z"));
    fileList.add(createFile(inputPath, fs, sub1.getName(), "file"));
    fileList.add(createFile(inputPath, fs, sub1.getName(), "x"));
    fileList.add(createFile(inputPath, fs, sub1.getName(), "y"));
    fileList.add(createFile(inputPath, fs, sub1.getName(), "z"));
    final Path sub2 = new Path(inputPath, "sub 1 with suffix");
    fs.mkdirs(sub2);
    fileList.add(createFile(inputPath, fs, sub2.getName(), "z"));
    // Generate a big binary file content:
    final byte[] binContent = prepareBin();
    fileList.add(createFile(inputPath, fs, binContent, sub2.getName(), "bin"));
    fileList.add(createFile(inputPath, fs, new byte[0], sub2.getName(), "zero-length"));

    final String fullHarPathStr = makeArchive();

    // Create fresh HarFs:
    final HarFileSystem harFileSystem = new HarFileSystem(fs);
    try {
      final URI harUri = new URI(fullHarPathStr);
      harFileSystem.initialize(harUri, fs.getConf());
      // now read the file content and compare it against the expected:
      int readFileCount = 0;
      for (final String pathStr0 : fileList) {
        final Path path = new Path(fullHarPathStr + Path.SEPARATOR + pathStr0);
        final String baseName = path.getName();
        final FileStatus status = harFileSystem.getFileStatus(path);
        if (status.isFile()) {
          // read the file:
          final byte[] actualContentSimple = readAllSimple(
              harFileSystem.open(path), true);
          
          final byte[] actualContentBuffer = readAllWithBuffer(
              harFileSystem.open(path), true);
          assertArrayEquals(actualContentSimple, actualContentBuffer);
          
          final byte[] actualContentFully = readAllWithReadFully(
              actualContentSimple.length,
              harFileSystem.open(path), true);
          assertArrayEquals(actualContentSimple, actualContentFully);
          
          final byte[] actualContentSeek = readAllWithSeek(
              actualContentSimple.length,
              harFileSystem.open(path), true);
          assertArrayEquals(actualContentSimple, actualContentSeek);
          
          final byte[] actualContentRead4
          = readAllWithRead4(harFileSystem.open(path), true);
          assertArrayEquals(actualContentSimple, actualContentRead4);
          
          final byte[] actualContentSkip = readAllWithSkip(
              actualContentSimple.length, 
              harFileSystem.open(path), 
              harFileSystem.open(path), 
              true);
          assertArrayEquals(actualContentSimple, actualContentSkip);
          
          if ("bin".equals(baseName)) {
            assertArrayEquals(binContent, actualContentSimple);
          } else if ("zero-length".equals(baseName)) {
            assertEquals(0, actualContentSimple.length);
          } else {
            String actual = new String(actualContentSimple, "UTF-8");
            assertEquals(baseName, actual);
          }
          readFileCount++;
        }
      }
      assertEquals(fileList.size(), readFileCount);
    } finally {
      harFileSystem.close();
    }
  }
  
  private static byte[] readAllSimple(FSDataInputStream fsdis, boolean close) throws IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
      int b;
      while (true) {
        b = fsdis.read();
        if (b < 0) {
          break;
        } else {
          baos.write(b);
        }
      }
      baos.close();
      return baos.toByteArray();
    } finally {
      if (close) {
        fsdis.close();
      }
    }
  }

  private static byte[] readAllWithBuffer(FSDataInputStream fsdis, boolean close)
      throws IOException {
    try {
      final int available = fsdis.available();
      final byte[] buffer;
      final ByteArrayOutputStream baos;
      if (available < 0) {
        buffer = new byte[1024];
        baos = new ByteArrayOutputStream(buffer.length * 2);
      } else {
        buffer = new byte[available];
        baos = new ByteArrayOutputStream(available);
      }
      int readIntoBuffer = 0;
      int read; 
      while (true) {
        read = fsdis.read(buffer, readIntoBuffer, buffer.length - readIntoBuffer);
        if (read < 0) {
          // end of stream:
          if (readIntoBuffer > 0) {
            baos.write(buffer, 0, readIntoBuffer);
          }
          return baos.toByteArray();
        } else {
          readIntoBuffer += read;
          if (readIntoBuffer == buffer.length) {
            // buffer is full, need to clean the buffer.
            // drop the buffered data to baos:
            baos.write(buffer);
            // reset the counter to start reading to the buffer beginning:
            readIntoBuffer = 0;
          } else if (readIntoBuffer > buffer.length) {
            throw new IOException("Read more than the buffer length: "
                + readIntoBuffer + ", buffer length = " + buffer.length);
          }
        }
      }
    } finally {
      if (close) {
        fsdis.close();
      }
    }
  }

  private static byte[] readAllWithReadFully(int totalLength, FSDataInputStream fsdis, boolean close)
      throws IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    // Simulate reading of some data structures of known length:
    final byte[] buffer = new byte[17];
    final int times = totalLength / buffer.length;
    final int remainder = totalLength % buffer.length;
    // it would be simpler to leave the position tracking to the 
    // InputStream, but we need to check the methods #readFully(2) 
    // and #readFully(4) that receive the position as a parameter:
    int position = 0;
    try {
      // read "data structures":
      for (int i=0; i<times; i++) {
        fsdis.readFully(position, buffer);
        position += buffer.length;
        baos.write(buffer);
      }
      if (remainder > 0) {
        // read the remainder:
        fsdis.readFully(position, buffer, 0, remainder);
        position += remainder;
        baos.write(buffer, 0, remainder);
      }
      try {
        fsdis.readFully(position, buffer, 0, 1);
        assertTrue(false);
      } catch (IOException ioe) {
        // okay
      }
      assertEquals(totalLength, position);
      final byte[] result = baos.toByteArray();
      assertEquals(totalLength, result.length);
      return result;
    } finally {
      if (close) {
        fsdis.close();
      }
    }
  }

  private static byte[] readAllWithRead4(FSDataInputStream fsdis, boolean close)
      throws IOException {
    try {
      final ByteArrayOutputStream baos = new ByteArrayOutputStream();
      final byte[] buffer = new byte[17];
      int totalRead = 0;
      int read;
      while (true) {
        read = fsdis.read(totalRead, buffer, 0, buffer.length);
        if (read > 0) {
          totalRead += read;
          baos.write(buffer, 0, read);
        } else if (read < 0) {
          break; // EOF
        } else {
          // read == 0:
          // zero result may be returned *only* in case if the 4th 
          // parameter is 0. Since in our case this is 'buffer.length',
          // zero return value clearly indicates a bug: 
          throw new AssertionError("FSDataInputStream#read(4) returned 0, while " +
          		" the 4th method parameter is " + buffer.length + ".");
        }
      }
      final byte[] result = baos.toByteArray();
      return result;
    } finally {
      if (close) {
        fsdis.close();
      }
    }
  }
  
  private static byte[] readAllWithSeek(final int totalLength, 
      final FSDataInputStream fsdis, final boolean close)
      throws IOException {
    final byte[] result = new byte[totalLength];
    long pos;
    try {
      // read the data in the reverse order, from 
      // the tail to the head by pieces of 'buffer' length:
      final byte[] buffer = new byte[17];
      final int times = totalLength / buffer.length;
      int read;
      int expectedRead;
      for (int i=times; i>=0; i--) {
        pos = i * buffer.length;
        fsdis.seek(pos);
        // check that seek is successful:
        assertEquals(pos, fsdis.getPos());
        read = fsdis.read(buffer);
        // check we read right number of bytes:
        if (i == times) {
          expectedRead = totalLength % buffer.length; // remainder
          if (expectedRead == 0) {
            // zero remainder corresponds to the EOS, so
            // by the contract of DataInpitStream#read(byte[]) -1 should be 
            // returned:
            expectedRead = -1;
          }
        } else {
          expectedRead = buffer.length;
        }
        assertEquals(expectedRead, read);
        if (read > 0) {
          System.arraycopy(buffer, 0, result, (int)pos, read);
        }
      }
      
      // finally, check that #seek() to not existing position leads to IOE:
      expectSeekIOE(fsdis, Long.MAX_VALUE, "Seek to Long.MAX_VALUE should lead to IOE.");
      expectSeekIOE(fsdis, Long.MIN_VALUE, "Seek to Long.MIN_VALUE should lead to IOE.");
      long pp = -1L;
      expectSeekIOE(fsdis, pp, "Seek to "+pp+" should lead to IOE.");
      
      // NB: is is *possible* to #seek(length), but *impossible* to #seek(length + 1):
      fsdis.seek(totalLength);
      assertEquals(totalLength, fsdis.getPos());
      pp = totalLength + 1;
      expectSeekIOE(fsdis, pp, "Seek to the length position + 1 ("+pp+") should lead to IOE.");
      
      return result;
    } finally {
      if (close) {
        fsdis.close();
      }
    }
  }  

  private static void expectSeekIOE(FSDataInputStream fsdis, long seekPos, String message) {
    try {
      fsdis.seek(seekPos);
      assertTrue(message + " (Position = " + fsdis.getPos() + ")", false);
    } catch (IOException ioe) {
      // okay
    }
  }
  
  /*
   * Reads data by chunks from 2 input streams:
   * reads chunk from stream 1, and skips this chunk in the stream 2;
   * Then reads next chunk from stream 2, and skips this chunk in stream 1. 
   */
  private static byte[] readAllWithSkip(
      final int totalLength, 
      final FSDataInputStream fsdis1, 
      final FSDataInputStream fsdis2, 
      final boolean close)
      throws IOException {
    // test negative skip arg: 
    assertEquals(0, fsdis1.skip(-1));
    // test zero skip arg: 
    assertEquals(0, fsdis1.skip(0));
    
    final ByteArrayOutputStream baos = new ByteArrayOutputStream(totalLength);
    try {
      // read the data in the reverse order, from 
      // the tail to the head by pieces of 'buffer' length:
      final byte[] buffer = new byte[17];
      final int times = totalLength / buffer.length;
      final int remainder = totalLength % buffer.length;
      long skipped;
      long expectedPosition;
      int toGo;
      for (int i=0; i<=times; i++) {
        toGo = (i < times) ? buffer.length : remainder;
        if (i % 2 == 0) {
          fsdis1.readFully(buffer, 0, toGo);
          skipped = skipUntilZero(fsdis2, toGo);
        } else {
          fsdis2.readFully(buffer, 0, toGo);
          skipped = skipUntilZero(fsdis1, toGo);
        }
        if (i < times) {
          assertEquals(buffer.length, skipped);
          expectedPosition = (i + 1) * buffer.length;
        } else { 
          // remainder:
          if (remainder > 0) {
            assertEquals(remainder, skipped);
          } else {
            assertEquals(0, skipped);
          }
          expectedPosition = totalLength;
        }
        // check if the 2 streams have equal and correct positions:
        assertEquals(expectedPosition, fsdis1.getPos());
        assertEquals(expectedPosition, fsdis2.getPos());
        // save the read data:
        if (toGo > 0) {
          baos.write(buffer, 0, toGo);
        }
      }

      // finally, check up if ended stream cannot skip:
      assertEquals(0, fsdis1.skip(-1));
      assertEquals(0, fsdis1.skip(0));
      assertEquals(0, fsdis1.skip(1));
      assertEquals(0, fsdis1.skip(Long.MAX_VALUE));
      
      return baos.toByteArray();
    } finally {
      if (close) {
        fsdis1.close();
        fsdis2.close();
      }
    }
  }  
  
  private static long skipUntilZero(final FilterInputStream fis, 
      final long toSkip) throws IOException {
    long skipped = 0;
    long remainsToSkip = toSkip;
    long s;
    while (skipped < toSkip) {
      s = fis.skip(remainsToSkip); // actually skippped
      if (s == 0) {
        return skipped; // EOF or impossible to skip.
      }
      skipped += s; 
      remainsToSkip -= s;
    }
    return skipped;
  }
  
  private static byte[] prepareBin() {
    byte[] bb = new byte[77777];
    for (int i=0; i<bb.length; i++) {
      // Generate unique values, as possible:
      double d = Math.log(i + 2);
      long bits = Double.doubleToLongBits(d);
      bb[i] = (byte)bits;
    }
    return bb;
  }

  /*
   * Run the HadoopArchives tool to create an archive on the 
   * given file system.
   */
  private String makeArchive() throws Exception {
    final String inputPathStr = inputPath.toUri().getPath();
    System.out.println("inputPathStr = " + inputPathStr);

    final URI uri = fs.getUri();
    final String prefix = "har://hdfs-" + uri.getHost() + ":" + uri.getPort()
        + archivePath.toUri().getPath() + Path.SEPARATOR;

    final String harName = "foo.har";
    final String fullHarPathStr = prefix + harName;
    final String[] args = { "-archiveName", harName, "-p", inputPathStr, "*",
        archivePath.toString() };
    System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH,
        HADOOP_ARCHIVES_JAR);
    final HadoopArchives har = new HadoopArchives(conf);
    assertEquals(0, ToolRunner.run(har, args));
    return fullHarPathStr;
  }
  
  @Test
  /*
   * Tests copying from archive file system to a local file system
   */
  public void testCopyToLocal() throws Exception {
    final String fullHarPathStr = makeArchive();

    // make path to copy the file to:
    final String tmpDir
      = System.getProperty("test.build.data","build/test/data") + "/work-dir/har-fs-tmp";
    final Path tmpPath = new Path(tmpDir);
    final LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    localFs.delete(tmpPath, true);
    localFs.mkdirs(tmpPath);
    assertTrue(localFs.exists(tmpPath));
    
    // Create fresh HarFs:
    final HarFileSystem harFileSystem = new HarFileSystem(fs);
    try {
      final URI harUri = new URI(fullHarPathStr);
      harFileSystem.initialize(harUri, fs.getConf());
      
      final Path sourcePath = new Path(fullHarPathStr + Path.SEPARATOR + "a");
      final Path targetPath = new Path(tmpPath, "straus");
      // copy the Har file to a local file system:
      harFileSystem.copyToLocalFile(false, sourcePath, targetPath);
      FileStatus straus = localFs.getFileStatus(targetPath);
      // the file should contain just 1 character:
      assertEquals(1, straus.getLen());
    } finally {
      harFileSystem.close();
      localFs.delete(tmpPath, true);      
    }
  }
  
}