cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.cli;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hive.ql.io.RCFile;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.junit.Test;

/**
 * test RCFileCat
 *
 */
public class TestRCFileCat {

  /**
   * test parse file
   */
  @Test
  public void testRCFileCat() throws Exception {
    File template = File.createTempFile("hive", "tmpTest");
    Configuration configuration = new Configuration();

    byte[][] record_1 = { Bytes.toBytes("123"), Bytes.toBytes("456"),
        Bytes.toBytes("789"), Bytes.toBytes("1000"), Bytes.toBytes("5.3"),
        Bytes.toBytes("hive and hadoop"), new byte[0], Bytes.toBytes("NULL") };
    byte[][] record_2 = { Bytes.toBytes("100"), Bytes.toBytes("200"),
        Bytes.toBytes("123"), Bytes.toBytes("1000"), Bytes.toBytes("5.3"),
        Bytes.toBytes("hive and hadoop"), new byte[0], Bytes.toBytes("NULL") };
    byte[][] record_3 = { Bytes.toBytes("200"), Bytes.toBytes("400"),
        Bytes.toBytes("678"), Bytes.toBytes("1000"), Bytes.toBytes("4.8"),
        Bytes.toBytes("hive and hadoop"), new byte[0], Bytes.toBytes("TEST") };

    RCFileOutputFormat.setColumnNumber(configuration, 8);

    Path file = new Path(template.getAbsolutePath());

    FileSystem fs = FileSystem.getLocal(configuration);
    RCFile.Writer writer = new RCFile.Writer(fs, configuration, file, null,
        RCFile.createMetadata(new Text("apple"), new Text("block"), new Text(
            "cat"), new Text("dog")), new DefaultCodec());
    write(writer, record_1);
    write(writer, record_2);
    write(writer, record_3);
    writer.close();

    RCFileCat fileCat = new RCFileCat();
    fileCat.test=true;
    fileCat.setConf(new Configuration());

    // set fake input and output streams
    PrintStream oldOutPrintStream= System.out;
    PrintStream oldErrPrintStream= System.err;
    ByteArrayOutputStream dataOut= new ByteArrayOutputStream();
    ByteArrayOutputStream dataErr= new ByteArrayOutputStream();
    System.setOut(new PrintStream(dataOut));
    System.setErr(new PrintStream(dataErr));


    try {
      String[] params = {"--verbose","file://" + template.toURI().getPath() };

      assertEquals(0, fileCat.run(params));
      assertTrue(dataOut.toString().contains("123\t456\t789\t1000\t5.3\thive and hadoop\t\tNULL"));
      assertTrue(dataOut.toString().contains("100\t200\t123\t1000\t5.3\thive and hadoop\t\tNULL"));
      assertTrue(dataOut.toString().contains("200\t400\t678\t1000\t4.8\thive and hadoop\t\tTEST"));
      dataOut.reset();
       params = new String[] { "--start=-10","--file-sizes", "file://" + template.toURI().getPath() };
      assertEquals(0, fileCat.run(params));
      assertTrue(dataOut.toString().contains("File size (uncompressed): 105. File size (compressed): 134. Number of rows: 3."));
      dataOut.reset();

      params = new String[] {"--start=0", "--column-sizes","file://" + template.toURI().getPath() };
      assertEquals(0, fileCat.run(params));
      assertTrue(dataOut.toString().contains("0\t9\t17"));
      assertTrue(dataOut.toString().contains("1\t9\t17"));
      assertTrue(dataOut.toString().contains("2\t9\t17"));
      assertTrue(dataOut.toString().contains("3\t12\t14"));
      assertTrue(dataOut.toString().contains("4\t9\t17"));
      assertTrue(dataOut.toString().contains("5\t45\t26"));


      dataOut.reset();
      params = new String[] {"--start=0", "--column-sizes-pretty",
          "file://" + template.toURI().getPath() };
      assertEquals(0, fileCat.run(params));
      assertTrue(dataOut.toString().contains("Column 0: Uncompressed size: 9 Compressed size: 17"));
      assertTrue(dataOut.toString().contains("Column 1: Uncompressed size: 9 Compressed size: 17"));
      assertTrue(dataOut.toString().contains("Column 2: Uncompressed size: 9 Compressed size: 17"));
      assertTrue(dataOut.toString().contains("Column 3: Uncompressed size: 12 Compressed size: 14"));
      assertTrue(dataOut.toString().contains("Column 4: Uncompressed size: 9 Compressed size: 17"));
      assertTrue(dataOut.toString().contains("Column 5: Uncompressed size: 45 Compressed size: 26"));

      params = new String[] { };
      assertEquals(-1, fileCat.run(params));
      assertTrue(dataErr.toString().contains("RCFileCat [--start=start_offet] [--length=len] [--verbose] " +
          "[--column-sizes | --column-sizes-pretty] [--file-sizes] fileName"));

      dataErr.reset();
      params = new String[] { "--fakeParameter",
          "file://" + template.toURI().getPath()};
      assertEquals(-1, fileCat.run(params));
      assertTrue(dataErr.toString().contains("RCFileCat [--start=start_offet] [--length=len] [--verbose] " +
          "[--column-sizes | --column-sizes-pretty] [--file-sizes] fileName"));

    } finally {
      // restore  input and output streams
      System.setOut(oldOutPrintStream);
      System.setErr(oldErrPrintStream);
    }

  }

  private void write(RCFile.Writer writer, byte[][] record) throws IOException {
    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record.length);
    for (int i = 0; i < record.length; i++) {
      BytesRefWritable cu = new BytesRefWritable(record[i], 0, record[i].length);
      bytes.set(i, cu);
    }
    writer.append(bytes);

  }
}