testdata/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.impala.datagenerator;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;

// Converts CSV files for the AllTypes table into HBase shell commands that insert the same data.
// Reads all CSV files from the given input directory and write a single file containing all HBase commands.
public class CsvToHBaseConverter {

  public static void convertAllTypesTable(String inDir, String outDir, String hbaseTableName) {
    File inDirFile = new File(inDir);
    File outDirFile = new File(outDir);
    if (!outDirFile.exists()) {
      outDirFile.mkdirs();
    }
    try {
      FileWriter outFile = new FileWriter(outDirFile.getAbsolutePath() + "/"
          + hbaseTableName + ".hbase");
      PrintWriter out = new PrintWriter(outFile);
      String[] children = inDirFile.list();
      for (String csvFile : children) {
        if (!csvFile.endsWith(".txt")) continue;

        FileInputStream finstream = new FileInputStream(
            inDirFile.getAbsolutePath() + "/" + csvFile);
        DataInputStream in = new DataInputStream(finstream);
        BufferedReader br = new BufferedReader(new InputStreamReader(in));
        String csvLine;
        while ((csvLine = br.readLine()) != null) {
          convertLine(csvLine, out, hbaseTableName);
        }        
        in.close();
      }
      out.close();
    } catch (FileNotFoundException e) {
      e.printStackTrace();
      System.exit(-1);
    } catch (IOException e) {
      e.printStackTrace();
      System.exit(-1);
    }
  }
  
  public static void convertLine(String csvLine, PrintWriter out, String hbaseTableName) throws IOException {
    String[] parts = csvLine.split(",");
    if (parts.length != 11) {
      throw new IOException("Line does not have 11 columns: " + csvLine);
    }        
    
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:bigint_col", parts[5]);
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:bool_col", parts[1]);
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:date_string_col", parts[8]);
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:double_col", parts[7]);
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:float_col", parts[6]);
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:int_col", parts[4]);
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:smallint_col", parts[3]);
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:string_col", parts[9]);
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:timestamp_col", parts[10]);
    out.format("put '%s', '%s', '%s', '%s'\n", hbaseTableName, parts[0], "d:tinyint_col", parts[2]);
  }
  
  
  /**
   * Convert CSV test data in a file with HBase shell commands for inserting the same data. 
   * 
   * @param BaseOutputDirectory
   *          : Required base output folder of generated data files.
   * @throws Exception
   *           something bad happened
   */
  public static void main(String args[]) throws Exception {
    convertAllTypesTable("AllTypesError", "HBaseAllTypesError", "functional_hbase.hbasealltypeserror");
    convertAllTypesTable("AllTypesErrorNoNulls", "HBaseAllTypesErrorNoNulls", 
        "functional_hbase.hbasealltypeserrornonulls");
    System.exit(0);
  }
}