itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hive.jdbc;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.session.HiveSessionHook;
import org.apache.hive.service.cli.session.HiveSessionHookContext;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

public class TestJdbcWithMiniMr {
  public static final String TEST_TAG = "miniHS2.miniMr.tag";
  public static final String TEST_TAG_VALUE = "miniHS2.miniMr.value";
  public static class MiniMrTestSessionHook implements HiveSessionHook {
    @Override
    public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
      sessionHookContext.getSessionConf().set(TEST_TAG, TEST_TAG_VALUE);
    }
  }

  private static MiniHS2 miniHS2 = null;
  private static HiveConf conf;
  private static Path dataFilePath;
  private static String  dbName = "mrTestDb";
  private Connection hs2Conn = null;
  private Statement stmt;

  @BeforeClass
  public static void beforeTest() throws Exception {
    Class.forName(MiniHS2.getJdbcDriverName());
    conf = new HiveConf();
    conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
    String dataFileDir = conf.get("test.data.files").replace('\\', '/')
        .replace("c:", "");
    dataFilePath = new Path(dataFileDir, "kv1.txt");
    DriverManager.setLoginTimeout(0);
    conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
    miniHS2 = new MiniHS2(conf, MiniClusterType.MR);
    Map<String, String> overlayProps = new HashMap<String, String>();
    overlayProps.put(ConfVars.HIVE_SERVER2_SESSION_HOOK.varname,
        MiniMrTestSessionHook.class.getName());
    miniHS2.start(overlayProps);
    createDb();
  }

  // setup DB
  private static void createDb() throws Exception {
    Connection conn = DriverManager.
        getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
    Statement stmt2 = conn.createStatement();
    stmt2.execute("DROP DATABASE IF EXISTS " + dbName + " CASCADE");
    stmt2.execute("CREATE DATABASE " + dbName);
    stmt2.close();
    conn.close();
  }

  @Before
  public void setUp() throws Exception {
    hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
        System.getProperty("user.name"), "bar");
    stmt = hs2Conn.createStatement();
    stmt.execute("USE " + dbName);
  }

  @After
  public void tearDown() throws Exception {
    if (hs2Conn != null) {
      hs2Conn.close();
    }
  }

  @AfterClass
  public static void afterTest() throws Exception {
    if (miniHS2 != null && miniHS2.isStarted()) {
      miniHS2.stop();
    }
  }

  /**
   * Verify that the connection to HS2 with MiniMr is successful
   * @throws Exception
   */
  @Test
  public void testConnection() throws Exception {
    // the session hook should set the property
    verifyProperty(TEST_TAG, TEST_TAG_VALUE);
  }

  /**
   * Run nonMr query
   * @throws Exception
   */
  @Test
  public void testNonMrQuery() throws Exception {
    String tableName = "testTab1";
    String resultVal = "val_238";
    String queryStr = "SELECT * FROM " + tableName;

    testKvQuery(tableName, queryStr, resultVal);
  }

  /**
   * Run nonMr query
   * @throws Exception
   */
  @Test
  public void testMrQuery() throws Exception {
    String tableName = "testTab2";
    String resultVal = "val_238";
    String queryStr = "SELECT * FROM " + tableName +
        " where value = '" + resultVal + "'";

    testKvQuery(tableName, queryStr, resultVal);
  }

  @Test
  public void testPermFunc() throws Exception {

    // This test assumes the hive-contrib JAR has been built as part of the Hive build.
    // Also dependent on the UDFExampleAdd class within that JAR.
    String udfClassName = "org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd";
    String mvnRepo = System.getProperty("maven.local.repository");
    String hiveVersion = System.getProperty("hive.version");
    String jarFileName = "hive-contrib-" + hiveVersion + ".jar";
    String[] pathParts = {
        "org", "apache", "hive",
        "hive-contrib", hiveVersion, jarFileName
    };

    // Create path to hive-contrib JAR on local filesystem
    Path contribJarPath = new Path(mvnRepo);
    for (String pathPart : pathParts) {
      contribJarPath = new Path(contribJarPath, pathPart);
    }
    FileSystem localFs = FileSystem.getLocal(conf);
    assertTrue("Hive contrib JAR exists at " + contribJarPath, localFs.exists(contribJarPath));

    String hdfsJarPathStr = "hdfs:///" + jarFileName;
    Path hdfsJarPath = new Path(hdfsJarPathStr);

    // Copy JAR to DFS
    FileSystem dfs = miniHS2.getDFS().getFileSystem();
    dfs.copyFromLocalFile(contribJarPath, hdfsJarPath);
    assertTrue("Verify contrib JAR copied to HDFS at " + hdfsJarPath, dfs.exists(hdfsJarPath));

    // Register function
    String queryStr = "CREATE FUNCTION example_add AS '" + udfClassName + "'"
        + " USING JAR '" + hdfsJarPathStr + "'";
    stmt.execute(queryStr);

    // Call describe
    ResultSet res;
    res = stmt.executeQuery("DESCRIBE FUNCTION " + dbName + ".example_add");
    checkForNotExist(res);

    // Use UDF in query
    String tableName = "testTab3";
    setupKv1Tabs(tableName);
    res = stmt.executeQuery("SELECT EXAMPLE_ADD(1, 2) FROM " + tableName + " LIMIT 1");
    assertTrue("query has results", res.next());
    assertEquals(3, res.getInt(1));
    assertFalse("no more results", res.next());

    // A new connection should be able to call describe/use function without issue
    Connection conn2 = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
        System.getProperty("user.name"), "bar");
    Statement stmt2 = conn2.createStatement();
    stmt2.execute("USE " + dbName);
    res = stmt2.executeQuery("DESCRIBE FUNCTION " + dbName + ".example_add");
    checkForNotExist(res);

    res = stmt2.executeQuery("SELECT " + dbName + ".example_add(1, 1) FROM " + tableName + " LIMIT 1");
    assertTrue("query has results", res.next());
    assertEquals(2, res.getInt(1));
    assertFalse("no more results", res.next());

    stmt.execute("DROP TABLE " + tableName);
  }

  @Test
  public void testTempTable() throws Exception {
    // Create temp table with current connection
    String tempTableName = "tmp1";
    stmt.execute("CREATE TEMPORARY TABLE " + tempTableName + " (key string, value string)");
    stmt.execute("load data local inpath '"
        + dataFilePath.toString() + "' into table " + tempTableName);

    String resultVal = "val_238";
    String queryStr = "SELECT * FROM " + tempTableName +
        " where value = '" + resultVal + "'";
    verifyResult(queryStr, resultVal, 2);


    // Test getTables()
    DatabaseMetaData md = hs2Conn.getMetaData();
    assertTrue(md.getConnection() == hs2Conn);

    ResultSet rs = md.getTables(null, null, tempTableName, null);
    boolean foundTable = false;
    while (rs.next()) {
      String tableName = rs.getString(3);
      if (tableName.equalsIgnoreCase(tempTableName)) {
        assertFalse("Table not found yet", foundTable);
        foundTable = true;
      }
    }
    assertTrue("Found temp table", foundTable);

    // Test getTables() with no table name pattern
    rs = md.getTables(null, null, null, null);
    foundTable = false;
    while (rs.next()) {
      String tableName = rs.getString(3);
      if (tableName.equalsIgnoreCase(tempTableName)) {
        assertFalse("Table not found yet", foundTable);
        foundTable = true;
      }
    }
    assertTrue("Found temp table", foundTable);

    // Test getColumns()
    rs = md.getColumns(null, null, tempTableName, null);
    assertTrue("First row", rs.next());
    assertTrue(rs.getString(3).equalsIgnoreCase(tempTableName));
    assertTrue(rs.getString(4).equalsIgnoreCase("key"));
    assertEquals(Types.VARCHAR, rs.getInt(5));

    assertTrue("Second row", rs.next());
    assertTrue(rs.getString(3).equalsIgnoreCase(tempTableName));
    assertTrue(rs.getString(4).equalsIgnoreCase("value"));
    assertEquals(Types.VARCHAR, rs.getInt(5));

    // A second connection should not be able to see the table
    Connection conn2 = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
        System.getProperty("user.name"), "bar");
    Statement stmt2 = conn2.createStatement();
    stmt2.execute("USE " + dbName);
    boolean gotException = false;
    try {
      ResultSet res;
      res = stmt2.executeQuery(queryStr);
    } catch (SQLException err) {
      // This is expected to fail.
      assertTrue("Expecting table not found error, instead got: " + err,
          err.getMessage().contains("Table not found"));
      gotException = true;
    }
    assertTrue("Exception while querying non-existing temp table", gotException);
  }

  private void checkForNotExist(ResultSet res) throws Exception {
    int numRows = 0;
    while (res.next()) {
      numRows++;
      String strVal = res.getString(1);
      assertEquals("Should not find 'not exist'", -1, strVal.toLowerCase().indexOf("not exist"));
    }
    assertTrue("Rows returned from describe function", numRows > 0);
  }

  /**
   * Verify if the given property contains the expected value
   * @param propertyName
   * @param expectedValue
   * @throws Exception
   */
  private void verifyProperty(String propertyName, String expectedValue) throws Exception {
    Statement stmt = hs2Conn .createStatement();
    ResultSet res = stmt.executeQuery("set " + propertyName);
    assertTrue(res.next());
    String results[] = res.getString(1).split("=");
    assertEquals("Property should be set", results.length, 2);
    assertEquals("Property should be set", expectedValue, results[1]);
  }

  // create tables, verify query
  private void testKvQuery(String tableName, String queryStr, String resultVal)
      throws SQLException {
    setupKv1Tabs(tableName);
    verifyResult(queryStr, resultVal, 2);
    stmt.execute("DROP TABLE " + tableName);
  }

  // create table and pupulate with kv1.txt
  private void setupKv1Tabs(String tableName) throws SQLException {
    Statement stmt = hs2Conn.createStatement();
    // create table
    stmt.execute("CREATE TABLE " + tableName
        + " (under_col INT COMMENT 'the under column', value STRING)"
        + " COMMENT ' test table'");

    // load data
    stmt.execute("load data local inpath '"
        + dataFilePath.toString() + "' into table " + tableName);
  }

  // run given query and validate expecated result
  private void verifyResult(String queryStr, String expString, int colPos)
      throws SQLException {
    ResultSet res = stmt.executeQuery(queryStr);
    assertTrue(res.next());
    assertEquals(expString, res.getString(colPos));
    res.close();
  }
}