| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.cloudera.sqoop.hive; |
| |
| import java.util.Map; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.sqoop.util.SqlTypeMap; |
| |
| import com.cloudera.sqoop.SqoopOptions; |
| import com.cloudera.sqoop.tool.ImportTool; |
| import com.cloudera.sqoop.testutil.HsqldbTestServer; |
| |
| import org.junit.Rule; |
| import org.junit.Test; |
| import org.junit.rules.ExpectedException; |
| |
| import java.sql.Types; |
| |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertFalse; |
| import static org.junit.Assert.assertNotNull; |
| import static org.junit.Assert.assertTrue; |
| |
| |
| /** |
| * Test Hive DDL statement generation. |
| */ |
| public class TestTableDefWriter { |
| |
| public static final Log LOG = LogFactory.getLog( |
| TestTableDefWriter.class.getName()); |
| |
| @Rule |
| public ExpectedException thrown = ExpectedException.none(); |
| |
| // Test getHiveOctalCharCode and expect an IllegalArgumentException. |
| private void expectExceptionInCharCode(int charCode) { |
| thrown.expect(IllegalArgumentException.class); |
| thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException with out-of-range Hive delimiter"); |
| TableDefWriter.getHiveOctalCharCode(charCode); |
| } |
| |
| @Test |
| public void testHiveOctalCharCode() { |
| assertEquals("\\000", TableDefWriter.getHiveOctalCharCode(0)); |
| assertEquals("\\001", TableDefWriter.getHiveOctalCharCode(1)); |
| assertEquals("\\012", TableDefWriter.getHiveOctalCharCode((int) '\n')); |
| assertEquals("\\177", TableDefWriter.getHiveOctalCharCode(0177)); |
| |
| expectExceptionInCharCode(4096); |
| expectExceptionInCharCode(0200); |
| expectExceptionInCharCode(254); |
| } |
| |
| @Test |
| public void testDifferentTableNames() throws Exception { |
| Configuration conf = new Configuration(); |
| SqoopOptions options = new SqoopOptions(); |
| TableDefWriter writer = new TableDefWriter(options, null, |
| "inputTable", "outputTable", conf, false); |
| |
| Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); |
| writer.setColumnTypes(colTypes); |
| |
| String createTable = writer.getCreateTableStmt(); |
| String loadData = writer.getLoadDataStmt(); |
| |
| LOG.debug("Create table stmt: " + createTable); |
| LOG.debug("Load data stmt: " + loadData); |
| |
| // Assert that the statements generated have the form we expect. |
| assertTrue(createTable.indexOf( |
| "CREATE TABLE IF NOT EXISTS `outputTable`") != -1); |
| assertTrue(loadData.indexOf("INTO TABLE `outputTable`") != -1); |
| assertTrue(loadData.indexOf("/inputTable'") != -1); |
| } |
| |
| @Test |
| public void testDifferentTargetDirs() throws Exception { |
| String targetDir = "targetDir"; |
| String inputTable = "inputTable"; |
| String outputTable = "outputTable"; |
| |
| Configuration conf = new Configuration(); |
| SqoopOptions options = new SqoopOptions(); |
| // Specify a different target dir from input table name |
| options.setTargetDir(targetDir); |
| TableDefWriter writer = new TableDefWriter(options, null, |
| inputTable, outputTable, conf, false); |
| |
| Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); |
| writer.setColumnTypes(colTypes); |
| |
| String createTable = writer.getCreateTableStmt(); |
| String loadData = writer.getLoadDataStmt(); |
| |
| LOG.debug("Create table stmt: " + createTable); |
| LOG.debug("Load data stmt: " + loadData); |
| |
| // Assert that the statements generated have the form we expect. |
| assertTrue(createTable.indexOf( |
| "CREATE TABLE IF NOT EXISTS `" + outputTable + "`") != -1); |
| assertTrue(loadData.indexOf("INTO TABLE `" + outputTable + "`") != -1); |
| assertTrue(loadData.indexOf("/" + targetDir + "'") != -1); |
| } |
| |
| @Test |
| public void testPartitions() throws Exception { |
| String[] args = { |
| "--hive-partition-key", "ds", |
| "--hive-partition-value", "20110413", |
| }; |
| Configuration conf = new Configuration(); |
| SqoopOptions options = |
| new ImportTool().parseArguments(args, null, null, false); |
| TableDefWriter writer = new TableDefWriter(options, |
| null, "inputTable", "outputTable", conf, false); |
| |
| Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); |
| writer.setColumnTypes(colTypes); |
| |
| String createTable = writer.getCreateTableStmt(); |
| String loadData = writer.getLoadDataStmt(); |
| |
| assertNotNull(createTable); |
| assertNotNull(loadData); |
| assertEquals("CREATE TABLE IF NOT EXISTS `outputTable` ( ) " |
| + "PARTITIONED BY (ds STRING) " |
| + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\054' " |
| + "LINES TERMINATED BY '\\012' STORED AS TEXTFILE", createTable); |
| assertTrue(loadData.endsWith(" PARTITION (ds='20110413')")); |
| } |
| |
| @Test |
| public void testLzoSplitting() throws Exception { |
| String[] args = { |
| "--compress", |
| "--compression-codec", "lzop", |
| }; |
| Configuration conf = new Configuration(); |
| SqoopOptions options = |
| new ImportTool().parseArguments(args, null, null, false); |
| TableDefWriter writer = new TableDefWriter(options, |
| null, "inputTable", "outputTable", conf, false); |
| |
| Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); |
| writer.setColumnTypes(colTypes); |
| |
| String createTable = writer.getCreateTableStmt(); |
| String loadData = writer.getLoadDataStmt(); |
| |
| assertNotNull(createTable); |
| assertNotNull(loadData); |
| assertEquals("CREATE TABLE IF NOT EXISTS `outputTable` ( ) " |
| + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\054' " |
| + "LINES TERMINATED BY '\\012' STORED AS " |
| + "INPUTFORMAT 'com.hadoop.mapred.DeprecatedLzoTextInputFormat' " |
| + "OUTPUTFORMAT " |
| + "'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'", |
| createTable); |
| } |
| |
| @Test |
| public void testUserMappingNoDecimal() throws Exception { |
| String[] args = { |
| "--map-column-hive", "id=STRING,value=INTEGER", |
| }; |
| Configuration conf = new Configuration(); |
| SqoopOptions options = |
| new ImportTool().parseArguments(args, null, null, false); |
| TableDefWriter writer = new TableDefWriter(options, |
| null, HsqldbTestServer.getTableName(), "outputTable", conf, false); |
| |
| Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); |
| colTypes.put("id", Types.INTEGER); |
| colTypes.put("value", Types.VARCHAR); |
| writer.setColumnTypes(colTypes); |
| |
| String createTable = writer.getCreateTableStmt(); |
| |
| assertNotNull(createTable); |
| |
| assertTrue(createTable.contains("`id` STRING")); |
| assertTrue(createTable.contains("`value` INTEGER")); |
| |
| assertFalse(createTable.contains("`id` INTEGER")); |
| assertFalse(createTable.contains("`value` STRING")); |
| } |
| |
| @Test |
| public void testUserMappingWithDecimal() throws Exception { |
| String[] args = { |
| "--map-column-hive", "id=STRING,value2=DECIMAL(13,5),value1=INTEGER," + |
| "value3=DECIMAL(4,5),value4=VARCHAR(255)", |
| }; |
| Configuration conf = new Configuration(); |
| SqoopOptions options = |
| new ImportTool().parseArguments(args, null, null, false); |
| TableDefWriter writer = new TableDefWriter(options, |
| null, HsqldbTestServer.getTableName(), "outputTable", conf, false); |
| |
| Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); |
| colTypes.put("id", Types.INTEGER); |
| colTypes.put("value1", Types.VARCHAR); |
| colTypes.put("value2", Types.DOUBLE); |
| colTypes.put("value3", Types.FLOAT); |
| colTypes.put("value4", Types.CHAR); |
| writer.setColumnTypes(colTypes); |
| |
| String createTable = writer.getCreateTableStmt(); |
| |
| assertNotNull(createTable); |
| |
| assertTrue(createTable.contains("`id` STRING")); |
| assertTrue(createTable.contains("`value1` INTEGER")); |
| assertTrue(createTable.contains("`value2` DECIMAL(13,5)")); |
| assertTrue(createTable.contains("`value3` DECIMAL(4,5)")); |
| assertTrue(createTable.contains("`value4` VARCHAR(255)")); |
| |
| assertFalse(createTable.contains("`id` INTEGER")); |
| assertFalse(createTable.contains("`value1` STRING")); |
| assertFalse(createTable.contains("`value2` DOUBLE")); |
| assertFalse(createTable.contains("`value3` FLOAT")); |
| assertFalse(createTable.contains("`value4` CHAR")); |
| } |
| |
| @Test |
| public void testUserMappingFailWhenCantBeApplied() throws Exception { |
| String[] args = { |
| "--map-column-hive", "id=STRING,value=INTEGER", |
| }; |
| Configuration conf = new Configuration(); |
| SqoopOptions options = |
| new ImportTool().parseArguments(args, null, null, false); |
| TableDefWriter writer = new TableDefWriter(options, |
| null, HsqldbTestServer.getTableName(), "outputTable", conf, false); |
| |
| Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); |
| colTypes.put("id", Types.INTEGER); |
| writer.setColumnTypes(colTypes); |
| |
| thrown.expect(IllegalArgumentException.class); |
| thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException on non applied Hive type mapping"); |
| String createTable = writer.getCreateTableStmt(); |
| } |
| |
| @Test |
| public void testHiveDatabase() throws Exception { |
| String[] args = { |
| "--hive-database", "db", |
| }; |
| Configuration conf = new Configuration(); |
| SqoopOptions options = |
| new ImportTool().parseArguments(args, null, null, false); |
| TableDefWriter writer = new TableDefWriter(options, |
| null, HsqldbTestServer.getTableName(), "outputTable", conf, false); |
| |
| Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); |
| writer.setColumnTypes(colTypes); |
| |
| String createTable = writer.getCreateTableStmt(); |
| assertNotNull(createTable); |
| assertTrue(createTable.contains("`db`.`outputTable`")); |
| |
| String loadStmt = writer.getLoadDataStmt(); |
| assertNotNull(loadStmt); |
| assertTrue(createTable.contains("`db`.`outputTable`")); |
| } |
| |
| } |