SQOOP-3027: Create fail fast for export with --hcatalog-table <HIVE_VIEW>, as it's not supported by HCat
(Anna Szonyi via Attila Szabo)
diff --git a/build.xml b/build.xml
index b1545b6..6e8d393 100644
--- a/build.xml
+++ b/build.xml
@@ -194,11 +194,12 @@
<property name="hbase95.version" value="1.2.0" />
<property name="zookeeper.version" value="3.4.5" />
<property name="hadoop.version.full" value="2.6.0" />
- <property name="hcatalog.version" value="0.13.0" />
+ <property name="hcatalog.version" value="1.2.0" />
<property name="hbasecompatprofile" value="2" />
<property name="avrohadoopprofile" value="2" />
</then>
</elseif>
+
<else>
<fail message="Unrecognized hadoopversion. Can only be 20, 23, 100, 200 or 210." />
</else>
diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
index ebe7209..2101b06 100644
--- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
+++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
@@ -40,6 +40,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.HadoopShims.HCatHadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -56,6 +59,7 @@
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.hcatalog.common.HCatConstants;
+import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.data.DefaultHCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
@@ -66,7 +70,6 @@
import org.apache.sqoop.config.ConfigurationHelper;
import org.apache.sqoop.hive.HiveTypes;
import org.apache.sqoop.manager.ConnManager;
-import org.apache.sqoop.tool.BaseSqoopTool;
import org.apache.sqoop.util.Executor;
import org.apache.sqoop.util.LoggingAsyncSink;
import org.apache.sqoop.util.SubprocessSecurityManager;
@@ -158,6 +161,43 @@
private static boolean testMode = false;
+ public static boolean isHCatView(final SqoopOptions opts) {
+
+ String hCatDatabaseName = getHCatDbNameFromOptions(opts);
+
+ String hCatTableName = getHCatTableNameFromOptions(opts);
+
+ Configuration conf = opts.getConf();
+ HiveConf hiveConf;
+ try {
+ if (conf != null) {
+ hiveConf = HCatUtil.getHiveConf(conf);
+ } else {
+ hiveConf = new HiveConf(HCatInputFormat.class);
+ }
+ HiveMetaStoreClient client = HCatUtil.getHiveClient(hiveConf);
+ Table table = HCatUtil.getTable(client, hCatDatabaseName, hCatTableName);
+ return table.isView();
+ }
+ catch(Exception e){
+ LOG.warn("We were not able to determine if "+hCatDatabaseName+ ":"+hCatTableName+ "is view or table.");
+ LOG.info("isHCatView threw an exception:", e);
+ return false;
+ }
+ }
+
+ private static String getHCatTableNameFromOptions(SqoopOptions opts) {
+ String optHCTabName = opts.getHCatTableName();
+ return optHCTabName.toLowerCase();
+ }
+
+ private static String getHCatDbNameFromOptions(SqoopOptions opts) {
+ String hCatDatabaseName = opts.getHCatDatabaseName() != null ? opts
+ .getHCatDatabaseName() : DEFHCATDB;
+ hCatDatabaseName = hCatDatabaseName.toLowerCase();
+ return hCatDatabaseName;
+ }
+
static class IntArrayWritable extends ArrayWritable {
public IntArrayWritable() {
super(IntWritable.class);
@@ -278,12 +318,10 @@
dbTableName = dbTable;
configuration = config;
hCatJob = job;
- hCatDatabaseName = options.getHCatDatabaseName() != null ? options
- .getHCatDatabaseName() : DEFHCATDB;
- hCatDatabaseName = hCatDatabaseName.toLowerCase();
+ hCatDatabaseName = getHCatDbNameFromOptions(opts);
String optHCTabName = options.getHCatTableName();
- hCatTableName = optHCTabName.toLowerCase();
+ hCatTableName = getHCatTableNameFromOptions(opts);
if (!hCatTableName.equals(optHCTabName)) {
LOG.warn("Provided HCatalog table name " + optHCTabName
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 468bf34..235f67a 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -33,6 +33,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
+import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities;
import org.apache.sqoop.util.CredentialsUtil;
import org.apache.sqoop.util.LoggingUtils;
import org.apache.sqoop.util.password.CredentialProviderHelper;
@@ -1560,6 +1561,9 @@
}
return;
}
+ if(isSet(options.getHCatTableName()) && SqoopHCatUtilities.isHCatView(options)){
+ throw new InvalidOptionsException("Reads/Writes from and to Views are not supported by HCatalog");
+ }
if (options.explicitInputDelims()) {
LOG.warn("Input field/record delimiter options are not "
@@ -1672,6 +1676,10 @@
}
}
+ private boolean isSet(String option) {
+ return org.apache.commons.lang.StringUtils.isNotBlank(option);
+ }
+
protected void validateHBaseOptions(SqoopOptions options)
throws InvalidOptionsException {
if ((options.getHBaseColFamily() != null && options.getHBaseTable() == null)
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java
index ff3dde1..5ef9b2b 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java
@@ -32,8 +32,11 @@
import java.util.Map;
import java.util.TimeZone;
+import com.cloudera.sqoop.SqoopOptions;
+import junit.framework.JUnit4TestAdapter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
@@ -46,14 +49,24 @@
import org.junit.Before;
import com.cloudera.sqoop.testutil.ExportJobTestCase;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
/**
* Test that we can export HCatalog tables into databases.
*/
+@RunWith(JUnit4.class)
public class HCatalogExportTest extends ExportJobTestCase {
private static final Log LOG =
LogFactory.getLog(HCatalogExportTest.class);
private HCatalogTestUtils utils = HCatalogTestUtils.instance();
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
@Before
@Override
public void setUp() {
@@ -142,6 +155,7 @@
}
}
+ @Test
public void testIntTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -166,6 +180,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testExportWithColumnNameValue() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -182,7 +197,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
-
+ @Test
public void testFloatTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -201,6 +216,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testNumberTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -220,6 +236,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testDateTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -248,6 +265,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testDateTypesToBigInt() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
long offset = TimeZone.getDefault().getRawOffset();
@@ -270,6 +288,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testStringTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -293,7 +312,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
-
+ @Test
public void testBinaryTypes() throws Exception {
ByteBuffer bb = ByteBuffer.wrap(new byte[] { 0, 1, 2 });
final int TOTAL_RECORDS = 1 * 10;
@@ -310,6 +329,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testColumnProjection() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -324,6 +344,8 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+
+ @Test
public void testStaticPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -341,6 +363,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testStaticPartitioningWithMultipleKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -361,6 +384,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -374,6 +398,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testStaticAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -394,6 +419,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -420,6 +446,7 @@
/**
* Test other file formats.
*/
+ @Test
public void testSequenceFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -440,6 +467,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testTextFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -460,6 +488,7 @@
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testPublishExportJobData() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -479,4 +508,63 @@
assert (DummyDataPublisher.storeType.equals("hsqldb"));
assert (DummyDataPublisher.operation.equals("export"));
}
+
+ @Test
+ public void testWeCanTellIfHCatViewOrTable() throws Exception {
+ String tableName = getTableName().toUpperCase();
+ String viewName = "view";
+ SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
+ createHCatTableAndView(tableName, viewName, utils);
+
+ Configuration conf = getConf();
+ conf.set("oraoop.disabled", "true");
+ SqoopOptions opts = getSqoopOptions(conf);
+ opts.setHCatTableName(tableName);
+
+ assertFalse(utils.isHCatView(opts));
+ opts.setHCatTableName(viewName);
+ assertTrue(utils.isHCatView(opts));
+ }
+
+ @Test
+ public void testExportHCatViewThrowsException() throws Exception {
+ String tableName = getTableName().toUpperCase();
+ String viewName = "view";
+ SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
+ createHCatTableAndView(tableName, viewName, utils);
+
+ ArrayList<String> args = new ArrayList<String>();
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--hcatalog-table");
+ args.add(viewName);
+ args.add("--connect");
+ args.add(getConnectString());
+
+ args.toArray(new String[0]);
+
+ SqoopHCatUtilities.instance().setConfigured(false);
+ exception.expect(java.io.IOException.class);
+ runExport(args.toArray(new String[0]));
+ }
+
+ private void createHCatTableAndView(String tableName, String viewName, SqoopHCatUtilities utils) throws Exception {
+ HCatalogTestUtils testUtils = HCatalogTestUtils.instance();
+ ColumnGenerator[] cols = new ColumnGenerator[]{
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
+ "boolean", Types.BOOLEAN, HCatFieldSchema.Type.BOOLEAN, 0, 0,
+ Boolean.TRUE, Boolean.TRUE, KeyType.NOT_A_KEY),
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
+ "int", Types.INTEGER, HCatFieldSchema.Type.INT, 5, 5, 10,
+ 10, KeyType.NOT_A_KEY)
+ };
+ testUtils.createHCatTable(CreateMode.CREATE_AND_LOAD, 10, tableName, cols);
+ String createViewCmd = "drop view " + viewName + "; create view " + viewName + " as select * from " + tableName;
+ utils.launchHCatCli(createViewCmd);
+ }
+
+ public static junit.framework.Test suite() {
+ return new JUnit4TestAdapter(HCatalogImportTest.class);
+ }
+
}
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
index fe5295a..28f83f2 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
@@ -35,6 +35,7 @@
import java.util.Map;
import java.util.TimeZone;
+import junit.framework.JUnit4TestAdapter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -57,10 +58,16 @@
import com.cloudera.sqoop.testutil.ImportJobTestCase;
import com.cloudera.sqoop.tool.ImportTool;
import com.cloudera.sqoop.tool.SqoopTool;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
/**
* Test that we can export HCatalog tables into databases.
*/
+@RunWith(JUnit4.class)
public class HCatalogImportTest extends ImportJobTestCase {
private static final Log LOG =
LogFactory.getLog(HCatalogImportTest.class);
@@ -68,6 +75,9 @@
private List<String> extraTestArgs = null;
private List<String> configParams = null;
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
@Override
@Before
public void setUp() {
@@ -343,6 +353,7 @@
validateHCatRecords(recs, tblSchema, 10, cols);
}
+ @Test
public void testIntTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -368,6 +379,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testFloatTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -387,6 +399,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testNumberTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -407,6 +420,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testDateTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -436,6 +450,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testDateTypesToBigInt() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
long offset = TimeZone.getDefault().getRawOffset();
@@ -459,6 +474,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testStringTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -483,14 +499,16 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testBinaryTypes() throws Exception {
ByteBuffer bb = ByteBuffer.wrap(new byte[] { 0, 1, 2 });
+ ByteBuffer bb10 = ByteBuffer.wrap(new byte[]{0, 1, 2, 0, 0, 0, 0, 0, 0, 0});
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
ColumnGenerator[] cols = new ColumnGenerator[] {
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
"binary(10)", Types.BINARY, HCatFieldSchema.Type.BINARY, 0, 0,
- bb.array(), bb.array(), KeyType.NOT_A_KEY),
+ bb10.array(), bb10.array(), KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
"longvarbinary", Types.BINARY, HCatFieldSchema.Type.BINARY, 0, 0,
bb.array(), bb.array(), KeyType.NOT_A_KEY),
@@ -500,6 +518,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testColumnProjection() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -518,6 +537,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, colNames);
}
+ @Test
public void testColumnProjectionMissingPartKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -543,6 +563,7 @@
LOG.info("Exception stack trace = " + sw);
}
}
+ @Test
public void testStaticPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -560,6 +581,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testStaticPartitioningWithMultipleKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -580,6 +602,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -593,6 +616,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testStaticAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -613,6 +637,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -639,6 +664,7 @@
/**
* Test other file formats.
*/
+ @Test
public void testSequenceFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -659,6 +685,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testTextFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -680,6 +707,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testTableCreation() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -694,10 +722,12 @@
List<String> addlArgsArray = new ArrayList<String>();
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
null, true, false);
}
+ @Test
public void testTableCreationWithPartition() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -716,9 +746,11 @@
addlArgsArray.add("2");
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
+ @Test
public void testTableCreationWithMultipleStaticPartKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -737,9 +769,11 @@
addlArgsArray.add("1,2");
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
+ @Test
public void testTableCreationWithStorageStanza() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -760,9 +794,11 @@
addlArgsArray.add("--hcatalog-storage-stanza");
addlArgsArray.add(HCatalogTestUtils.STORED_AS_TEXT);
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
+ @Test
public void testHiveDropDelims() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -780,6 +816,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testHiveDelimsReplacement() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -798,6 +835,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testDynamicKeyInMiddle() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -814,6 +852,7 @@
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testQueryImport() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -830,6 +869,7 @@
runHCatQueryImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testCreateTableWithPreExistingTable() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -856,6 +896,7 @@
}
}
+ @Test
public void testDropAndCreateWithPreExistingTable() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -875,6 +916,7 @@
null, true, false);
}
+ @Test
public void testDropAndCreateWithoutPreExistingTable() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -893,6 +935,7 @@
null, true, false);
}
+ @Test
public void testTableWithNonIdentColChars() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -905,6 +948,8 @@
setExtraArgs(addlArgsArray);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+
+ @Test
public void testTableCreationWithNonIdentColChars() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -916,10 +961,12 @@
List<String> addlArgsArray = new ArrayList<String>();
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
null, true, false);
}
+ @Test
public void testPublishQueryImportData() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -941,6 +988,7 @@
assert (DummyDataPublisher.storeTable.equals(getTableName()));
}
+ @Test
public void testPublishTableImportData() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -959,9 +1007,70 @@
List<String> addlArgsArray = new ArrayList<String>();
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
assert (DummyDataPublisher.storeType.equals("hsqldb"));
assert (DummyDataPublisher.operation.equals("import"));
assert (DummyDataPublisher.storeTable.equals(getTableName()));
}
+
+ @Test
+ public void testWeCanTellIfHCatViewOrTable() throws Exception {
+ String tableName = getTableName().toUpperCase();
+ String viewName = "view";
+ SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
+ createHCatTableAndView(tableName, viewName, utils);
+
+ Configuration conf = getConf();
+ conf.set("oraoop.disabled", "true");
+ SqoopOptions opts = getSqoopOptions(conf);
+ opts.setHCatTableName(tableName);
+
+ assertFalse(utils.isHCatView(opts));
+ opts.setHCatTableName(viewName);
+ assertTrue(utils.isHCatView(opts));
+ }
+
+ @Test
+ public void testImportHCatViewThrowsException() throws Exception {
+ String tableName = getTableName().toUpperCase();
+ String viewName = "view";
+ SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
+ createHCatTableAndView(tableName, viewName, utils);
+
+ ArrayList<String> args = new ArrayList<String>();
+
+ args.addAll(getConfigParams());
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--hcatalog-table");
+ args.add(viewName);
+ args.add("--connect");
+ args.add(getConnectString());
+
+ args.toArray(new String[0]);
+
+ SqoopHCatUtilities.instance().setConfigured(false);
+ exception.expect(java.io.IOException.class);
+ runImport(new ImportTool(), args.toArray(new String[0]));
+ }
+
+ private void createHCatTableAndView(String tableName, String viewName, SqoopHCatUtilities utils) throws Exception {
+ HCatalogTestUtils testUtils = HCatalogTestUtils.instance();
+ ColumnGenerator[] cols = new ColumnGenerator[]{
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
+ "boolean", Types.BOOLEAN, HCatFieldSchema.Type.BOOLEAN, 0, 0,
+ Boolean.TRUE, Boolean.TRUE, KeyType.NOT_A_KEY),
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
+ "int", Types.INTEGER, HCatFieldSchema.Type.INT, 5, 5, 10,
+ 10, KeyType.NOT_A_KEY)
+ };
+ testUtils.createHCatTable(CreateMode.CREATE_AND_LOAD, 10, tableName, cols);
+ String createViewCmd = "drop view " + viewName + "; create view " + viewName + " as select * from " + tableName;
+ utils.launchHCatCli(createViewCmd);
+ }
+
+ public static junit.framework.Test suite() {
+ return new JUnit4TestAdapter(HCatalogImportTest.class);
+ }
}
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
index fb4a200..4301a33 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
@@ -177,29 +177,33 @@
* memory list.
*/
public void createHCatTableUsingSchema(String dbName,
- String tableName, List<HCatFieldSchema> tableCols,
- List<HCatFieldSchema> partKeys)
- throws Exception {
+ String tableName, List<HCatFieldSchema> tableCols,
+ List<HCatFieldSchema> partKeys)
+ throws Exception {
String databaseName = dbName == null
- ? SqoopHCatUtilities.DEFHCATDB : dbName;
+ ? SqoopHCatUtilities.DEFHCATDB : dbName;
+ dropHCatTableIfExists(tableName, databaseName);
+ LOG.info("Successfully dropped HCatalog table if it existed previously " + databaseName
+ + '.' + tableName);
+ String createCmd = getHCatCreateTableCmd(databaseName, tableName,
+ tableCols, partKeys);
+ utils.launchHCatCli(createCmd);
+ LOG.info("Created HCatalog table " + dbName + "." + tableName);
+ }
+
+ public void dropHCatTableIfExists(String tableName, String databaseName) {
LOG.info("Dropping HCatalog table if it exists " + databaseName
- + '.' + tableName);
+ + '.' + tableName);
String dropCmd = getHCatDropTableCmd(databaseName, tableName);
try {
utils.launchHCatCli(dropCmd);
} catch (Exception e) {
LOG.debug("Drop hcatalog table exception : " + e);
- LOG.info("Unable to drop table." + dbName + "."
- + tableName + ". Assuming it did not exist");
+ LOG.info("Unable to drop table." + databaseName + "."
+ + tableName + ". Assuming it did not exist");
}
- LOG.info("Creating HCatalog table if it exists " + databaseName
- + '.' + tableName);
- String createCmd = getHCatCreateTableCmd(databaseName, tableName,
- tableCols, partKeys);
- utils.launchHCatCli(createCmd);
- LOG.info("Created HCatalog table " + dbName + "." + tableName);
}
/**
