SQOOP-3206: Make sqoop fail if user uses --direct connector and tries to encode a null value when using a MySQL database
(Zach Berkowitz via Anna Szonyi)
diff --git a/src/java/org/apache/sqoop/tool/ExportTool.java b/src/java/org/apache/sqoop/tool/ExportTool.java
index 5512fa7..cd6cdf3 100644
--- a/src/java/org/apache/sqoop/tool/ExportTool.java
+++ b/src/java/org/apache/sqoop/tool/ExportTool.java
@@ -33,6 +33,7 @@
import com.cloudera.sqoop.cli.ToolOptions;
import com.cloudera.sqoop.manager.ExportJobContext;
import com.cloudera.sqoop.util.ExportException;
+import static org.apache.sqoop.manager.SupportedManagers.MYSQL;
/**
* Tool that performs HDFS exports to databases.
@@ -385,8 +386,21 @@
void vaildateDirectExportOptions(SqoopOptions options) throws InvalidOptionsException {
if (options.isDirect()) {
validateHasDirectConnectorOption(options);
+ validateDirectMysqlOptions(options);
}
}
+
+ public void validateDirectMysqlOptions(SqoopOptions options) throws InvalidOptionsException {
+ if (!MYSQL.isTheManagerTypeOf(options)) {
+ return;
+ }
+ if (options.getInNullStringValue() != null || options.getInNullNonStringValue() != null) {
+ throw new InvalidOptionsException(
+ "The --direct option is not compatible with the --input-null-string or " +
+ "--input-null-non-string command for MySQL exports");
+ }
+ }
+
private void applyNewUpdateOptions(CommandLine in, SqoopOptions out)
throws InvalidOptionsException {
if (in.hasOption(UPDATE_MODE_ARG)) {
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index 4b1b12d..78c7758 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -1120,13 +1120,20 @@
}
void validateDirectMysqlOptions(SqoopOptions options) throws InvalidOptionsException {
- if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile
- && MYSQL.isTheManagerTypeOf(options)) {
+ if (!MYSQL.isTheManagerTypeOf(options)) {
+ return;
+ }
+ if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile) {
throw new InvalidOptionsException(
"MySQL direct import currently supports only text output format. "
+ "Parameters --as-sequencefile --as-avrodatafile and --as-parquetfile are not "
+ "supported with --direct params in MySQL case.");
}
+ if (options.getNullStringValue() != null || options.getNullNonStringValue() != null) {
+ throw new InvalidOptionsException(
+ "The --direct option is not compatible with the --null-string or " +
+ "--null-non-string command for MySQL imports");
+ }
}
/**
* Validate the incremental import options.
diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
index 9fa8816..f9e3cde 100644
--- a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
@@ -303,6 +303,24 @@
}
}
+ @Test(expected = IOException.class)
+ public void testExportInputNullStringFailsValidate() throws IOException {
+ runExport(getArgv(true, 10, 10,
+ "--username", MySQLAuthTest.AUTH_TEST_USER,
+ "--password", MySQLAuthTest.AUTH_TEST_PASS,
+ "--connect", MySQLAuthTest.AUTH_CONNECT_STRING,
+ "--input-null-string", "null"));
+ }
+
+ @Test(expected = IOException.class)
+ public void testExportInputNullNonStringFailsValidate() throws IOException {
+ runExport(getArgv(true, 10, 10,
+ "--username", MySQLAuthTest.AUTH_TEST_USER,
+ "--password", MySQLAuthTest.AUTH_TEST_PASS,
+ "--connect", MySQLAuthTest.AUTH_CONNECT_STRING,
+ "--input-null-non-string", "null"));
+ }
+
@Ignore("Ignoring this test as staging is not supported in direct mode.")
@Override
@Test
diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
index a58fa17..247ce0b 100644
--- a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
+++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
@@ -192,13 +192,7 @@
}
String [] argv = getArgv(mysqlOutputDelims, isDirect, tableName, extraArgs);
- try {
- runImport(argv);
- } catch (IOException ioe) {
- LOG.error("Got IOException during import: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- }
+ runImport(argv);
File f = new File(filePath.toString());
assertTrue("Could not find imported data file: " + f, f.exists());
@@ -358,6 +352,22 @@
}
+ @Test(expected = IOException.class)
+ public void testSqoopNullStringValueFailsValidate() throws Exception {
+ String [] expectedResults = {};
+ String [] extraArgs = {"--null-string", "abc"};
+
+ doImport(false, true, getTableName(), expectedResults, extraArgs);
+ }
+
+ @Test(expected = IOException.class)
+ public void testSqoopNullNonStringValueFailsValidate() throws Exception {
+ String [] expectedResults = {};
+ String [] extraArgs = {"--null-non-string", "abc"};
+
+ doImport(false, true, getTableName(), expectedResults, extraArgs);
+ }
+
@Test
public void testJdbcEscapedColumnName() throws Exception {
// Test a JDBC-based import of a table with a column whose name is
diff --git a/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java b/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java
index dfe1952..0018fb1 100644
--- a/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java
+++ b/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java
@@ -66,6 +66,22 @@
exportTool.vaildateDirectExportOptions(options);
}
+ @Test(expected = SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportInputNullStringThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL);
+ when(options.getInNullNonStringValue()).thenReturn("abc");
+
+ exportTool.validateDirectMysqlOptions(options);
+ }
+
+ @Test(expected = SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportInputNullNonStringThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL);
+ when(options.getInNullNonStringValue()).thenReturn("abc");
+
+ exportTool.validateDirectMysqlOptions(options);
+ }
+
private SqoopOptions stubDirectOptions(SupportedManagers supportedManagers) {
return stubOptions(supportedManagers, true);
}
diff --git a/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java b/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java
index acf4fcf..d4084ed 100644
--- a/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java
+++ b/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java
@@ -44,7 +44,6 @@
when(options.getConnectString()).thenReturn(mysqlConnectionString);
importTool.validateDirectMysqlOptions(options);
verify(options, times(1)).getFileLayout();
- verifyNoMoreInteractions(options);
}
@@ -107,6 +106,22 @@
importTool.validateDirectImportOptions(options);
}
+ @Test(expected = SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportNullNonStringThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL);
+ when(options.getNullNonStringValue()).thenReturn("abc");
+
+ importTool.validateDirectMysqlOptions(options);
+ }
+
+ @Test(expected = SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportNullStringThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL);
+ when(options.getNullStringValue()).thenReturn("abc");
+
+ importTool.validateDirectMysqlOptions(options);
+ }
+
private SqoopOptions stubDirectOptions(SupportedManagers supportedManagers) {
return stubOptions(supportedManagers, true);
}
