SQOOP-3066: Introduce an option + env variable to
enable/disable SQOOP-2737 feature
(Attila Szabo)
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 2350c7a..53cc03a 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -59,6 +59,8 @@
*/
public class SqoopOptions implements Cloneable {
+ public static final String ORACLE_ESCAPING_DISABLED = "sqoop.oracle.escaping.disabled";
+
private static final String OLD_SQOOP_TEST_IMPORT_ROOT_DIR = "sqoop.test.import.rootDir";
public static final Log LOG = LogFactory.getLog(SqoopOptions.class.getName());
@@ -381,6 +383,9 @@
private Class validationThresholdClass; // ValidationThreshold implementation
private Class validationFailureHandlerClass; // FailureHandler implementation
+ @StoredAsProperty(ORACLE_ESCAPING_DISABLED)
+ private boolean oracleEscapingDisabled;
+
public SqoopOptions() {
initDefaults(null);
}
@@ -701,6 +706,10 @@
if (this.verbose) {
LoggingUtils.setDebugLevel();
}
+
+ // Ensuring that oracleEscapingDisabled property is propagated to
+ // the level of Hadoop configuration as well
+ this.setOracleEscapingDisabled(this.isOracleEscapingDisabled());
}
private void loadPasswordProperty(Properties props) {
@@ -1039,6 +1048,8 @@
//to support backward compatibility. Do not exchange it.
this.throwOnError = isSqoopRethrowSystemPropertySet();
+ setOracleEscapingDisabled(Boolean.parseBoolean(System.getProperty(ORACLE_ESCAPING_DISABLED, "true")));
+
this.isValidationEnabled = false; // validation is disabled by default
this.validatorClass = RowCountValidator.class;
this.validationThresholdClass = AbsoluteValidationThreshold.class;
@@ -2708,4 +2719,16 @@
public void setMetadataTransactionIsolationLevel(int transactionIsolationLevel) {
this.metadataTransactionIsolationLevel = transactionIsolationLevel;
}
+
+ public boolean isOracleEscapingDisabled() {
+ return oracleEscapingDisabled;
+ }
+
+ public void setOracleEscapingDisabled(boolean escapingDisabled) {
+ this.oracleEscapingDisabled = escapingDisabled;
+ // important to have custom setter to ensure option is available through
+ // Hadoop configuration on those places where SqoopOptions is not reachable
+ getConf().setBoolean(ORACLE_ESCAPING_DISABLED, escapingDisabled);
+ }
+
}
diff --git a/src/java/org/apache/sqoop/manager/OracleManager.java b/src/java/org/apache/sqoop/manager/OracleManager.java
index d769972..3273435 100644
--- a/src/java/org/apache/sqoop/manager/OracleManager.java
+++ b/src/java/org/apache/sqoop/manager/OracleManager.java
@@ -917,20 +917,20 @@
}
}
- @Override
- public String escapeColName(String colName) {
- return OracleUtils.escapeIdentifier(colName);
- }
+ @Override
+ public String escapeColName(String colName) {
+ return OracleUtils.escapeIdentifier(colName, options.isOracleEscapingDisabled());
+ }
- @Override
- public String escapeTableName(String tableName) {
- return OracleUtils.escapeIdentifier(tableName);
- }
+ @Override
+ public String escapeTableName(String tableName) {
+ return OracleUtils.escapeIdentifier(tableName, options.isOracleEscapingDisabled());
+ }
- @Override
- public boolean escapeTableNameOnExport() {
- return true;
- }
+ @Override
+ public boolean escapeTableNameOnExport() {
+ return true;
+ }
@Override
public String[] getColumnNames(String tableName) {
diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java b/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java
index 216c771..2026c43 100644
--- a/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java
+++ b/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java
@@ -122,13 +122,17 @@
Configuration conf = this.options.getConf();
this.columnNamesInOracleTable =
- OraOopOracleQueries.getTableColumnNames(getConnection(),
- tableContext, OraOopUtilities
- .omitLobAndLongColumnsDuringImport(conf), OraOopUtilities
- .recallSqoopJobType(conf), true, // <-
- // onlyOraOopSupportedTypes
- true // <- omitOraOopPseudoColumns
- );
+ OraOopOracleQueries.getTableColumnNames(
+ getConnection(),
+ tableContext,
+ OraOopUtilities
+ .omitLobAndLongColumnsDuringImport(conf),
+ OraOopUtilities
+ .recallSqoopJobType(conf),
+ true, // <- onlyOraOopSupportedTypes
+ true, // <- omitOraOopPseudoColumns
+ options.isOracleEscapingDisabled()
+ );
} catch (SQLException ex) {
throw new RuntimeException(ex);
}
@@ -529,14 +533,14 @@
@Override
public String escapeColName(String colName) {
- return OracleUtils.escapeIdentifier(colName); // <- See notes at top about escaped
- // column names
+ return OracleUtils.escapeIdentifier(colName, options.isOracleEscapingDisabled()); // <- See notes at top about escaped
+ // column names
}
- @Override
- public String escapeTableName(String tableName) {
- return OracleUtils.escapeIdentifier(tableName);
- }
+ @Override
+ public String escapeTableName(String tableName) {
+ return OracleUtils.escapeIdentifier(tableName, options.isOracleEscapingDisabled());
+ }
@Override
public boolean escapeTableNameOnExport() {
diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopDBRecordReader.java b/src/java/org/apache/sqoop/manager/oracle/OraOopDBRecordReader.java
index f3ee4aa..f7d1889 100644
--- a/src/java/org/apache/sqoop/manager/oracle/OraOopDBRecordReader.java
+++ b/src/java/org/apache/sqoop/manager/oracle/OraOopDBRecordReader.java
@@ -31,6 +31,8 @@
import com.cloudera.sqoop.mapreduce.db.DBInputFormat;
import com.cloudera.sqoop.mapreduce.db.DBInputFormat.DBInputSplit;
import com.cloudera.sqoop.mapreduce.db.DataDrivenDBRecordReader;
+
+import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.manager.oracle.OraOopConstants.
OraOopTableImportWhereClauseLocation;
import org.apache.sqoop.manager.oracle.OraOopUtilities.OraOopStatsReports;
@@ -198,7 +200,8 @@
OraOopUtilities.recallSqoopJobType(conf)
, true // <- onlyOraOopSupportedTypes
, true // <- omitOraOopPseudoColumns
- );
+ , OracleUtils.isOracleEscapingDisabled(conf)
+ );
} catch (SQLException ex) {
LOG.error(String.format(
"Unable to obtain the data-types of the columns in table %s.\n"
diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopOracleQueries.java b/src/java/org/apache/sqoop/manager/oracle/OraOopOracleQueries.java
index 2603f83..b26bc71 100644
--- a/src/java/org/apache/sqoop/manager/oracle/OraOopOracleQueries.java
+++ b/src/java/org/apache/sqoop/manager/oracle/OraOopOracleQueries.java
@@ -604,9 +604,9 @@
}
public static List<String> getTableColumnNames(Connection connection,
- OracleTable table) throws SQLException {
+ OracleTable table, boolean escapingDisabled) throws SQLException {
- OracleTableColumns oracleTableColumns = getTableColumns(connection, table);
+ OracleTableColumns oracleTableColumns = getTableColumns(connection, table, escapingDisabled);
List<String> result = new ArrayList<String>(oracleTableColumns.size());
for (int idx = 0; idx < oracleTableColumns.size(); idx++) {
@@ -619,11 +619,11 @@
public static List<String> getTableColumnNames(Connection connection,
OracleTable table, boolean omitLobAndLongColumnsDuringImport,
OraOopConstants.Sqoop.Tool sqoopTool, boolean onlyOraOopSupportedTypes,
- boolean omitOraOopPseudoColumns) throws SQLException {
+ boolean omitOraOopPseudoColumns, boolean escapingDisabled) throws SQLException {
OracleTableColumns oracleTableColumns =
getTableColumns(connection, table, omitLobAndLongColumnsDuringImport,
- sqoopTool, onlyOraOopSupportedTypes, omitOraOopPseudoColumns);
+ sqoopTool, onlyOraOopSupportedTypes, omitOraOopPseudoColumns, escapingDisabled);
List<String> result = new ArrayList<String>(oracleTableColumns.size());
@@ -636,7 +636,7 @@
private static OracleTableColumns getTableColumns(Connection connection,
OracleTable table, boolean omitLobColumns, String dataTypesClause,
- HashSet<String> columnNamesToOmit) throws SQLException {
+ HashSet<String> columnNamesToOmit, boolean escapingDisabled) throws SQLException {
String sql =
"SELECT column_name, data_type " + " FROM dba_tab_columns"
@@ -680,7 +680,7 @@
if (idx > 0) {
columnList.append(",");
}
- columnList.append(OracleUtils.escapeIdentifier(result.get(idx).getName()));
+ columnList.append(OracleUtils.escapeIdentifier(result.get(idx).getName(), escapingDisabled));
}
sql =
String.format("SELECT %s FROM %s WHERE 0=1", columnList.toString(),
@@ -700,16 +700,16 @@
}
public static OracleTableColumns getTableColumns(Connection connection,
- OracleTable table) throws SQLException {
+ OracleTable table, boolean escapingDisabled) throws SQLException {
return getTableColumns(connection, table, false, null // <- dataTypesClause
- , null); // <-columnNamesToOmit
+ , null, escapingDisabled); // <-columnNamesToOmit
}
public static OracleTableColumns getTableColumns(Connection connection,
OracleTable table, boolean omitLobAndLongColumnsDuringImport,
OraOopConstants.Sqoop.Tool sqoopTool, boolean onlyOraOopSupportedTypes,
- boolean omitOraOopPseudoColumns) throws SQLException {
+ boolean omitOraOopPseudoColumns, boolean escapingDisabled) throws SQLException {
String dataTypesClause = "";
HashSet<String> columnNamesToOmit = null;
@@ -765,7 +765,7 @@
}
return getTableColumns(connection, table,
- omitLobAndLongColumnsDuringImport, dataTypesClause, columnNamesToOmit);
+ omitLobAndLongColumnsDuringImport, dataTypesClause, columnNamesToOmit, escapingDisabled);
}
public static List<OracleActiveInstance> getOracleActiveInstances(
@@ -1232,7 +1232,7 @@
public static void updateTable(Connection connection,
OracleTable targetTable, OracleTable sourceTable,
String[] mergeColumnNames, OracleTableColumns oracleTableColumns,
- Object oraOopSysDate, int oraOopMapperId, boolean parallelizationEnabled)
+ Object oraOopSysDate, int oraOopMapperId, boolean parallelizationEnabled, boolean escapingDisabled)
throws SQLException {
StringBuilder targetColumnsClause = new StringBuilder();
@@ -1244,12 +1244,13 @@
if (targetColumnsClause.length() > 0) {
targetColumnsClause.append(",");
}
- targetColumnsClause.append(String.format("a.%s", OracleUtils.escapeIdentifier(columnName)));
+ targetColumnsClause
+ .append(String.format("a.%s", OracleUtils.escapeIdentifier(columnName, escapingDisabled)));
if (sourceColumnsClause.length() > 0) {
sourceColumnsClause.append(",");
}
- sourceColumnsClause.append(String.format("b.%s", OracleUtils.escapeIdentifier(columnName)));
+ sourceColumnsClause.append(String.format("b.%s", OracleUtils.escapeIdentifier(columnName, escapingDisabled)));
}
String sourceClause = sourceColumnsClause.toString();
@@ -1298,7 +1299,7 @@
OracleTable tableToCreate, String tableToCreateStorageClause,
OracleTable tableContainingUpdates, OracleTable tableToBeUpdated,
String[] joinColumnNames, CreateExportChangesTableOptions options,
- boolean parallelizationEnabled) throws SQLException {
+ boolean parallelizationEnabled, boolean escapingDisabled) throws SQLException {
List<String> columnNames =
getTableColumnNames(connection, tableToBeUpdated
@@ -1306,6 +1307,7 @@
, OraOopConstants.Sqoop.Tool.EXPORT
, true // <- onlyOraOopSupportedTypes
, false // <- omitOraOopPseudoColumns
+ , escapingDisabled
);
StringBuilder columnClause = new StringBuilder(2 * columnNames.size());
@@ -1313,7 +1315,7 @@
if (idx > 0) {
columnClause.append(",");
}
- columnClause.append(String.format("a.%s", OracleUtils.escapeIdentifier(columnNames.get(idx))));
+ columnClause.append(String.format("a.%s", OracleUtils.escapeIdentifier(columnNames.get(idx), escapingDisabled)));
}
StringBuilder rowEqualityClause = new StringBuilder();
@@ -1432,10 +1434,10 @@
public static void insertRowsIntoExportTable(Connection connection,
OracleTable tableToInsertRowsInto,
OracleTable tableContainingRowsToInsert, Object oraOopSysDate,
- int oraOopMapperId, boolean parallelizationEnabled) throws SQLException {
+ int oraOopMapperId, boolean parallelizationEnabled, boolean escapingDisabled) throws SQLException {
List<String> columnNames =
- getTableColumnNames(connection, tableToInsertRowsInto);
+ getTableColumnNames(connection, tableToInsertRowsInto, escapingDisabled);
StringBuilder columnClause =
new StringBuilder(2 + (2 * columnNames.size()));
diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatBase.java b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatBase.java
index 8f94cf8..c6b7098 100644
--- a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatBase.java
+++ b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatBase.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.sqoop.SqoopOptions;
import com.cloudera.sqoop.lib.SqoopRecord;
import com.cloudera.sqoop.mapreduce.AsyncSqlOutputFormat;
@@ -279,6 +280,7 @@
.omitLobAndLongColumnsDuringImport(conf), OraOopUtilities
.recallSqoopJobType(conf), true // <- onlyOraOopSupportedTypes
, false // <- omitOraOopPseudoColumns
+ , OracleUtils.isOracleEscapingDisabled(conf)
));
}
@@ -354,7 +356,8 @@
if (colCount > 0) {
sqlNames.append("\n,");
}
- sqlNames.append(OracleUtils.escapeIdentifier(columnName));
+ boolean escapingDisabled = OracleUtils.isOracleEscapingDisabled(getConf());
+ sqlNames.append(OracleUtils.escapeIdentifier(columnName, escapingDisabled));
// column values...
if (colCount > 0) {
diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatInsert.java b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatInsert.java
index d430ef7..1874b9f 100644
--- a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatInsert.java
+++ b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatInsert.java
@@ -27,6 +27,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.sqoop.SqoopOptions;
import com.cloudera.sqoop.lib.SqoopRecord;
@@ -148,11 +149,14 @@
// a subpartition of the 'real' export table...
this.oracleTable = createUniqueMapperTable(context);
- setOracleTableColumns(OraOopOracleQueries.getTableColumns(this
- .getConnection(), this.oracleTable, OraOopUtilities
- .omitLobAndLongColumnsDuringImport(conf), OraOopUtilities
- .recallSqoopJobType(conf), true // <- onlyOraOopSupportedTypes
- , false) // <- omitOraOopPseudoColumns
+ setOracleTableColumns(OraOopOracleQueries.getTableColumns(
+ this.getConnection(),
+ this.oracleTable,
+ OraOopUtilities.omitLobAndLongColumnsDuringImport(conf),
+ OraOopUtilities.recallSqoopJobType(conf),
+ true, // <- onlyOraOopSupportedTypes
+ false, // <- omitOraOopPseudoColumns
+ OracleUtils.isOracleEscapingDisabled(conf))
);
this.subPartitionName =
diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatUpdate.java b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatUpdate.java
index a33768f..33bcb84 100644
--- a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatUpdate.java
+++ b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatUpdate.java
@@ -29,6 +29,8 @@
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import com.cloudera.sqoop.lib.SqoopRecord;
+
+import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.manager.oracle.OraOopOracleQueries.
CreateExportChangesTableOptions;
@@ -161,11 +163,14 @@
Configuration conf = context.getConfiguration();
this.oracleTable = createUniqueMapperTable(context);
- setOracleTableColumns(OraOopOracleQueries.getTableColumns(this
- .getConnection(), this.oracleTable, OraOopUtilities
- .omitLobAndLongColumnsDuringImport(conf), OraOopUtilities
- .recallSqoopJobType(conf), true // <- onlyOraOopSupportedTypes
- , false) // <- omitOraOopPseudoColumns
+ setOracleTableColumns(OraOopOracleQueries.getTableColumns(
+ this.getConnection(),
+ this.oracleTable,
+ OraOopUtilities.omitLobAndLongColumnsDuringImport(conf),
+ OraOopUtilities.recallSqoopJobType(conf),
+ true, // <- onlyOraOopSupportedTypes
+ false, // <- omitOraOopPseudoColumns
+ OracleUtils.isOracleEscapingDisabled(conf))
);
}
@@ -292,7 +297,7 @@
OraOopOracleQueries.createExportChangesTable(this.getConnection(),
changesTable, temporaryTableStorageClause, this.oracleTable,
targetTable, this.updateColumnNames, changesTableOptions,
- parallelizationEnabled);
+ parallelizationEnabled, OracleUtils.isOracleEscapingDisabled(context.getConfiguration()));
if (changeTableRowCount == 0) {
LOG.debug(String.format(
@@ -312,7 +317,7 @@
OraOopOracleQueries.insertRowsIntoExportTable(this.getConnection(),
targetTable, changesTable, sysDateTime, this.mapperId,
- parallelizationEnabled);
+ parallelizationEnabled, OracleUtils.isOracleEscapingDisabled(context.getConfiguration()));
break;
case UpdateSql:
@@ -322,7 +327,7 @@
OraOopOracleQueries.updateTable(this.getConnection(), targetTable,
changesTable, this.updateColumnNames, this
.getOracleTableColumns(), sysDateTime, this.mapperId,
- parallelizationEnabled);
+ parallelizationEnabled, context.getConfiguration().getBoolean(SqoopOptions.ORACLE_ESCAPING_DISABLED,false));
double timeInSec = (System.nanoTime() - start) / Math.pow(10, 9);
LOG.info(String.format("Time spent performing an update: %f sec.",
diff --git a/src/java/org/apache/sqoop/manager/oracle/OracleUtils.java b/src/java/org/apache/sqoop/manager/oracle/OracleUtils.java
index b46034b..aa56e70 100644
--- a/src/java/org/apache/sqoop/manager/oracle/OracleUtils.java
+++ b/src/java/org/apache/sqoop/manager/oracle/OracleUtils.java
@@ -19,6 +19,8 @@
package org.apache.sqoop.manager.oracle;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.SqoopOptions;
/**
* Utility class for Oracle.
@@ -28,12 +30,20 @@
private static final String PERIOD_REGEX = "\\.";
private static final String PERIOD_DELIMITER = ".";
+ public static boolean isOracleEscapingDisabled(Configuration conf) {
+ return conf.getBoolean(SqoopOptions.ORACLE_ESCAPING_DISABLED, true);
+ }
+
public static boolean isEscaped(final String identifier) {
return !StringUtils.isBlank(identifier) && identifier.startsWith("\"") && identifier.endsWith("\"");
}
public static String escapeIdentifier(final String identifier) {
- if (StringUtils.isBlank(identifier)) {
+ return escapeIdentifier(identifier, false);
+ }
+
+ public static String escapeIdentifier(final String identifier, boolean escapingDisabled) {
+ if (escapingDisabled || StringUtils.isBlank(identifier)) {
return identifier;
}
diff --git a/src/java/org/apache/sqoop/mapreduce/db/OracleDBRecordReader.java b/src/java/org/apache/sqoop/mapreduce/db/OracleDBRecordReader.java
index f1647b8..4d7da39 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/OracleDBRecordReader.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/OracleDBRecordReader.java
@@ -22,6 +22,7 @@
import java.sql.Connection;
import java.sql.SQLException;
+import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.manager.oracle.OracleUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -66,8 +67,9 @@
if (dbConf.getInputQuery() == null) {
query.append("SELECT ");
+ boolean escapingDisabled = OracleUtils.isOracleEscapingDisabled(getConf());
for (int i = 0; i < fieldNames.length; i++) {
- query.append(OracleUtils.escapeIdentifier(fieldNames[i]));
+ query.append(OracleUtils.escapeIdentifier(fieldNames[i], escapingDisabled));
if (i != fieldNames.length -1) {
query.append(", ");
}
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 6082296..3d37859 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -171,6 +171,7 @@
public static final String SKIP_DISTCACHE_ARG = "skip-dist-cache";
public static final String RELAXED_ISOLATION = "relaxed-isolation";
public static final String THROW_ON_ERROR_ARG = "throw-on-error";
+ public static final String ORACLE_ESCAPING_DISABLED = "oracle-escaping-disabled";
// Arguments for validation.
public static final String VALIDATE_ARG = "validate";
@@ -511,6 +512,13 @@
.withLongOpt(RELAXED_ISOLATION)
.create());
+ commonOpts.addOption(OptionBuilder
+ .withDescription("Disable the escaping mechanism of the Oracle/OraOop connection managers")
+ .withLongOpt(ORACLE_ESCAPING_DISABLED)
+ .hasArg()
+ .withArgName("boolean")
+ .create());
+
return commonOpts;
}
@@ -1072,6 +1080,10 @@
+ "java.sql.Connection javadocs for more details", e);
}
}
+
+ if (in.hasOption(ORACLE_ESCAPING_DISABLED)) {
+ out.setOracleEscapingDisabled(Boolean.parseBoolean(in.getOptionValue(ORACLE_ESCAPING_DISABLED)));
+ }
}
private void applyCredentialsOptions(CommandLine in, SqoopOptions out)
diff --git a/src/test/com/cloudera/sqoop/TestSqoopOptions.java b/src/test/com/cloudera/sqoop/TestSqoopOptions.java
index d5d09b6..d95f904 100644
--- a/src/test/com/cloudera/sqoop/TestSqoopOptions.java
+++ b/src/test/com/cloudera/sqoop/TestSqoopOptions.java
@@ -24,18 +24,26 @@
import junit.framework.TestCase;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.sqoop.manager.oracle.OracleUtils;
+
import com.cloudera.sqoop.lib.DelimiterSet;
import com.cloudera.sqoop.tool.ImportTool;
import com.cloudera.sqoop.testutil.HsqldbTestServer;
import org.junit.Before;
import org.junit.After;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
import static org.apache.sqoop.Sqoop.SQOOP_RETHROW_PROPERTY;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
/**
* Test aspects of the SqoopOptions class.
*/
+@RunWith(JUnit4.class)
public class TestSqoopOptions extends TestCase {
private Properties originalSystemProperties;
@@ -51,10 +59,12 @@
}
// tests for the toChar() parser
+ @Test
public void testNormalChar() throws Exception {
assertEquals('a', SqoopOptions.toChar("a"));
}
+ @Test
public void testEmptyString() throws Exception {
try {
SqoopOptions.toChar("");
@@ -64,6 +74,7 @@
}
}
+ @Test
public void testNullString() throws Exception {
try {
SqoopOptions.toChar(null);
@@ -73,45 +84,55 @@
}
}
+ @Test
public void testTooLong() throws Exception {
// Should just use the first character and log a warning.
assertEquals('x', SqoopOptions.toChar("xyz"));
}
+ @Test
public void testHexChar1() throws Exception {
assertEquals(0xF, SqoopOptions.toChar("\\0xf"));
}
+ @Test
public void testHexChar2() throws Exception {
assertEquals(0xF, SqoopOptions.toChar("\\0xF"));
}
+ @Test
public void testHexChar3() throws Exception {
assertEquals(0xF0, SqoopOptions.toChar("\\0xf0"));
}
+ @Test
public void testHexChar4() throws Exception {
assertEquals(0xF0, SqoopOptions.toChar("\\0Xf0"));
}
+ @Test
public void testEscapeChar1() throws Exception {
assertEquals('\n', SqoopOptions.toChar("\\n"));
}
+ @Test
public void testEscapeChar2() throws Exception {
assertEquals('\\', SqoopOptions.toChar("\\\\"));
}
+ @Test
public void testEscapeChar3() throws Exception {
assertEquals('\\', SqoopOptions.toChar("\\"));
}
+ @Test
public void testWhitespaceToChar() throws Exception {
assertEquals(' ', SqoopOptions.toChar(" "));
assertEquals(' ', SqoopOptions.toChar(" "));
assertEquals('\t', SqoopOptions.toChar("\t"));
}
+ @Test
public void testUnknownEscape1() throws Exception {
try {
SqoopOptions.toChar("\\Q");
@@ -121,6 +142,7 @@
}
}
+ @Test
public void testUnknownEscape2() throws Exception {
try {
SqoopOptions.toChar("\\nn");
@@ -130,30 +152,37 @@
}
}
+ @Test
public void testEscapeNul1() throws Exception {
assertEquals(DelimiterSet.NULL_CHAR, SqoopOptions.toChar("\\0"));
}
+ @Test
public void testEscapeNul2() throws Exception {
assertEquals(DelimiterSet.NULL_CHAR, SqoopOptions.toChar("\\00"));
}
+ @Test
public void testEscapeNul3() throws Exception {
assertEquals(DelimiterSet.NULL_CHAR, SqoopOptions.toChar("\\0000"));
}
+ @Test
public void testEscapeNul4() throws Exception {
assertEquals(DelimiterSet.NULL_CHAR, SqoopOptions.toChar("\\0x0"));
}
+ @Test
public void testOctalChar1() throws Exception {
assertEquals(04, SqoopOptions.toChar("\\04"));
}
+ @Test
public void testOctalChar2() throws Exception {
assertEquals(045, SqoopOptions.toChar("\\045"));
}
+ @Test
public void testErrOctalChar() throws Exception {
try {
SqoopOptions.toChar("\\095");
@@ -163,6 +192,7 @@
}
}
+ @Test
public void testErrHexChar() throws Exception {
try {
SqoopOptions.toChar("\\0x9K5");
@@ -178,6 +208,7 @@
}
// test that setting output delimiters also sets input delimiters
+ @Test
public void testDelimitersInherit() throws Exception {
String [] args = {
"--fields-terminated-by",
@@ -191,6 +222,7 @@
// Test that setting output delimiters and setting input delims
// separately works.
+ @Test
public void testDelimOverride1() throws Exception {
String [] args = {
"--fields-terminated-by",
@@ -205,6 +237,7 @@
}
// test that the order in which delims are specified doesn't matter
+ @Test
public void testDelimOverride2() throws Exception {
String [] args = {
"--input-fields-terminated-by",
@@ -218,6 +251,7 @@
assertEquals('|', opts.getOutputFieldDelim());
}
+ @Test
public void testBadNumMappers1() throws Exception {
String [] args = {
"--num-mappers",
@@ -232,6 +266,7 @@
}
}
+ @Test
public void testBadNumMappers2() throws Exception {
String [] args = {
"-m",
@@ -246,6 +281,7 @@
}
}
+ @Test
public void testGoodNumMappers() throws Exception {
String [] args = {
"-m",
@@ -256,6 +292,7 @@
assertEquals(4, opts.getNumMappers());
}
+ @Test
public void testHivePartitionParams() throws Exception {
String[] args = {
"--hive-partition-key", "ds",
@@ -266,6 +303,7 @@
assertEquals("20110413", opts.getHivePartitionValue());
}
+ @Test
public void testBoundaryQueryParams() throws Exception {
String[] args = {
"--boundary-query", "select 1, 2",
@@ -275,6 +313,7 @@
assertEquals("select 1, 2", opts.getBoundaryQuery());
}
+ @Test
public void testMapColumnHiveParams() throws Exception {
String[] args = {
"--map-column-hive", "id=STRING",
@@ -286,6 +325,7 @@
assertEquals("STRING", mapping.get("id"));
}
+ @Test
public void testMalformedMapColumnHiveParams() throws Exception {
String[] args = {
"--map-column-hive", "id",
@@ -298,6 +338,7 @@
}
}
+ @Test
public void testMapColumnJavaParams() throws Exception {
String[] args = {
"--map-column-java", "id=String",
@@ -309,6 +350,7 @@
assertEquals("String", mapping.get("id"));
}
+ @Test
public void testMalfromedMapColumnJavaParams() throws Exception {
String[] args = {
"--map-column-java", "id",
@@ -321,12 +363,14 @@
}
}
+ @Test
public void testSkipDistCacheOption() throws Exception {
String[] args = {"--skip-dist-cache"};
SqoopOptions opts = parse(args);
assertTrue(opts.isSkipDistCache());
}
+ @Test
public void testPropertySerialization1() {
// Test that if we write a SqoopOptions out to a Properties,
// and then read it back in, we get all the same results.
@@ -367,6 +411,7 @@
connParams, in.getConnectionParams());
}
+ @Test
public void testPropertySerialization2() {
// Test that if we write a SqoopOptions out to a Properties,
// and then read it back in, we get all the same results.
@@ -407,12 +452,14 @@
connParams, in.getConnectionParams());
}
+ @Test
public void testDefaultTempRootDir() {
SqoopOptions opts = new SqoopOptions();
assertEquals("_sqoop", opts.getTempRootDir());
}
+ @Test
public void testDefaultLoadedTempRootDir() {
SqoopOptions out = new SqoopOptions();
Properties props = out.writeProperties();
@@ -422,6 +469,7 @@
assertEquals("_sqoop", opts.getTempRootDir());
}
+ @Test
public void testLoadedTempRootDir() {
SqoopOptions out = new SqoopOptions();
final String tempRootDir = "customRoot";
@@ -433,6 +481,7 @@
assertEquals(tempRootDir, opts.getTempRootDir());
}
+ @Test
public void testNulledTempRootDir() {
SqoopOptions out = new SqoopOptions();
out.setTempRootDir(null);
@@ -501,7 +550,122 @@
assertTrue(opts.isThrowOnError());
}
+ @Test
+ public void defaultValueOfOracleEscapingDisabledShouldBeFalse() {
+ System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED);
+ SqoopOptions opts = new SqoopOptions();
+
+ assertThat(opts.isOracleEscapingDisabled(), is(equalTo(true)));
+ }
+
+ @Test
+ public void valueOfOracleEscapingDisabledShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToFalse() {
+ System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "false");
+ SqoopOptions opts = new SqoopOptions();
+
+ assertThat(opts.isOracleEscapingDisabled(), is(equalTo(false)));
+ }
+
+ @Test
+ public void valueOfOracleEscapingDisabledShouldBeTrueIfTheValueOfTheRelatedEnvironmentVariableIsSetToTrue() {
+ System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "true");
+ SqoopOptions opts = new SqoopOptions();
+
+ assertThat(opts.isOracleEscapingDisabled(), is(equalTo(true)));
+ }
+
+ @Test
+ public void valueOfOracleEscapingDisabledShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToAnyNonBooleanValue() {
+ System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "falsetrue");
+ SqoopOptions opts = new SqoopOptions();
+
+ assertThat(opts.isOracleEscapingDisabled(), is(equalTo(false)));
+ }
+
+ @Test
+ public void hadoopConfigurationInstanceOfSqoopOptionsShouldContainTheSameValueForOracleEscapingDisabledAsSqoopOptionsProperty() {
+ SqoopOptions opts = new SqoopOptions();
+
+ assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()),
+ is(equalTo(opts.isOracleEscapingDisabled())));
+ }
+
+ @Test
+ public void hadoopConfigurationInstanceOfSqoopOptionsShouldContainTrueForOracleEscapingDisabledAsTheValueDirectlyHasBeenSetToSqoopOptions() {
+ System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED);
+ SqoopOptions opts = new SqoopOptions();
+ opts.setOracleEscapingDisabled(true);
+
+ assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()),
+ is(equalTo(true)));
+ }
+
+ @Test
+ public void hadoopConfigurationInstanceOfSqoopOptionsShouldContainFalseForOracleEscapingDisabledAsTheValueDirectlyHasBeenSetToSqoopOptions() {
+ System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED);
+ SqoopOptions opts = new SqoopOptions();
+ opts.setOracleEscapingDisabled(false);
+
+ assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()),
+ is(equalTo(false)));
+ }
+
+ @Test
+ public void valueOfOracleEscapingDisabledInHadoopConfigurationInstanceOfSqoopOptionsShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToFalse() {
+ System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "false");
+ SqoopOptions opts = new SqoopOptions();
+
+ assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()),
+ is(equalTo(false)));
+ }
+
+ @Test
+ public void valueOfOracleEscapingDisabledInHadoopConfigurationInstanceOfSqoopOptionsShouldBeTrueIfTheValueOfTheRelatedEnvironmentVariableIsSetToTrue() {
+ System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "true");
+ SqoopOptions opts = new SqoopOptions();
+
+ assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()),
+ is(equalTo(true)));
+ }
+
+ @Test
+ public void valueOfOracleEscapingDisabledInHadoopConfigurationInstanceOfSqoopOptionsShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToAnyNonBooleanValue() {
+ System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "falsetrue");
+ SqoopOptions opts = new SqoopOptions();
+
+ assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()),
+ is(equalTo(false)));
+ }
+
+ @Test
+ public void valueOfOracleEscapingDisabledShouldBeAbleToSavedAndLoadedBackWithTheSameValue() {
+ System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED);
+ SqoopOptions opts = new SqoopOptions();
+ opts.setOracleEscapingDisabled(false);
+ Properties out = opts.writeProperties();
+ opts = new SqoopOptions();
+ opts.loadProperties(out);
+
+ assertThat(opts.isOracleEscapingDisabled(), is(equalTo(false)));
+ assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()),
+ is(equalTo(false)));
+ }
+
+ @Test
+ public void valueOfOracleEscapingDisabledShouldBeEqualToNullIfASqoopOptionsInstanceWasLoadedWhichDidntContainASavedValueForIt() {
+ System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED);
+ SqoopOptions opts = new SqoopOptions();
+ Properties out = opts.writeProperties();
+ opts = new SqoopOptions();
+ opts.loadProperties(out);
+
+ assertThat(opts.isOracleEscapingDisabled(), is(equalTo(true)));
+ assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()),
+ is(equalTo(true)));
+ }
+
// test that hadoop-home is accepted as an option
+ @Test
public void testHadoopHome() throws Exception {
String [] args = {
"--hadoop-home",
@@ -513,12 +677,12 @@
}
// test that hadoop-home is accepted as an option
+ @Test
public void testHadoopMapRedOverridesHadoopHome() throws Exception {
- String[] args = { "--hadoop-home", "/usr/lib/hadoop-ignored",
- "--hadoop-mapred-home", "/usr/lib/hadoop", };
+ String[] args = { "--hadoop-home", "/usr/lib/hadoop-ignored", "--hadoop-mapred-home", "/usr/lib/hadoop", };
- SqoopOptions opts = parse(args);
- assertEquals("/usr/lib/hadoop", opts.getHadoopMapRedHome());
+ SqoopOptions opts = parse(args);
+ assertEquals("/usr/lib/hadoop", opts.getHadoopMapRedHome());
}
@@ -536,6 +700,7 @@
}
//test compatability of --detele-target-dir with import
+ @Test
public void testDeteleTargetDir() throws Exception {
String [] extraArgs = {
"--delete-target-dir",
@@ -548,6 +713,7 @@
}
//test incompatability of --delete-target-dir & --append with import
+ @Test
public void testDeleteTargetDirWithAppend() throws Exception {
String [] extraArgs = {
"--append",
@@ -562,6 +728,7 @@
}
//test incompatability of --delete-target-dir with incremental import
+ @Test
public void testDeleteWithIncrementalImport() throws Exception {
String [] extraArgs = {
"--incremental", "append",
@@ -577,6 +744,7 @@
// test that hbase bulk load import with table name and target dir
// passes validation
+ @Test
public void testHBaseBulkLoad() throws Exception {
String [] extraArgs = {
longArgument(BaseSqoopTool.HBASE_BULK_LOAD_ENABLED_ARG),
@@ -588,6 +756,7 @@
}
// test that hbase bulk load import with a missing --hbase-table fails
+ @Test
public void testHBaseBulkLoadMissingHbaseTable() throws Exception {
String [] extraArgs = {
longArgument(BaseSqoopTool.HBASE_BULK_LOAD_ENABLED_ARG),
@@ -604,6 +773,7 @@
return String.format("--%s", argument);
}
+ @Test
public void testRelaxedIsolation() throws Exception {
String extraArgs[] = {
"--relaxed-isolation",
@@ -611,6 +781,7 @@
validateImportOptions(extraArgs);
}
+ @Test
public void testResetToOneMapper() throws Exception {
String extraArgs[] = {
"--autoreset-to-one-mapper",
@@ -618,6 +789,7 @@
validateImportOptions(extraArgs);
}
+ @Test
public void testResetToOneMapperAndSplitBy() throws Exception {
String extraArgs[] = {
"--autoreset-to-one-mapper",
@@ -631,5 +803,4 @@
// Expected
}
}
-
}
diff --git a/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java b/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java
index 355b1e2..79638eb 100644
--- a/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java
+++ b/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java
@@ -53,7 +53,8 @@
StringBuilder result = new StringBuilder();
String delim = "";
for (OracleDataDefinition column : tableDefinition.getColumnList()) {
- result.append(delim).append(OracleUtils.escapeIdentifier(column.getColumnName())).append(" ").append(
+ result.append(delim).append(OracleUtils.escapeIdentifier(column.getColumnName())).append(" ")
+ .append(
column.getDataType());
delim = ",\n";
}
