SQOOP-2880: Provide argument for overriding temporary directory
(Attila Szabo via Jarek Jarcec Cecho)
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 17751c7..ff96280 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -55,6 +55,8 @@
*/
public class SqoopOptions implements Cloneable {
+ private static final String OLD_SQOOP_TEST_IMPORT_ROOT_DIR = "sqoop.test.import.rootDir";
+
public static final Log LOG = LogFactory.getLog(SqoopOptions.class.getName());
/**
@@ -106,6 +108,8 @@
@StoredAsProperty("verbose") private boolean verbose;
+ @StoredAsProperty("temporary.dirRoot") private String tempRootDir;
+
@StoredAsProperty("mapreduce.job.name") private String mapreduceJobName;
@StoredAsProperty("db.connect.string") private String connectString;
@@ -1007,6 +1011,10 @@
// We do not want to be verbose too much if not explicitly needed
this.verbose = false;
+ //This name of the system property is intentionally OLD_SQOOP_TEST_IMPORT_ROOT_DIR
+ //to support backward compatibility. Do not exchange it with
+ //org.apache.sqoop.tool.BaseSqoopTool#TEMP_ROOTDIR_ARG
+ this.tempRootDir = System.getProperty(OLD_SQOOP_TEST_IMPORT_ROOT_DIR, "_sqoop");
this.isValidationEnabled = false; // validation is disabled by default
this.validatorClass = RowCountValidator.class;
this.validationThresholdClass = AbsoluteValidationThreshold.class;
@@ -1110,6 +1118,14 @@
this.verbose = beVerbose;
}
+ public String getTempRootDir() {
+ return tempRootDir;
+ }
+
+ public void setTempRootDir(String tempRootDir) {
+ this.tempRootDir = tempRootDir;
+ }
+
/**
* Get the temporary directory; guaranteed to end in File.separator
* (e.g., '/').
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 50dd67d..fecdf43 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -162,6 +162,7 @@
public static final String SQL_QUERY_SHORT_ARG = "e";
public static final String VERBOSE_ARG = "verbose";
public static final String HELP_ARG = "help";
+ public static final String TEMP_ROOTDIR_ARG = "temporary-rootdir";
public static final String UPDATE_KEY_ARG = "update-key";
public static final String UPDATE_MODE_ARG = "update-mode";
public static final String CALL_ARG = "call";
@@ -467,6 +468,12 @@
.withDescription("Print usage instructions")
.withLongOpt(HELP_ARG)
.create());
+ commonOpts.addOption(OptionBuilder
+ .withDescription("Defines the temporary root directory for the import")
+ .withLongOpt(TEMP_ROOTDIR_ARG)
+ .hasArg()
+ .withArgName("rootdir")
+ .create());
// relax isolation requirements
commonOpts.addOption(OptionBuilder
.withDescription("Use read-uncommitted isolation for imports")
@@ -939,6 +946,10 @@
throw new InvalidOptionsException("");
}
+ if (in.hasOption(TEMP_ROOTDIR_ARG)) {
+ out.setTempRootDir(in.getOptionValue(TEMP_ROOTDIR_ARG));
+ }
+
if (in.hasOption(CONNECT_STRING_ARG)) {
out.setConnectString(in.getOptionValue(CONNECT_STRING_ARG));
}
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index ad1d48b..ff7b822 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -571,7 +571,7 @@
if(salt == null && options.getSqlQuery() != null) {
salt = Integer.toHexString(options.getSqlQuery().hashCode());
}
- outputPath = AppendUtils.getTempAppendDir(salt);
+ outputPath = AppendUtils.getTempAppendDir(salt, options);
LOG.debug("Using temporary folder: " + outputPath.getName());
} else {
// Try in this order: target-dir or warehouse-dir
diff --git a/src/java/org/apache/sqoop/util/AppendUtils.java b/src/java/org/apache/sqoop/util/AppendUtils.java
index 0102ee7..a3082c4 100644
--- a/src/java/org/apache/sqoop/util/AppendUtils.java
+++ b/src/java/org/apache/sqoop/util/AppendUtils.java
@@ -39,9 +39,6 @@
public static final Log LOG = LogFactory.getLog(AppendUtils.class.getName());
- private static final String TEMP_IMPORT_ROOT =
- System.getProperty("sqoop.test.import.rootDir", "_sqoop");
-
private static final int PARTITION_DIGITS = 5;
private static final String FILEPART_SEPARATOR = "-";
private static final String FILEEXT_SEPARATOR = ".";
@@ -273,9 +270,9 @@
* Can be arbitrary string, for example table name or query checksum.
* @return a path pointing to the temporary directory
*/
- public static Path getTempAppendDir(String salt) {
+ public static Path getTempAppendDir(String salt, SqoopOptions options) {
String uuid = UUID.randomUUID().toString().replace("-", "");
- String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + uuid + "_" + salt;
+ String tempDir = options.getTempRootDir() + Path.SEPARATOR + uuid + "_" + salt;
return new Path(tempDir);
}
diff --git a/src/test/com/cloudera/sqoop/TestSqoopOptions.java b/src/test/com/cloudera/sqoop/TestSqoopOptions.java
index f4660e5..f9d1d54 100644
--- a/src/test/com/cloudera/sqoop/TestSqoopOptions.java
+++ b/src/test/com/cloudera/sqoop/TestSqoopOptions.java
@@ -390,6 +390,42 @@
connParams, in.getConnectionParams());
}
+ public void testDefaultTempRootDir() {
+ SqoopOptions opts = new SqoopOptions();
+
+ assertEquals("_sqoop", opts.getTempRootDir());
+ }
+
+ public void testDefaultLoadedTempRootDir() {
+ SqoopOptions out = new SqoopOptions();
+ Properties props = out.writeProperties();
+ SqoopOptions opts = new SqoopOptions();
+ opts.loadProperties(props);
+
+ assertEquals("_sqoop", opts.getTempRootDir());
+ }
+
+ public void testLoadedTempRootDir() {
+ SqoopOptions out = new SqoopOptions();
+ final String tempRootDir = "customRoot";
+ out.setTempRootDir(tempRootDir);
+ Properties props = out.writeProperties();
+ SqoopOptions opts = new SqoopOptions();
+ opts.loadProperties(props);
+
+ assertEquals(tempRootDir, opts.getTempRootDir());
+ }
+
+ public void testNulledTempRootDir() {
+ SqoopOptions out = new SqoopOptions();
+ out.setTempRootDir(null);
+ Properties props = out.writeProperties();
+ SqoopOptions opts = new SqoopOptions();
+ opts.loadProperties(props);
+
+ assertEquals("_sqoop", opts.getTempRootDir());
+ }
+
// test that hadoop-home is accepted as an option
public void testHadoopHome() throws Exception {
String [] args = {
diff --git a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
index f222dd7..cec7614 100644
--- a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
+++ b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.sqoop.tool.BaseSqoopTool;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
@@ -86,6 +87,15 @@
return args.toArray(new String[0]);
}
+ public void testTemporaryRootDirParse() throws Exception {
+ String customRoot = "customroot";
+ String[] args = new String[] {"--"+BaseSqoopTool.TEMP_ROOTDIR_ARG, customRoot};
+
+ SqoopOptions opts = new ImportTool().parseArguments(args, null, null, true);
+
+ assertEquals(customRoot, opts.getTempRootDir());
+ }
+
public void runParseTest(String fieldTerminator, String lineTerminator,
String encloser, String escape, boolean encloseRequired)
throws IOException {
