SQOOP-3346: Upgrade Hadoop version to 2.8.0
(Boglarka Egyed via Szabolcs Vasas)
diff --git a/ivy/libraries.properties b/ivy/libraries.properties
index c506ca8..d679b12 100644
--- a/ivy/libraries.properties
+++ b/ivy/libraries.properties
@@ -55,7 +55,7 @@
slf4j.version=1.7.7
-hadoop.version=2.6.0
+hadoop.version=2.8.0
hbase.version=1.2.4
hcatalog.version=1.2.1
diff --git a/src/java/org/apache/sqoop/config/ConfigurationConstants.java b/src/java/org/apache/sqoop/config/ConfigurationConstants.java
index 666852c..3724f25 100644
--- a/src/java/org/apache/sqoop/config/ConfigurationConstants.java
+++ b/src/java/org/apache/sqoop/config/ConfigurationConstants.java
@@ -55,23 +55,14 @@
"mapred.reduce.tasks.speculative.execution";
/**
- * The Configuration property identifying the job tracker address.
- */
- public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS =
- "mapred.job.tracker";
-
- /**
- * The Configuration property identifying the job tracker address (new).
- */
- public static final String PROP_MAPREDUCE_JOB_TRACKER_ADDRESS =
- "mapreduce.jobtracker.address";
-
- /**
* The Configuration property identifying the framework name. If set to YARN
* then we will not be in local mode.
*/
public static final String PROP_MAPREDUCE_FRAMEWORK_NAME =
"mapreduce.framework.name";
+
+ public static final String MAPREDUCE_FRAMEWORK_LOCAL = "local";
+
/**
* The group name of task counters.
*/
diff --git a/src/java/org/apache/sqoop/config/ConfigurationHelper.java b/src/java/org/apache/sqoop/config/ConfigurationHelper.java
index fb2ab03..5a3d8ff 100644
--- a/src/java/org/apache/sqoop/config/ConfigurationHelper.java
+++ b/src/java/org/apache/sqoop/config/ConfigurationHelper.java
@@ -31,6 +31,9 @@
import org.apache.hadoop.util.ReflectionUtils;
+import static org.apache.sqoop.config.ConfigurationConstants.MAPREDUCE_FRAMEWORK_LOCAL;
+import static org.apache.sqoop.config.ConfigurationConstants.PROP_MAPREDUCE_FRAMEWORK_NAME;
+
/**
* This class provides static helper methods that allow access and manipulation
* of job configuration. It is convenient to keep such access in one place in
@@ -121,13 +124,6 @@
}
/**
- * Sets the Jobtracker address to use for a job.
- */
- public static void setJobtrackerAddr(Configuration conf, String addr) {
- conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr);
- }
-
- /**
* @return the Configuration property identifying a DBWritable to use.
*/
public static String getDbInputClassProperty() {
@@ -233,17 +229,8 @@
return config.getInt(ConfigurationConstants.PROP_SPLIT_LIMIT, -1);
}
public static boolean isLocalJobTracker(Configuration conf) {
- // If framework is set to YARN, then we can't be running in local mode
- if ("yarn".equalsIgnoreCase(conf
- .get(ConfigurationConstants.PROP_MAPREDUCE_FRAMEWORK_NAME))) {
- return false;
- }
- String jtAddr = conf
- .get(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS);
- String jtAddr2 = conf
- .get(ConfigurationConstants.PROP_MAPREDUCE_JOB_TRACKER_ADDRESS);
- return (jtAddr != null && jtAddr.equals("local"))
- || (jtAddr2 != null && jtAddr2.equals("local"));
+ String frameworkName = conf.get(PROP_MAPREDUCE_FRAMEWORK_NAME);
+ return MAPREDUCE_FRAMEWORK_LOCAL.equals(frameworkName);
}
private ConfigurationHelper() {
diff --git a/src/java/org/apache/sqoop/mapreduce/JobBase.java b/src/java/org/apache/sqoop/mapreduce/JobBase.java
index 6d1e049..f7b1b46 100644
--- a/src/java/org/apache/sqoop/mapreduce/JobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/JobBase.java
@@ -272,9 +272,7 @@
protected void loadJars(Configuration conf, String ormJarFile,
String tableClassName) throws IOException {
- boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address"))
- || "local".equals(conf.get("mapred.job.tracker"));
- if (isLocal) {
+ if (ConfigurationHelper.isLocalJobTracker(conf)) {
// If we're using the LocalJobRunner, then instead of using the compiled
// jar file as the job source, we're running in the current thread. Push
// on another classloader that loads from that jar in addition to
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index f7310b9..ccded65 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -42,6 +42,7 @@
import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
import org.apache.sqoop.cli.RelatedOptions;
import org.apache.sqoop.cli.ToolOptions;
+import org.apache.sqoop.config.ConfigurationHelper;
import org.apache.sqoop.hive.HiveClient;
import org.apache.sqoop.hive.HiveClientFactory;
import org.apache.sqoop.manager.ImportJobContext;
@@ -118,9 +119,7 @@
private void loadJars(Configuration conf, String ormJarFile,
String tableClassName) throws IOException {
- boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address"))
- || "local".equals(conf.get("mapred.job.tracker"));
- if (isLocal) {
+ if (ConfigurationHelper.isLocalJobTracker(conf)) {
// If we're using the LocalJobRunner, then instead of using the compiled
// jar file as the job source, we're running in the current thread. Push
// on another classloader that loads from that jar in addition to
diff --git a/src/test/org/apache/sqoop/TestSqoopOptions.java b/src/test/org/apache/sqoop/TestSqoopOptions.java
index ba4a4d4..d0591ad 100644
--- a/src/test/org/apache/sqoop/TestSqoopOptions.java
+++ b/src/test/org/apache/sqoop/TestSqoopOptions.java
@@ -858,6 +858,7 @@
@Test
public void testSqoopOptionsCloneIsEqual() throws Exception {
SqoopOptions options = createSqoopOptionsFilledWithRandomData();
+ options.getConf().setAllowNullValueProperties(false); // always false in cloned conf
SqoopOptions clonedOptions = (SqoopOptions) options.clone();
assertThat(options).isEqualToComparingFieldByFieldRecursively(clonedOptions);
}
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
index 833ebe8..fbd8d96 100644
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
@@ -172,8 +172,6 @@
// Tell the job what class we're testing.
job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
- // use local mode in the same JVM.
- ConfigurationHelper.setJobtrackerAddr(job, "local");
job.set("fs.default.name", "file:///");
String warehouseDir = getWarehouseDir();
diff --git a/src/test/org/apache/sqoop/orm/TestParseMethods.java b/src/test/org/apache/sqoop/orm/TestParseMethods.java
index 46bb52d..0d34f5a 100644
--- a/src/test/org/apache/sqoop/orm/TestParseMethods.java
+++ b/src/test/org/apache/sqoop/orm/TestParseMethods.java
@@ -135,8 +135,6 @@
// Tell the job what class we're testing.
job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
- // use local mode in the same JVM.
- ConfigurationHelper.setJobtrackerAddr(job, "local");
if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
}
@@ -263,8 +261,6 @@
job.set(ExplicitSetMapper.SET_COL_KEY, BASE_COL_NAME + "0");
job.set(ExplicitSetMapper.SET_VAL_KEY, "this-is-a-test");
- // use local mode in the same JVM.
- ConfigurationHelper.setJobtrackerAddr(job, "local");
if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
}
