Merge pull request #445 from metamx/hadoop-version-update
Hadoop version update
diff --git a/docs/content/Tasks.md b/docs/content/Tasks.md
index 061f02d..80f5d09 100644
--- a/docs/content/Tasks.md
+++ b/docs/content/Tasks.md
@@ -74,14 +74,14 @@
|--------|-----------|---------|
|type|The task type, this should always be "index_hadoop".|yes|
|config|A Hadoop Index Config. See [Batch Ingestion](Batch-ingestion.html)|yes|
-|hadoopCoordinates|The Maven \<groupId\>:\<artifactId\>:\<version\> of Hadoop to use. The default is "org.apache.hadoop:hadoop-core:1.0.3".|no|
+|hadoopCoordinates|The Maven \<groupId\>:\<artifactId\>:\<version\> of Hadoop to use. The default is "org.apache.hadoop:hadoop-client:2.3.0".|no|
The Hadoop Index Config submitted as part of an Hadoop Index Task is identical to the Hadoop Index Config used by the `HadoopBatchIndexer` except that three fields must be omitted: `segmentOutputPath`, `workingPath`, `updaterJobSpec`. The Indexing Service takes care of setting these fields internally.
#### Using your own Hadoop distribution
-Druid is compiled against Apache hadoop-core 1.0.3. However, if you happen to use a different flavor of hadoop that is API compatible with hadoop-core 1.0.3, you should only have to change the hadoopCoordinates property to point to the maven artifact used by your distribution.
+Druid is compiled against Apache hadoop-client 2.3.0. However, if you happen to use a different flavor of hadoop that is API compatible with hadoop-client 2.3.0, you should only have to change the hadoopCoordinates property to point to the maven artifact used by your distribution.
#### Resolving dependency conflicts running HadoopIndexTask
diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml
index f3dd043..edf3703 100644
--- a/hdfs-storage/pom.xml
+++ b/hdfs-storage/pom.xml
@@ -52,7 +52,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <artifactId>hadoop-client</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml
index d303649..58e1b2c 100644
--- a/indexing-hadoop/pom.xml
+++ b/indexing-hadoop/pom.xml
@@ -67,7 +67,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml
index 29dbcb5..ecbe4a9 100644
--- a/indexing-service/pom.xml
+++ b/indexing-service/pom.xml
@@ -71,7 +71,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <artifactId>hadoop-client</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java
index ca38f90..7e9d403 100644
--- a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java
+++ b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java
@@ -66,7 +66,7 @@
extensionsConfig = Initialization.makeStartupInjector().getInstance(ExtensionsConfig.class);
}
- private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-core:1.0.3";
+ private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-client:2.3.0";
@JsonIgnore
private final HadoopDruidIndexerSchema schema;
@JsonIgnore
diff --git a/pom.xml b/pom.xml
index ca808db..464fd9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -373,7 +373,7 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>2.4.0a</version>
+ <version>2.5.0</version>
</dependency>
<dependency>
<groupId>io.tesla.aether</groupId>
@@ -402,8 +402,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>1.0.3</version>
+ <artifactId>hadoop-client</artifactId>
+ <version>2.3.0</version>
<scope>provided</scope>
</dependency>
diff --git a/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java b/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java
index 965859f..e792595 100644
--- a/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java
+++ b/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java
@@ -23,6 +23,7 @@
package io.druid.data.input;
import com.google.protobuf.AbstractMessage;
+import com.google.protobuf.UnknownFieldSet;
public final class ProtoTestEventWrapper {
private ProtoTestEventWrapper() {}
@@ -85,7 +86,13 @@
public ProtoTestEvent getDefaultInstanceForType() {
return defaultInstance;
}
-
+
+ @Override
+ public UnknownFieldSet getUnknownFields()
+ {
+ return UnknownFieldSet.getDefaultInstance();
+ }
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_descriptor;
@@ -1049,6 +1056,7 @@
new com.google.protobuf.Descriptors.FileDescriptor[] {
}, assigner);
}
+
// @@protoc_insertion_point(outer_class_scope)
}
diff --git a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java
index 3b20a3a..cf81cc7 100644
--- a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java
+++ b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java
@@ -45,18 +45,21 @@
)
public class CliHadoopIndexer implements Runnable
{
+
+ private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-client:2.3.0";
+
private static final Logger log = new Logger(CliHadoopIndexer.class);
@Arguments(description = "A JSON object or the path to a file that contains a JSON object", required = true)
private String argumentSpec;
@Option(name = "hadoop",
- description = "The maven coordinates to the version of hadoop to run with. Defaults to org.apache.hadoop:hadoop-core:1.0.3")
- private String hadoopCoordinates = "org.apache.hadoop:hadoop-core:1.0.3";
+ description = "The maven coordinates to the version of hadoop to run with. Defaults to org.apache.hadoop:hadoop-client:2.3.0")
+ private String hadoopCoordinates = defaultHadoopCoordinates;
@Option(name = "hadoopDependencies",
- description = "The maven coordinates to the version of hadoop and all dependencies to run with. Defaults to using org.apache.hadoop:hadoop-core:1.0.3")
- private List<String> hadoopDependencyCoordinates = Arrays.<String>asList("org.apache.hadoop:hadoop-core:1.0.3");
+ description = "The maven coordinates to the version of hadoop and all dependencies to run with. Defaults to using org.apache.hadoop:hadoop-client:2.3.0")
+ private List<String> hadoopDependencyCoordinates = Arrays.<String>asList(defaultHadoopCoordinates);
@Inject
private ExtensionsConfig extensionsConfig = null;
