PIG-5428: Update hadoop2,3 and tez to recent versions (knoguchi)
git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1904553 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index d942bda..f9f9916 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -26,6 +26,8 @@
IMPROVEMENTS
+PIG-5428: Update hadoop2,3 and tez to recent versions (knoguchi)
+
PIG-5422: Upgrade guava/groovy dependency (knoguchi via rohini)
PIG-5421: Upgrade commons dependencies (knoguchi via rohini)
diff --git a/ivy.xml b/ivy.xml
index 0f171b4..dddd086 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -86,8 +86,8 @@
conf="hadoop2->master;hadoop3->master"/>
<dependency org="commons-collections" name="commons-collections" rev="${commons-collections.version}"
conf="hadoop2->master;hadoop3->master"/>
- <dependency org="javax.servlet" name="servlet-api" rev="${servlet-api.version}"
- conf="hadoop2->master;hadoop3->master"/>
+ <dependency org="javax.servlet" name="servlet-api" rev="${servlet-api.version}" conf="hadoop2->master"/>
+ <dependency org="javax.servlet" name="javax.servlet-api" rev="${servlet-api.version}" conf="hadoop3->master"/>
<dependency org="javax.ws.rs" name="jsr311-api" rev="${jsr311-api.version}"
conf="hadoop2->master;hadoop3->master"/>
<dependency org="com.google.protobuf" name="protobuf-java" rev="${protobuf-java.version}"
@@ -113,6 +113,11 @@
<dependency org="com.codahale.metrics" name="metrics-core" rev="${codahale.metrics-core.version}"
conf="hadoop3->master"/>
+ <!-- Hadoop 2 dependencies -->
+ <dependency org="com.fasterxml.woodstox" name="woodstox-core" rev="${woodstox.version}" conf="hadoop2->master"/>
+ <dependency org="org.codehaus.woodstox" name="stax2-api" rev="${stax2-api.version}" conf="hadoop2->master"/>
+ <dependency org="org.mortbay.jetty" name="jetty-sslengine" rev="6.1.26" conf="hadoop2->master"/>
+
<!-- Hadoop 2/3 dependencies -->
<dependency org="org.apache.hadoop" name="hadoop-annotations"
rev="${hadoop-common.version}" conf="hadoop2->master;hadoop3->master"/>
@@ -168,16 +173,13 @@
rev="${hadoop-mapreduce.version}" conf="hadoop2->master;hadoop3->master"/>
<dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-hs"
rev="${hadoop-mapreduce.version}" conf="hadoop2->master;hadoop3->master"/>
-
- <!--Hadoop 3 dependencies-->
<dependency org="org.apache.hadoop" name="hadoop-hdfs-client"
- rev="${hadoop-hdfs.version}" conf="hadoop3->master">
+ rev="${hadoop-hdfs.version}" conf="hadoop2->master;hadoop3->master">
<artifact name="hadoop-hdfs-client" ext="jar" />
<artifact name="hadoop-hdfs-client" type="tests" ext="jar" m:classifier="tests" />
</dependency>
-
<dependency org="org.apache.hadoop" name="hadoop-yarn-server-timelineservice"
- rev="${hadoop-mapreduce.version}" conf="hadoop3->master"/>
+ rev="${hadoop-mapreduce.version}" conf="hadoop2->master;hadoop3->master"/>
<dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"
conf="compile->master;checkstyle->master"/>
@@ -484,15 +486,15 @@
<dependency org="org.apache.hbase.thirdparty" name="hbase-shaded-protobuf" rev="2.1.0" conf="hbase2->master" />
- <dependency org="org.eclipse.jetty" name="jetty-http" rev="9.3.20.v20170531" conf="hbase2->master"/>
- <dependency org="org.eclipse.jetty" name="jetty-io" rev="9.3.20.v20170531" conf="hbase2->master"/>
- <dependency org="org.eclipse.jetty" name="jetty-security" rev="9.3.20.v20170531" conf="hbase2->master"/>
- <dependency org="org.eclipse.jetty" name="jetty-server" rev="9.3.20.v20170531" conf="hbase2->master"/>
- <dependency org="org.eclipse.jetty" name="jetty-servlet" rev="9.3.20.v20170531" conf="hbase2->master"/>
- <dependency org="org.eclipse.jetty" name="jetty-util" rev="9.3.20.v20170531" conf="hbase2->master"/>
- <dependency org="org.eclipse.jetty" name="jetty-util-ajax" rev="9.3.20.v20170531" conf="hbase2->master"/>
- <dependency org="org.eclipse.jetty" name="jetty-webapp" rev="9.3.20.v20170531" conf="hbase2->master"/>
- <dependency org="org.eclipse.jetty" name="jetty-xml" rev="9.3.20.v20170531" conf="hbase2->master"/>
+ <dependency org="org.eclipse.jetty" name="jetty-http" rev="9.4.40.v20210413" conf="hbase2->master"/>
+ <dependency org="org.eclipse.jetty" name="jetty-io" rev="9.4.40.v20210413" conf="hbase2->master"/>
+ <dependency org="org.eclipse.jetty" name="jetty-security" rev="9.4.40.v20210413" conf="hbase2->master"/>
+ <dependency org="org.eclipse.jetty" name="jetty-server" rev="9.4.40.v20210413" conf="hbase2->master"/>
+ <dependency org="org.eclipse.jetty" name="jetty-servlet" rev="9.4.40.v20210413" conf="hbase2->master"/>
+ <dependency org="org.eclipse.jetty" name="jetty-util" rev="9.4.40.v20210413" conf="hbase2->master"/>
+ <dependency org="org.eclipse.jetty" name="jetty-util-ajax" rev="9.4.40.v20210413" conf="hbase2->master"/>
+ <dependency org="org.eclipse.jetty" name="jetty-webapp" rev="9.4.40.v20210413" conf="hbase2->master"/>
+ <dependency org="org.eclipse.jetty" name="jetty-xml" rev="9.4.40.v20210413" conf="hbase2->master"/>
<dependency org="com.lmax" name="disruptor" rev="3.3.6" conf="hbase2->master"/>
<!-- End of HBase dependencies -->
@@ -627,6 +629,8 @@
<exclude org="com.esotericsoftware.kryo" />
<exclude org="jline" module="jline"/>
<exclude org="com.google.guava" />
+ <exclude org="io.netty" module="netty"/>
+ <exclude org="io.netty" name="netty-all"/>
</dependency>
<dependency org="org.apache.spark" name="spark-yarn_2.11" rev="${spark2.version}" conf="spark2->default">
<exclude org="org.apache.hadoop" />
@@ -654,6 +658,8 @@
conf="hadoop2->master;hadoop3->master"/>
<dependency org="org.apache.tez" name="tez-yarn-timeline-history-with-acls" rev="${tez.version}"
conf="hadoop2->master;hadoop3->master"/>
+ <dependency org="org.apache.tez" name="hadoop-shim" rev="${tez.version}"
+ conf="hadoop2->master;hadoop3->master"/>
<dependency org="org.apache.commons" name="commons-collections4" rev="${commons-collections4.version}"
conf="hadoop2->master;hadoop3->master"/>
<dependency org="org.codehaus.jettison" name="jettison" rev="${jettison.version}"
diff --git a/ivy/libraries-h2.properties b/ivy/libraries-h2.properties
index df2c9df..4108c47 100644
--- a/ivy/libraries-h2.properties
+++ b/ivy/libraries-h2.properties
@@ -14,10 +14,12 @@
#It drives ivy and the generation of a maven POM
#These are the versions of our Hadoop 2 dependencies (core and other libs separated and in alphabetical order)
-hadoop-common.version=2.7.3
-hadoop-hdfs.version=2.7.3
-hadoop-mapreduce.version=2.7.3
+hadoop-common.version=2.10.2
+hadoop-hdfs.version=2.10.2
+hadoop-mapreduce.version=2.10.2
accumulo.version=1.5.0
-netty.version=3.6.6.Final
-netty-all.version=4.0.23.Final
\ No newline at end of file
+netty.version=3.10.6.Final
+netty-all.version=4.1.50.Final
+tez.version=0.9.2
+servlet-api.version=2.5
diff --git a/ivy/libraries-h3.properties b/ivy/libraries-h3.properties
index a05e037..41317ff 100644
--- a/ivy/libraries-h3.properties
+++ b/ivy/libraries-h3.properties
@@ -14,12 +14,14 @@
#It drives ivy and the generation of a maven POM
#These are the versions of our Hadoop 3 dependencies (core and other libs separated and in alphabetical order)
-hadoop-common.version=3.1.1
-hadoop-hdfs.version=3.1.1
-hadoop-mapreduce.version=3.1.1
+hadoop-common.version=3.2.4
+hadoop-hdfs.version=3.2.4
+hadoop-mapreduce.version=3.2.4
accumulo.version=1.6.0
-netty.version=3.10.5.Final
+netty.version=3.10.6.Final
codahale.metrics-core.version=3.0.1
-netty-all.version=4.1.0.Beta5
-re2j.version=1.0
\ No newline at end of file
+netty-all.version=4.1.77.Final
+re2j.version=1.0
+tez.version=0.10.2
+servlet-api.version=3.1.0
diff --git a/ivy/libraries.properties b/ivy/libraries.properties
index 321a48f..2b52dc8 100644
--- a/ivy/libraries.properties
+++ b/ivy/libraries.properties
@@ -17,7 +17,7 @@
apacheant.version=1.7.1
apacherat.version=0.8
automaton.version=1.11-8
-avro.version=1.7.5
+avro.version=1.7.7
basjes-httpdlog-pigloader.version=5.0
commons-beanutils.version=1.9.4
commons-cli.version=1.5.0
@@ -63,8 +63,6 @@
antlr.version=3.4
stringtemplate.version=4.0.4
log4j.version=1.2.16
-netty.version=3.6.6.Final
-netty-all.version=4.1.17.Final
rats-lib.version=0.5.1
slf4j-api.version=1.6.1
slf4j-log4j12.version=1.6.1
@@ -75,7 +73,6 @@
wagon-http.version=1.0-beta-2
zookeeper.version=3.4.5
servlet.version=4.0.6
-servlet-api.version=2.5
protobuf-java.version=2.5.0
metrics-core.version=2.1.2
guice.version=3.0
@@ -86,15 +83,16 @@
jansi.version=1.9
asm.version=3.3.1
snappy-java.version=1.1.1.3
-tez.version=0.7.0
parquet-pig-bundle.version=1.9.0
snappy.version=0.2
leveldbjni.version=1.8
curator.version=2.6.0
htrace.version=3.2.0-incubating
htrace4.version=4.0.1-incubating
-commons-lang3.version=3.6
+commons-lang3.version=3.12.0
scala-xml.version=1.0.5
glassfish.el.version=3.0.1-b08
roaring-bitmap-shaded.version=0.7.14
dependency-check-ant.version=5.2.4
+woodstox.version=5.3.0
+stax2-api.version=4.2.1
diff --git a/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java b/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
index 4b5da89..649734d 100644
--- a/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
+++ b/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
@@ -116,6 +116,7 @@
if (pc.getExecType().isLocal()) {
pc.getProperties().setProperty(TezConfiguration.TEZ_LOCAL_MODE, "true");
pc.getProperties().setProperty(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, "true");
+ pc.getProperties().setProperty("tez.runtime.transfer.data-via-events.enabled","false");
pc.getProperties().setProperty(TezConfiguration.TEZ_IGNORE_LIB_URIS, "true");
pc.getProperties().setProperty(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, DAGSchedulerNaturalOrderControlled.class.getName());
}
diff --git a/src/org/apache/pig/impl/util/avro/AvroRecordWriter.java b/src/org/apache/pig/impl/util/avro/AvroRecordWriter.java
index 144c510..f69e525 100644
--- a/src/org/apache/pig/impl/util/avro/AvroRecordWriter.java
+++ b/src/org/apache/pig/impl/util/avro/AvroRecordWriter.java
@@ -20,7 +20,7 @@
import static org.apache.avro.file.DataFileConstants.DEFAULT_SYNC_INTERVAL;
import static org.apache.avro.file.DataFileConstants.DEFLATE_CODEC;
-import static org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL;
+import static org.apache.avro.file.CodecFactory.DEFAULT_DEFLATE_LEVEL;
import static org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY;
import static org.apache.avro.mapred.AvroOutputFormat.SYNC_INTERVAL_KEY;
diff --git a/test/org/apache/pig/test/TezMiniCluster.java b/test/org/apache/pig/test/TezMiniCluster.java
index 56c908c..87ed87e 100644
--- a/test/org/apache/pig/test/TezMiniCluster.java
+++ b/test/org/apache/pig/test/TezMiniCluster.java
@@ -57,6 +57,7 @@
// Write tez-site.xml
Configuration tez_conf = new Configuration(false);
tez_conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, "false");
+ tez_conf.set("tez.runtime.transfer.data-via-events.enabled", "false");
tez_conf.set("tez.lib.uris", "hdfs:///tez,hdfs:///tez/lib");
// Set to a lower value so that tests don't get stuck for long because of 1 AM running at a time
tez_conf.set(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, "20");
