BIGTOP-3287. Bump Hadoop to 2.10.0. (#578)
diff --git a/bigtop-packages/src/common/hadoop/do-component-build b/bigtop-packages/src/common/hadoop/do-component-build
index 2a1a634..adc02cc 100644
--- a/bigtop-packages/src/common/hadoop/do-component-build
+++ b/bigtop-packages/src/common/hadoop/do-component-build
@@ -120,7 +120,7 @@
if [ ! -z ${LOCAL_DOWNLOADS_DIR-} ] && [ -f $LOCAL_DOWNLOADS_DIR/tomcat/apache-tomcat-$BIGTOP_TOMCAT_VERSION.tar.gz ]; then
BIGTOP_TOMCAT_DOWNLOAD_URL=file://$LOCAL_DOWNLOADS_DIR/tomcat/apache-tomcat-$BIGTOP_TOMCAT_VERSION.tar.gz
else
- BIGTOP_TOMCAT_DOWNLOAD_URL=http://archive.apache.org/dist/tomcat/tomcat-6/v${BIGTOP_TOMCAT_VERSION}/bin/apache-tomcat-${BIGTOP_TOMCAT_VERSION}.tar.gz
+ BIGTOP_TOMCAT_DOWNLOAD_URL=http://archive.apache.org/dist/tomcat/tomcat-8/v${BIGTOP_TOMCAT_VERSION}/bin/apache-tomcat-${BIGTOP_TOMCAT_VERSION}.tar.gz
fi
# Build artifacts
diff --git a/bigtop-packages/src/common/hadoop/init-hdfs.sh b/bigtop-packages/src/common/hadoop/init-hdfs.sh
index 0a94c77..a669e1d 100755
--- a/bigtop-packages/src/common/hadoop/init-hdfs.sh
+++ b/bigtop-packages/src/common/hadoop/init-hdfs.sh
@@ -32,15 +32,4 @@
. /usr/lib/bigtop-utils/bigtop-detect-javahome
fi
-HADOOP_LIB_DIR=/usr/lib/hadoop/lib
-HDFS_LIB_DIR=/usr/lib/hadoop-hdfs/lib
-HADOOP_DEPENDENCIES="commons-logging*.jar commons-io*.jar guava*.jar commons-configuration*.jar commons-collections*.jar slf4j-api*.jar protobuf-java*.jar commons-lang*.jar servlet-api-2.5.jar"
-HDFS_DEPENDENCIES="htrace-core*.jar jackson-*.jar"
-for i in /usr/lib/hadoop/*.jar; do CLASSPATH=$CLASSPATH:$i; done
-CLASSPATH=/etc/hadoop/conf:$CLASSPATH:/usr/lib/hadoop-hdfs/hadoop-hdfs.jar:/usr/lib/hadoop-hdfs/hadoop-hdfs-client.jar
-pushd .
-cd $HADOOP_LIB_DIR
-for d in $HADOOP_DEPENDENCIES; do CLASSPATH=$CLASSPATH:$HADOOP_LIB_DIR/$d; done
-for d in $HDFS_DEPENDENCIES; do CLASSPATH=$CLASSPATH:$HDFS_LIB_DIR/$d; done
-popd
-su -s /bin/bash hdfs -c "/usr/lib/bigtop-groovy/bin/groovy -classpath $CLASSPATH /usr/lib/hadoop/libexec/init-hcfs.groovy /usr/lib/hadoop/libexec/init-hcfs.json"
+su -s /bin/bash hdfs -c "/usr/lib/bigtop-groovy/bin/groovy -classpath $(hadoop classpath) /usr/lib/hadoop/libexec/init-hcfs.groovy /usr/lib/hadoop/libexec/init-hcfs.json"
diff --git a/bigtop-packages/src/common/hadoop/install_hadoop.sh b/bigtop-packages/src/common/hadoop/install_hadoop.sh
index 9957dcf..1926862 100755
--- a/bigtop-packages/src/common/hadoop/install_hadoop.sh
+++ b/bigtop-packages/src/common/hadoop/install_hadoop.sh
@@ -200,6 +200,8 @@
cp ${BUILD_DIR}/share/hadoop/hdfs/*.jar ${HDFS_DIR}/
install -d -m 0755 ${YARN_DIR}
cp ${BUILD_DIR}/share/hadoop/yarn/hadoop-yarn*.jar ${YARN_DIR}/
+install -d -m 0755 ${YARN_DIR}/timelineservice
+cp ${BUILD_DIR}/share/hadoop/yarn/timelineservice/hadoop-yarn*.jar ${YARN_DIR}/timelineservice
chmod 644 ${HADOOP_DIR}/*.jar ${MAPREDUCE_DIR}/*.jar ${HDFS_DIR}/*.jar ${YARN_DIR}/*.jar
# lib jars
@@ -211,7 +213,9 @@
cp ${BUILD_DIR}/share/hadoop/hdfs/lib/*.jar ${HDFS_DIR}/lib
install -d -m 0755 ${YARN_DIR}/lib
cp ${BUILD_DIR}/share/hadoop/yarn/lib/*.jar ${YARN_DIR}/lib
-chmod 644 ${HADOOP_DIR}/lib/*.jar ${MAPREDUCE_DIR}/lib/*.jar ${HDFS_DIR}/lib/*.jar ${YARN_DIR}/lib/*.jar
+install -d -m 0755 ${YARN_DIR}/timelineservice/lib
+cp ${BUILD_DIR}/share/hadoop/yarn/timelineservice/lib/*.jar ${YARN_DIR}/timelineservice/lib
+chmod 644 ${HADOOP_DIR}/lib/*.jar ${MAPREDUCE_DIR}/lib/*.jar ${HDFS_DIR}/lib/*.jar ${YARN_DIR}/lib/*.jar ${YARN_DIR}/timelineservice/lib/*.jar
# Install webapps
cp -ra ${BUILD_DIR}/share/hadoop/hdfs/webapps ${HDFS_DIR}/
diff --git a/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff b/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff
deleted file mode 100644
index 026c01f..0000000
--- a/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff
+++ /dev/null
@@ -1,125 +0,0 @@
-From f10ec9f9bec56bcfa5e2015ea87a8a5054d096c5 Mon Sep 17 00:00:00 2001
-From: Anton Chevychalov <pub@mnu.pp.ru>
-Date: Tue, 5 Sep 2017 15:14:19 +0300
-Subject: [PATCH] [PATCH] HADOOP-12366. expose calculated paths
-
-Adapted the patch to apply cleanly on branch-2.8.1
----
- hadoop-common-project/hadoop-common/src/main/bin/hadoop | 10 ++++++++++
- hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 9 +++++++++
- hadoop-mapreduce-project/bin/mapred | 9 +++++++++
- hadoop-yarn-project/hadoop-yarn/bin/yarn | 9 +++++++++
- 4 files changed, 37 insertions(+)
-
-diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
-index 1575996..44be2e3 100755
---- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
-+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
-@@ -38,6 +38,7 @@ function print_usage(){
- echo " YARN applications, not this command."
- echo " checknative [-a|-h] check native hadoop and compression libraries availability"
- echo " distcp <srcurl> <desturl> copy file or directories recursively"
-+ echo " envvars display computed Hadoop environment variables"
- echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive"
- echo " classpath prints the class path needed to get the"
- echo " Hadoop jar and the required libraries"
-@@ -116,6 +117,15 @@ case $COMMAND in
- CLASSPATH=${CLASSPATH}:${TOOL_PATH}
- elif [ "$COMMAND" = "daemonlog" ] ; then
- CLASS=org.apache.hadoop.log.LogLevel
-+ elif [ "$COMMAND" = "envvars" ] ; then
-+ echo "JAVA_HOME='${JAVA_HOME}'"
-+ echo "HADOOP_COMMON_HOME='${HADOOP_COMMON_HOME}'"
-+ echo "HADOOP_COMMON_DIR='${HADOOP_COMMON_DIR}'"
-+ echo "HADOOP_COMMON_LIB_JARS_DIR='${HADOOP_COMMON_LIB_JARS_DIR}'"
-+ echo "HADOOP_COMMON_LIB_NATIVE_DIR='${HADOOP_COMMON_LIB_NATIVE_DIR}'"
-+ echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
-+ echo "HADOOP_TOOLS_PATH='${TOOL_PATH}'"
-+ exit 0
- elif [ "$COMMAND" = "archive" ] ; then
- CLASS=org.apache.hadoop.tools.HadoopArchives
- CLASSPATH=${CLASSPATH}:${TOOL_PATH}
-diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
-index fbfbaf2..714280e 100755
---- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
-+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
-@@ -47,6 +47,7 @@ function print_usage(){
- echo " datanode run a DFS datanode"
- echo " debug run a Debug Admin to execute HDFS debug commands"
- echo " dfsadmin run a DFS admin client"
-+ echo " envvars display computed Hadoop environment variables"
- echo " haadmin run a DFS HA admin client"
- echo " fsck run a DFS filesystem checking utility"
- echo " balancer run a cluster balancing utility"
-@@ -161,6 +162,14 @@ elif [ "$COMMAND" = "haadmin" ] ; then
- CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin
- CLASSPATH=${CLASSPATH}:${TOOL_PATH}
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-+elif [ "$COMMAND" = "envvars" ] ; then
-+ echo "JAVA_HOME='${JAVA_HOME}'"
-+ echo "HADOOP_HDFS_HOME='${HADOOP_HDFS_HOME}'"
-+ echo "HDFS_DIR='${HDFS_DIR}'"
-+ echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'"
-+ echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
-+ echo "HADOOP_TOOLS_PATH='${TOOL_PATH}'"
-+ exit 0
- elif [ "$COMMAND" = "fsck" ] ; then
- CLASS=org.apache.hadoop.hdfs.tools.DFSck
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
-index 6f3bc04..fd1300d 100755
---- a/hadoop-mapreduce-project/bin/mapred
-+++ b/hadoop-mapreduce-project/bin/mapred
-@@ -38,6 +38,7 @@ function print_usage(){
- echo " mapreduce subcommands"
- echo " historyserver run job history servers as a standalone daemon"
- echo " distcp <srcurl> <desturl> copy file or directories recursively"
-+ echo " envvars display computed Hadoop environment variables"
- echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive"
- echo " archive-logs combine aggregated logs into hadoop archives"
- echo " hsadmin job history server admin interface"
-@@ -75,6 +76,14 @@ elif [ "$COMMAND" = "sampler" ] ; then
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
- elif [ "$COMMAND" = "classpath" ] ; then
- echo -n
-+elif [ "$COMMAND" = "envvars" ] ; then
-+ echo "JAVA_HOME='${JAVA_HOME}'"
-+ echo "HADOOP_MAPRED_HOME='${HADOOP_MAPRED_HOME}'"
-+ echo "MAPRED_DIR='${MAPRED_DIR}'"
-+ echo "MAPRED_LIB_JARS_DIR='${MAPRED_LIB_JARS_DIR}'"
-+ echo "HADOOP_CONF_DIR=${HADOOP_CONF_DIR}'"
-+ echo "HADOOP_TOOLS_PATH='${TOOL_PATH}'"
-+ exit 0
- elif [ "$COMMAND" = "historyserver" ] ; then
- CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
- HADOOP_OPTS="$HADOOP_OPTS -Dmapred.jobsummary.logger=${HADOOP_JHS_LOGGER:-INFO,console} $HADOOP_JOB_HISTORYSERVER_OPTS"
-diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
-index 552cef4..5495387 100644
---- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
-+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
-@@ -89,6 +89,7 @@ function print_usage(){
- echo " cluster prints cluster information"
- echo " daemonlog get/set the log level for each"
- echo " daemon"
-+ echo " envvars display computed Hadoop environment variables"
- echo " top run cluster usage tool"
- echo ""
- echo "Most commands print help when invoked w/o parameters."
-@@ -217,6 +218,14 @@ if [ "$COMMAND" = "classpath" ] ; then
- echo $CLASSPATH
- exit 0
- fi
-+elif [ "$COMMAND" = "envvars" ] ; then
-+ echo "JAVA_HOME='${JAVA_HOME}'"
-+ echo "HADOOP_YARN_HOME='${HADOOP_YARN_HOME}'"
-+ echo "YARN_DIR='${YARN_DIR}'"
-+ echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'"
-+ echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
-+ echo "HADOOP_TOOLS_PATH='${TOOL_PATH}'"
-+ exit 0
- elif [ "$COMMAND" = "rmadmin" ] ; then
- CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
---
-2.7.4
-
diff --git a/bigtop-packages/src/common/hadoop/patch1-YARN-9945.diff b/bigtop-packages/src/common/hadoop/patch1-YARN-9945.diff
new file mode 100644
index 0000000..fb8eba3
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/patch1-YARN-9945.diff
@@ -0,0 +1,18 @@
+commit ae3f5d66e4fac859f31a66870e079c91ba8993d9
+Author: Jonathan Hung <jhung@linkedin.com>
+Date: Thu Oct 31 15:30:23 2019 -0700
+
+ YARN-9945. Fix javadoc in FederationProxyProviderUtil in branch-2
+
+diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/Feder
+index 33883b9..91924da 100644
+--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
++++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
+@@ -28,6 +28,7 @@
+ import org.apache.hadoop.security.token.Token;
+ import org.apache.hadoop.security.token.TokenIdentifier;
+ import org.apache.hadoop.yarn.client.AMRMClientUtils;
++import org.apache.hadoop.yarn.client.ClientRMProxy;
+ import org.apache.hadoop.yarn.client.RMFailoverProxyProvider;
+ import org.apache.hadoop.yarn.conf.HAUtil;
+ import org.apache.hadoop.yarn.conf.YarnConfiguration;
diff --git a/bigtop.bom b/bigtop.bom
index 40db046..07d29d2 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -143,7 +143,7 @@
'hadoop' {
name = 'hadoop'
relNotes = 'Apache Hadoop'
- version { base = '2.8.5'; pkg = base; release = 1 }
+ version { base = '2.10.0'; pkg = base; release = 1 }
tarball { destination = "${name}-${version.base}.tar.gz"
source = "${name}-${version.base}-src.tar.gz" }
url { download_path = "/$name/common/$name-${version.base}"
