BIGTOP-3501. Update hadoop-kms resources based on hadoop-3. (#738)
* BIGTOP-3501. Update hadoop-kms resources based on hadoop-3.
* added blank kms-site.xml to hadoop-kms package.
* added example hadoop-env.sh to hadoop-kms package.
diff --git a/bigtop-packages/src/common/hadoop/hadoop-kms.svc b/bigtop-packages/src/common/hadoop/hadoop-kms.svc
index cf44a48..50d83e4 100644
--- a/bigtop-packages/src/common/hadoop/hadoop-kms.svc
+++ b/bigtop-packages/src/common/hadoop/hadoop-kms.svc
@@ -15,11 +15,11 @@
HADOOP_NAME="kms"
DAEMON="hadoop-$HADOOP_NAME"
DESC="Hadoop $HADOOP_NAME"
-EXEC_PATH="/usr/lib/hadoop-kms/sbin/kms.sh"
+EXEC_PATH="/usr/lib/hadoop/sbin/kms.sh"
SVC_USER="$HADOOP_NAME"
WORKING_DIR="/var/run/hadoop-kms"
DAEMON_FLAGS="$HADOOP_NAME"
-CONF_DIR="/etc/$DAEMON/conf"
+CONF_DIR="/etc/hadoop/conf"
PIDFILE="/var/run/$DAEMON/hadoop-$SVC_USER-$HADOOP_NAME.pid"
CHKCONFIG=${CHKCONFIG:-"2345 90 10"}
@@ -28,42 +28,27 @@
generate_start() {
cat <<'__EOT__'
+
start() {
[ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
[ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED
- export KMS_USER="$SVC_USER"
- export KMS_CONFIG="$CONF_DIR"
- export KMS_LOG=${KMS_LOG:-"/var/log/hadoop-kms/"}
- export KMS_TEMP="$(dirname $PIDFILE)"
- export KMS_SLEEP_TIME="$SLEEP_TIME"
- export CATALINA_BASE=${CATALINA_BASE:-"/var/lib/hadoop-kms/tomcat-deployment"}
- export CATALINA_PID="$PIDFILE"
- export CATALINA_TMPDIR="$KMS_TEMP"
+ runuser -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && $EXEC_PATH start $DAEMON_FLAGS"
- . /usr/lib/hadoop-kms/tomcat-deployment.sh
-
- runuser -s /bin/bash -c "${EXEC_PATH} start $DAEMON_FLAGS" $KMS_USER
-
- for second in {5..0}
- do
- checkstatusofproc
- RETVAL=$?
- if [ "$RETVAL" -eq $RETVAL_SUCCESS ] ; then
- break
- fi
- sleep 1
- done
+ # Some processes are slow to start
+ sleep $SLEEP_TIME
+ checkstatusofproc
+ RETVAL=$?
if [ $RETVAL -eq $STATUS_RUNNING ]; then
touch $LOCKFILE
- log_success_msg "Started ${DESC} (${DAEMON}): "
+ log_success_msg "Started ${DESC}: "
else
- log_failure_msg "Failed to start ${DESC}. Return value: $RETVAL"
+ log_failure_msg "Failed to start ${DESC}. Return value: $RETVAL"
fi
-
return $RETVAL
}
+
__EOT__
}
@@ -71,24 +56,19 @@
generate_stop() {
cat <<'__EOT__'
-stop() {
- # FIXME: workaround for BIGTOP-537
- checkstatusofproc
- if [ "$?" = "$STATUS_RUNNING" ] ; then
- su -s /bin/bash $SVC_USER -c "${EXEC_PATH} stop $SLEEP_TIME -force"
- RETVAL=$?
- else
- RETVAL=$RETVAL_SUCCESS
- fi
+stop() {
+ start_daemon $EXEC_PATH stop $DAEMON_FLAGS
+ RETVAL=$?
if [ $RETVAL -eq $RETVAL_SUCCESS ]; then
- log_success_msg "Stopped ${DESC}: "
- rm -f $LOCKFILE $PIDFILE
- else
- log_failure_msg "Failure to stop ${DESC}. Return value: $RETVAL"
- fi
+ log_success_msg "Stopped ${DESC}: "
+ rm -f $LOCKFILE $PIDFILE
+ else
+ log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL."
+ fi
}
+
__EOT__
}
diff --git a/bigtop-packages/src/common/hadoop/install_hadoop.sh b/bigtop-packages/src/common/hadoop/install_hadoop.sh
index 7d0b902..b79e82f 100755
--- a/bigtop-packages/src/common/hadoop/install_hadoop.sh
+++ b/bigtop-packages/src/common/hadoop/install_hadoop.sh
@@ -41,7 +41,6 @@
-l 'installed-lib-dir:' \
-l 'hadoop-dir:' \
-l 'httpfs-dir:' \
- -l 'kms-dir:' \
-l 'hdfs-dir:' \
-l 'yarn-dir:' \
-l 'mapreduce-dir:' \
@@ -72,9 +71,6 @@
--httpfs-dir)
HTTPFS_DIR=$2 ; shift 2
;;
- --kms-dir)
- KMS_DIR=$2 ; shift 2
- ;;
--hadoop-dir)
HADOOP_DIR=$2 ; shift 2
;;
@@ -144,7 +140,6 @@
MAPREDUCE_DIR=${MAPREDUCE_DIR:-$PREFIX/usr/lib/hadoop-mapreduce}
CLIENT_DIR=${CLIENT_DIR:-$PREFIX/usr/lib/hadoop/client}
HTTPFS_DIR=${HTTPFS_DIR:-$PREFIX/usr/lib/hadoop-httpfs}
-KMS_DIR=${KMS_DIR:-$PREFIX/usr/lib/hadoop-kms}
SYSTEM_LIB_DIR=${SYSTEM_LIB_DIR:-/usr/lib}
BIN_DIR=${BIN_DIR:-$PREFIX/usr/bin}
DOC_DIR=${DOC_DIR:-$PREFIX/usr/share/doc/hadoop}
@@ -236,7 +231,7 @@
# sbin
install -d -m 0755 ${HADOOP_DIR}/sbin
-cp -a ${BUILD_DIR}/sbin/{hadoop-daemon,hadoop-daemons,workers}.sh ${HADOOP_DIR}/sbin
+cp -a ${BUILD_DIR}/sbin/{hadoop-daemon,hadoop-daemons,workers,kms}.sh ${HADOOP_DIR}/sbin
install -d -m 0755 ${HDFS_DIR}/sbin
cp -a ${BUILD_DIR}/sbin/{distribute-exclude,refresh-namenodes}.sh ${HDFS_DIR}/sbin
install -d -m 0755 ${YARN_DIR}/sbin
@@ -330,8 +325,6 @@
install -d -m 0755 ${PREFIX}/var/lib/hadoop-httpfs
# KMS
-install -d -m 0755 ${KMS_DIR}/sbin
-cp ${BUILD_DIR}/sbin/kms.sh ${KMS_DIR}/sbin/
install -d -m 0755 ${PREFIX}/var/lib/hadoop-kms
@@ -362,7 +355,7 @@
install -d -m 0755 $PREFIX/var/{log,run,lib}/hadoop-mapreduce
# Remove all source and create version-less symlinks to offer integration point with other projects
-for DIR in ${HADOOP_DIR} ${HDFS_DIR} ${YARN_DIR} ${MAPREDUCE_DIR} ${HTTPFS_DIR} ${KMS_DIR}; do
+for DIR in ${HADOOP_DIR} ${HDFS_DIR} ${YARN_DIR} ${MAPREDUCE_DIR} ${HTTPFS_DIR} ; do
(cd $DIR &&
rm -fv *-sources.jar
rm -fv lib/hadoop-*.jar
diff --git a/bigtop-packages/src/common/hadoop/kms-tomcat-deployment.sh b/bigtop-packages/src/common/hadoop/kms-tomcat-deployment.sh
deleted file mode 100644
index 3872fd0..0000000
--- a/bigtop-packages/src/common/hadoop/kms-tomcat-deployment.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script must be sourced so that it can set CATALINA_BASE for the parent process
-
-TOMCAT_CONF=${TOMCAT_CONF:-`readlink -e /etc/hadoop-kms/tomcat-conf`}
-TOMCAT_DEPLOYMENT=${TOMCAT_DEPLOYMENT:-/var/lib/hadoop-kms/tomcat-deployment}
-KMS_HOME=${KMS_HOME:-/usr/lib/hadoop-kms}
-
-rm -rf ${TOMCAT_DEPLOYMENT}
-mkdir ${TOMCAT_DEPLOYMENT}
-cp -r ${TOMCAT_CONF}/conf ${TOMCAT_DEPLOYMENT}/
-cp -r ${KMS_HOME}/webapps ${TOMCAT_DEPLOYMENT}/
-cp -r ${TOMCAT_CONF}/WEB-INF/* ${TOMCAT_DEPLOYMENT}/webapps/kms/WEB-INF/
-
-if [ -n "${BIGTOP_CLASSPATH}" ] ; then
- sed -i -e "s#^\(common.loader=.*\)\$#\1,${BIGTOP_CLASSPATH/:/,}#" ${TOMCAT_DEPLOYMENT}/conf/catalina.properties
-fi
-
-chown -R kms:kms ${TOMCAT_DEPLOYMENT}
-chmod -R 755 ${TOMCAT_DEPLOYMENT}
-
-export CATALINA_BASE=${TOMCAT_DEPLOYMENT}
diff --git a/bigtop-packages/src/common/hadoop/kms.default b/bigtop-packages/src/common/hadoop/kms.default
index ac537f8..692d477 100644
--- a/bigtop-packages/src/common/hadoop/kms.default
+++ b/bigtop-packages/src/common/hadoop/kms.default
@@ -12,13 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-export KMS_USER=kms
-export KMS_CONFIG=/etc/hadoop-kms/conf
-export KMS_LOG=/var/log/hadoop-kms/
-export KMS_TEMP=/var/run/hadoop-kms/
-export KMS_CATALINA_HOME=/usr/lib/bigtop-tomcat
-export CATALINA_PID=/var/run/hadoop-kms/hadoop-kms-kms.pid
-export CATALINA_BASE=/var/lib/hadoop-kms/tomcat-deployment
-export CATALINA_TMPDIR=/var/run/hadoop-kms/
-# KMS_HTTP_PORT
-# KMS_ADMIN_PORT
+export HADOOP_CONF_DIR=/etc/hadoop/conf
+export HADOOP_LOG_DIR=/var/log/hadoop-kms
+export HADOOP_PID_DIR=/var/run/hadoop-kms
diff --git a/bigtop-packages/src/common/hadoop/patch5-fix-kms-shellprofile.diff b/bigtop-packages/src/common/hadoop/patch5-fix-kms-shellprofile.diff
new file mode 100644
index 0000000..75413b0
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/patch5-fix-kms-shellprofile.diff
@@ -0,0 +1,14 @@
+diff --git a/hadoop-common-project/hadoop-kms/src/main/libexec/shellprofile.d/hadoop-kms.sh b/hadoop-common-project/hadoop-kms/src/main/libexec/shellprofile.d/hadoop-kms.sh
+index 0d084bb36e6..dafe7459f4a 100755
+--- a/hadoop-common-project/hadoop-kms/src/main/libexec/shellprofile.d/hadoop-kms.sh
++++ b/hadoop-common-project/hadoop-kms/src/main/libexec/shellprofile.d/hadoop-kms.sh
+@@ -49,9 +49,4 @@ function hadoop_subcommand_kms
+ "-Dkms.config.dir=${HADOOP_CONF_DIR}"
+ hadoop_add_param HADOOP_OPTS "-Dkms.log.dir=" \
+ "-Dkms.log.dir=${HADOOP_LOG_DIR}"
+-
+- if [[ "${HADOOP_DAEMON_MODE}" == "default" ]] ||
+- [[ "${HADOOP_DAEMON_MODE}" == "start" ]]; then
+- hadoop_mkdir "${KMS_TEMP:-${HADOOP_HOME}/temp}"
+- fi
+ }
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-kms.dirs b/bigtop-packages/src/deb/hadoop/hadoop-kms.dirs
index df40a84..d5ca2e8 100644
--- a/bigtop-packages/src/deb/hadoop/hadoop-kms.dirs
+++ b/bigtop-packages/src/deb/hadoop/hadoop-kms.dirs
@@ -1,2 +1 @@
-/usr/lib/hadoop-kms
/var/log/hadoop-kms
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-kms.install b/bigtop-packages/src/deb/hadoop/hadoop-kms.install
index 63cdd3c..64aa17f 100644
--- a/bigtop-packages/src/deb/hadoop/hadoop-kms.install
+++ b/bigtop-packages/src/deb/hadoop/hadoop-kms.install
@@ -1,3 +1,5 @@
-/etc/default/hadoop-kms
-/usr/lib/hadoop-kms
+/etc/hadoop/conf.empty/kms-acls.xml
+/etc/hadoop/conf.empty/kms-env.sh
+/etc/hadoop/conf.empty/kms-log4j.properties
+/etc/hadoop/conf.empty/kms-site.xml
/var/lib/hadoop-kms
diff --git a/bigtop-packages/src/deb/hadoop/rules b/bigtop-packages/src/deb/hadoop/rules
index 5a00588..de0a1f7 100755
--- a/bigtop-packages/src/deb/hadoop/rules
+++ b/bigtop-packages/src/deb/hadoop/rules
@@ -58,7 +58,6 @@
--distro-dir=debian \
--build-dir=${PWD}/build \
--httpfs-dir=debian/tmp/usr/lib/hadoop-httpfs \
- --kms-dir=debian/tmp/usr/lib/hadoop-kms \
--system-lib-dir=debian/tmp/usr/lib/ \
--system-libexec-dir=debian/tmp/usr/lib/hadoop/libexec/ \
--system-include-dir=debian/tmp/usr/include \
diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
index 9c7e6f6..729f4e3 100644
--- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
+++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
@@ -29,7 +29,6 @@
%define lib_hadoop_dirname /usr/lib
%define lib_hadoop %{lib_hadoop_dirname}/%{name}
%define lib_httpfs %{lib_hadoop_dirname}/%{name}-httpfs
-%define lib_kms %{lib_hadoop_dirname}/%{name}-kms
%define lib_hdfs %{lib_hadoop_dirname}/%{name}-hdfs
%define lib_yarn %{lib_hadoop_dirname}/%{name}-yarn
%define lib_mapreduce %{lib_hadoop_dirname}/%{name}-mapreduce
@@ -511,7 +510,6 @@
--distro-dir=$RPM_SOURCE_DIR \
--build-dir=$PWD/build \
--httpfs-dir=$RPM_BUILD_ROOT%{lib_httpfs} \
- --kms-dir=$RPM_BUILD_ROOT%{lib_kms} \
--system-include-dir=$RPM_BUILD_ROOT%{_includedir} \
--system-lib-dir=$RPM_BUILD_ROOT%{_libdir} \
--system-libexec-dir=$RPM_BUILD_ROOT/%{lib_hadoop}/libexec \
@@ -730,9 +728,12 @@
%files kms
%defattr(-,root,root)
+%config(noreplace) %{etc_hadoop}/conf.empty/kms-acls.xml
+%config(noreplace) %{etc_hadoop}/conf.empty/kms-env.sh
+%config(noreplace) %{etc_hadoop}/conf.empty/kms-log4j.properties
+%config(noreplace) %{etc_hadoop}/conf.empty/kms-site.xml
%config(noreplace) /etc/default/%{name}-kms
%{initd_dir}/%{name}-kms
-%{lib_kms}
%attr(0775,kms,kms) %{run_kms}
%attr(0775,kms,kms) %{log_kms}
%attr(0775,kms,kms) %{state_kms}
