bigtop-packages/src/common/oozie/do-component-build

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -x

. `dirname $0`/bigtop.bom
#
# Resolve source directory
#

# $0 may be a softlink
PRG="$0"
while [ -h "${PRG}" ]; do
  ls=`ls -ld "${PRG}"`
  link=`expr "$ls" : '.*-> \(.*\)$'`
  if expr "$link" : '/.*' > /dev/null; then
    PRG="$link"
  else
    PRG=`dirname "${PRG}"`/"$link"
  fi
done

RELEASE_DIR=`dirname ${PRG}`
RELEASE_DIR=`cd ${RELEASE_DIR}/..;pwd`

RELEASE_DIR=$PWD

# IMPORTANT: THE FULL_VERSION, if set, will be used only for the final TARBALL,
#            Everything inside of the TARBALL uses the version specified in the POM
if [ -z "${FULL_VERSION}" ]; then
  FULL_VERSION=${OOZIE_VERSION}
  echo "Warning: FULL_VERSION not specified. Default version [${FULL_VERSION}] will be used."
fi
VERSION_SUFFIX="-${FULL_VERSION}"

#
# Build Oozie
#

echo "Starting system build"

# Test if there is a + sign in the dir path
# We need to do this because in Linux Sun JVM inteprets '+' as ' ' in URLs
# testcases uses URLs to access testcases directories and OpenJPA enhancer uses them as well
WORKDIR=${RELEASE_DIR}
if [[ "${WORKDIR}" == *\+* ]]; then
  # There is a + sign. Create a temp work dir
  # and copy over the files to that place
  WORKDIR=`mktemp -d /tmp/oozie.XXXXXX`
  echo "Created temporary workspace directory: ${WORKDIR}"
  cp -R ${RELEASE_DIR}/* ${WORKDIR}
fi

# Delete and recreate build directory
BUILDDIR=${WORKDIR}/build
rm -rf ${BUILDDIR}
mkdir -p ${BUILDDIR}

EXTRA_GOALS=""

# Check if maven deploy should be done or not
if [ "${DO_MAVEN_DEPLOY}" = "deploy" ]; then
  EXTRA_GOALS="${EXTRA_GOALS} source:jar deploy"
fi

# Disable Codehaus repository since it's out of service
sed -i.bak "/http:\/\/repository.codehaus.org/a\\
            <releases>\\
                <enabled>false</enabled>\\
            </releases>" ${WORKDIR}/pom.xml

export MAVEN_OPTS="-Xmx2g"

# Invoke Oozie build script
MKDISTRO_ARGS=" -DjavaVersion=${JDK_VERSION} -DtargetJavaVersion=${JDK_VERSION} -DminJavaVersion=${JDK_VERSION} "
MKDISTRO_ARGS+="-DskipTests -Phadoop-3 -Pspark-2 -Puber "
MKDISTRO_ARGS+="-Dhadoop.version=${HADOOP_VERSION} "
MKDISTRO_ARGS+="-Dhadoop.auth.version=${HADOOP_VERSION} "
MKDISTRO_ARGS+="-Dhive.version=${HIVE_VERSION} "
MKDISTRO_ARGS+="-Dsqoop.version=${SQOOP_VERSION} "
MKDISTRO_ARGS+="-Dspark.version=${SPARK_VERSION} "
MKDISTRO_ARGS+="-Dspark.scala.binary.version=${SCALA_VERSION%.*} "
MKDISTRO_ARGS+="-Dspark.guava.version=27.0-jre "
MKDISTRO_ARGS+="-Dcommons.lang3.version=3.9 "
MKDISTRO_ARGS+="-Djackson.version=2.9.10 "
${WORKDIR}/bin/mkdistro.sh ${MKDISTRO_ARGS} ${EXTRA_GOALS} -Dmaven.repo.local=${HOME}/.m2/repository "$@"
MKDISTRO_RESULT=$?
if [ "${MKDISTRO_RESULT}" != "0" ]; then
  echo "ERROR: mkdistro.sh failed with error ${MKDISTRO_RESULT}"
  echo "Build files from failed build: ${WORKDIR}"
  exit ${MKDISTRO_RESULT}
fi

#
# Expand Oozie TAR and create Bigtop Ooozie TAR
#
# (TODO: This logic is kind of twisted, revisit.)
# (TODO: Consider doing the source copy before the build if we can assume a clean checkout.)
#

EXPTMPDIR=${BUILDDIR}/temp
mkdir -p ${EXPTMPDIR}
URLSAFE_VERSION_SUFFIX=`echo ${VERSION_SUFFIX}| sed -e "s/+/-/"`
ORIGINAL_NAME=oozie${VERSION_SUFFIX}
URLSAFE_NAME=oozie${URLSAFE_VERSION_SUFFIX}
EXPDIR=${EXPTMPDIR}/${URLSAFE_NAME}
(
cd ${EXPTMPDIR}
tar zxf ${WORKDIR}/distro/target/oozie-*.tar.gz
if [ "${FULL_VERSION}" != "${OOZIE_VERSION}" ]; then
  mv ${EXPTMPDIR}/oozie-* ${EXPDIR}
fi

cd ${EXPDIR}

SRCDIR=${EXPDIR}/src
mkdir -p ${SRCDIR}
for srcfile in ${WORKDIR}/*
do
  if [ "${srcfile}" != "${BUILDDIR}" ]; then
    cp -R ${srcfile} ${SRCDIR}
  fi
done
rm -rf ${SRCDIR}/target \
       ${SRCDIR}/*/target \
       ${SRCDIR}/core/build \
       ${SRCDIR}/core/pig*.log \
       ${SRCDIR}/core/pig*.properties \
       ${SRCDIR}/examples/mem \
       ${SRCDIR}/examples/oozietests \
       ${SRCDIR}/mkdistro-*.out \
       ${SRCDIR}/*.iml \
       ${SRCDIR}/*/*.iml \
       ${SRCDIR}/*.ipr \
       ${SRCDIR}/*.iws 

cp ${SRCDIR}/LICENSE.txt ${EXPDIR}
cp ${SRCDIR}/NOTICE.txt ${EXPDIR}
cd ${EXPDIR}
jar xf ${EXPDIR}/docs.zip
rm -rf ${EXPDIR}/docs.zip
rm -rf ${EXPDIR}/WEB-INF
rm -rf ${EXPDIR}/META-INF
cd ${EXPTMPDIR}
if [ "${URLSAFE_NAME}" != "${ORIGINAL_NAME}" ]; then
  mv ${URLSAFE_NAME} ${ORIGINAL_NAME}
fi
tar cfz oozie${VERSION_SUFFIX}.tar.gz ${ORIGINAL_NAME}
mv oozie${VERSION_SUFFIX}.tar.gz ${BUILDDIR}
cd ${BUILDDIR}
rm -rf ${EXPTMPDIR}
)

# If a temp work dir was created, move the files back to
# basedir and clean it up.

if [ "${WORKDIR}" != "${RELEASE_DIR}" ]; then
  echo "Moving built files from temporary location to work dir"
  mv ${WORKDIR}/build ${RELEASE_DIR}/build
  rm -rf ${WORKDIR}
fi

echo "Build available at: ${RELEASE_DIR}/build"

(cd ${RELEASE_DIR} ; tar --strip-components 1 -xzvf build/oozie${VERSION_SUFFIX}.tar.gz; rm -f bin/mkdistro.sh)

echo "SUCCESS: Build complete"
exit 0