build/mvn - spark - Git at Google

 #!/usr/bin/env bash

 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #

 # Determine the current working directory
 _DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # Preserve the calling directory
 _CALLING_DIR="$(pwd)"
 # Options used during compilation
 _COMPILE_JVM_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g"

 # Installs any application tarball given a URL, the expected tarball name,
 # and, optionally, a checkable binary path to determine if the binary has
 # already been installed
 ## Arg1 - URL
 ## Arg2 - Tarball Name
 ## Arg3 - Checkable Binary
 install_app() {
   local remote_tarball="$1/$2"
   local local_tarball="${_DIR}/$2"
   local binary="${_DIR}/$3"

   # setup `curl` and `wget` silent options if we're running on Jenkins
   local curl_opts="-L"
   local wget_opts=""
   if [ -n "$AMPLAB_JENKINS" ]; then
     curl_opts="-s ${curl_opts}"
     wget_opts="--quiet ${wget_opts}"
   else
     curl_opts="--progress-bar ${curl_opts}"
     wget_opts="--progress=bar:force ${wget_opts}"
   fi

   if [ -z "$3" -o ! -f "$binary" ]; then
     # check if we already have the tarball
     # check if we have curl installed
     # download application
     [ ! -f "${local_tarball}" ] && [ $(command -v curl) ] && \
       echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2 && \
       curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
     # if the file still doesn't exist, lets try `wget` and cross our fingers
     [ ! -f "${local_tarball}" ] && [ $(command -v wget) ] && \
       echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2 && \
       wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
     # if both were unsuccessful, exit
     [ ! -f "${local_tarball}" ] && \
       echo -n "ERROR: Cannot download $2 with cURL or wget; " && \
       echo "please install manually and try again." && \
       exit 2
     cd "${_DIR}" && tar -xzf "$2"
     rm -rf "$local_tarball"
   fi
 }

 # See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
 function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }

 # Determine the Maven version from the root pom.xml file and
 # install maven under the build/ folder if needed.
 install_mvn() {
   local MVN_VERSION=`grep "<maven.version>" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'`
   MVN_BIN="$(command -v mvn)"
   if [ "$MVN_BIN" ]; then
     local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')"
   fi
   if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then
     local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='}

     if [ $(command -v curl) ]; then
       local TEST_MIRROR_URL="${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries/apache-maven-${MVN_VERSION}-bin.tar.gz"
       if ! curl -L --output /dev/null --silent --head --fail "$TEST_MIRROR_URL" ; then
         # Fall back to archive.apache.org for older Maven
         echo "Falling back to archive.apache.org to download Maven"
         APACHE_MIRROR="https://archive.apache.org/dist"
       fi
     fi

     install_app \
       "${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries" \
       "apache-maven-${MVN_VERSION}-bin.tar.gz" \
       "apache-maven-${MVN_VERSION}/bin/mvn"

     MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn"
   fi
 }

 # Install zinc under the build/ folder
 install_zinc() {
   local ZINC_VERSION=0.3.15
   ZINC_BIN="$(command -v zinc)"
   if [ "$ZINC_BIN" ]; then
     local ZINC_DETECTED_VERSION="$(zinc -version | head -n1 | awk '{print $5}')"
   fi

   if [ $(version $ZINC_DETECTED_VERSION) -lt $(version $ZINC_VERSION) ]; then
     local zinc_path="zinc-${ZINC_VERSION}/bin/zinc"
     [ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
     local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}

     install_app \
       "${TYPESAFE_MIRROR}/zinc/${ZINC_VERSION}" \
       "zinc-${ZINC_VERSION}.tgz" \
       "${zinc_path}"
     ZINC_BIN="${_DIR}/${zinc_path}"
   fi
 }

 # Determine the Scala version from the root pom.xml file, set the Scala URL,
 # and, with that, download the specific version of Scala necessary under
 # the build/ folder
 install_scala() {
   # determine the Scala version used in Spark
   local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'`
   local scala_bin="${_DIR}/scala-${scala_version}/bin/scala"
   local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}

   install_app \
     "${TYPESAFE_MIRROR}/scala/${scala_version}" \
     "scala-${scala_version}.tgz" \
     "scala-${scala_version}/bin/scala"

   SCALA_COMPILER="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-compiler.jar"
   SCALA_LIBRARY="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-library.jar"
 }

 # Setup healthy defaults for the Zinc port if none were provided from
 # the environment
 ZINC_PORT=${ZINC_PORT:-"3030"}

 # Remove `--force` for backward compatibility.
 if [ "$1" == "--force" ]; then
   echo "WARNING: '--force' is deprecated and ignored."
   shift
 fi

 # Install the proper version of Scala, Zinc and Maven for the build
 install_zinc
 install_scala
 install_mvn

 # Reset the current working directory
 cd "${_CALLING_DIR}"

 # Now that zinc is ensured to be installed, check its status and, if its
 # not running or just installed, start it
 if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port ${ZINC_PORT}`" ]; then
   export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
   "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
   "${ZINC_BIN}" -start -port ${ZINC_PORT} \
     -server 127.0.0.1 -idle-timeout 3h \
     -scala-compiler "${SCALA_COMPILER}" \
     -scala-library "${SCALA_LIBRARY}" &>/dev/null
 fi

 # Set any `mvn` options if not already present
 export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}

 echo "Using \`mvn\` from path: $MVN_BIN" 1>&2

 # call the `mvn` command as usual
 # SPARK-25854
 "${MVN_BIN}" -DzincPort=${ZINC_PORT} "$@"
 MVN_RETCODE=$?

 # Try to shut down zinc explicitly if the server is still running.
 "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}

 exit $MVN_RETCODE
	#!/usr/bin/env bash

	#
	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright ownership.
	# The ASF licenses this file to You under the Apache License, Version 2.0
	# (the "License"); you may not use this file except in compliance with
	# the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#

	# Determine the current working directory
	_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
	# Preserve the calling directory
	_CALLING_DIR="$(pwd)"
	# Options used during compilation
	_COMPILE_JVM_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g"

	# Installs any application tarball given a URL, the expected tarball name,
	# and, optionally, a checkable binary path to determine if the binary has
	# already been installed
	## Arg1 - URL
	## Arg2 - Tarball Name
	## Arg3 - Checkable Binary
	install_app() {
	local remote_tarball="$1/$2"
	local local_tarball="${_DIR}/$2"
	local binary="${_DIR}/$3"

	# setup `curl` and `wget` silent options if we're running on Jenkins
	local curl_opts="-L"
	local wget_opts=""
	if [ -n "$AMPLAB_JENKINS" ]; then
	curl_opts="-s ${curl_opts}"
	wget_opts="--quiet ${wget_opts}"
	else
	curl_opts="--progress-bar ${curl_opts}"
	wget_opts="--progress=bar:force ${wget_opts}"
	fi

	if [ -z "$3" -o ! -f "$binary" ]; then
	# check if we already have the tarball
	# check if we have curl installed
	# download application
	[ ! -f "${local_tarball}" ] && [ $(command -v curl) ] && \
	echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2 && \
	curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
	# if the file still doesn't exist, lets try `wget` and cross our fingers
	[ ! -f "${local_tarball}" ] && [ $(command -v wget) ] && \
	echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2 && \
	wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
	# if both were unsuccessful, exit
	[ ! -f "${local_tarball}" ] && \
	echo -n "ERROR: Cannot download $2 with cURL or wget; " && \
	echo "please install manually and try again." && \
	exit 2
	cd "${_DIR}" && tar -xzf "$2"
	rm -rf "$local_tarball"
	fi
	}

	# See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
	function version { echo "$@" \| awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }

	# Determine the Maven version from the root pom.xml file and
	# install maven under the build/ folder if needed.
	install_mvn() {
	local MVN_VERSION=`grep "<maven.version>" "${_DIR}/../pom.xml" \| head -n1 \| awk -F '[<>]' '{print $3}'`
	MVN_BIN="$(command -v mvn)"
	if [ "$MVN_BIN" ]; then
	local MVN_DETECTED_VERSION="$(mvn --version \| head -n1 \| awk '{print $3}')"
	fi
	if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then
	local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='}

	if [ $(command -v curl) ]; then
	local TEST_MIRROR_URL="${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries/apache-maven-${MVN_VERSION}-bin.tar.gz"
	if ! curl -L --output /dev/null --silent --head --fail "$TEST_MIRROR_URL" ; then
	# Fall back to archive.apache.org for older Maven
	echo "Falling back to archive.apache.org to download Maven"
	APACHE_MIRROR="https://archive.apache.org/dist"
	fi
	fi

	install_app \
	"${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries" \
	"apache-maven-${MVN_VERSION}-bin.tar.gz" \
	"apache-maven-${MVN_VERSION}/bin/mvn"

	MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn"
	fi
	}

	# Install zinc under the build/ folder
	install_zinc() {
	local ZINC_VERSION=0.3.15
	ZINC_BIN="$(command -v zinc)"
	if [ "$ZINC_BIN" ]; then
	local ZINC_DETECTED_VERSION="$(zinc -version \| head -n1 \| awk '{print $5}')"
	fi

	if [ $(version $ZINC_DETECTED_VERSION) -lt $(version $ZINC_VERSION) ]; then
	local zinc_path="zinc-${ZINC_VERSION}/bin/zinc"
	[ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
	local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}

	install_app \
	"${TYPESAFE_MIRROR}/zinc/${ZINC_VERSION}" \
	"zinc-${ZINC_VERSION}.tgz" \
	"${zinc_path}"
	ZINC_BIN="${_DIR}/${zinc_path}"
	fi
	}

	# Determine the Scala version from the root pom.xml file, set the Scala URL,
	# and, with that, download the specific version of Scala necessary under
	# the build/ folder
	install_scala() {
	# determine the Scala version used in Spark
	local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" \| head -n1 \| awk -F '[<>]' '{print $3}'`
	local scala_bin="${_DIR}/scala-${scala_version}/bin/scala"
	local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}

	install_app \
	"${TYPESAFE_MIRROR}/scala/${scala_version}" \
	"scala-${scala_version}.tgz" \
	"scala-${scala_version}/bin/scala"

	SCALA_COMPILER="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-compiler.jar"
	SCALA_LIBRARY="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-library.jar"
	}

	# Setup healthy defaults for the Zinc port if none were provided from
	# the environment
	ZINC_PORT=${ZINC_PORT:-"3030"}

	# Remove `--force` for backward compatibility.
	if [ "$1" == "--force" ]; then
	echo "WARNING: '--force' is deprecated and ignored."
	shift
	fi

	# Install the proper version of Scala, Zinc and Maven for the build
	install_zinc
	install_scala
	install_mvn

	# Reset the current working directory
	cd "${_CALLING_DIR}"

	# Now that zinc is ensured to be installed, check its status and, if its
	# not running or just installed, start it
	if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port ${ZINC_PORT}`" ]; then
	export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
	"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
	"${ZINC_BIN}" -start -port ${ZINC_PORT} \
	-server 127.0.0.1 -idle-timeout 3h \
	-scala-compiler "${SCALA_COMPILER}" \
	-scala-library "${SCALA_LIBRARY}" &>/dev/null
	fi

	# Set any `mvn` options if not already present
	export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}

	echo "Using \`mvn\` from path: $MVN_BIN" 1>&2

	# call the `mvn` command as usual
	# SPARK-25854
	"${MVN_BIN}" -DzincPort=${ZINC_PORT} "$@"
	MVN_RETCODE=$?

	# Try to shut down zinc explicitly if the server is still running.
	"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}

	exit $MVN_RETCODE