| #!/bin/bash |
| ################################################################################ |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| ################################################################################ |
| # |
| # Mini cluster test binaries |
| # ------------------------- |
| # |
| # This script generates optimized, dynamically-linked, stripped, fully |
| # relocatable Kudu server binaries for easy use by integration tests using a |
| # mini cluster. The resulting binaries should never be deployed to run an |
| # actual Kudu service, whether in production or development, because all |
| # security dependencies are copied from the build system and will not be |
| # updated if the operating system on the runtime host is patched. |
| # |
| # Properties file |
| # --------------- |
| # |
| # The JAR that is generated contains a properties file in META-INF that allows |
| # it to be located by Java code at runtime if the JAR is on the classpath. |
| # The location of that file in the archive is: |
| # |
| # META-INF/apache-kudu-test-binary.properties |
| # |
| # The following properties are supported: |
| # |
| # * format.version: The format of the artifact, in case we need to extend it |
| # later and be able to tell the difference between the archive formats. |
| # * artifact.version: The version of the release (or snapshot, i.e. |
| # 1.9.0-SNAPSHOT) in order for the format to support support multiple |
| # versions on the classpath or to allow a client to be able to request a |
| # specific release version. |
| # * artifact.prefix: The directory name at the root of the JAR under which the |
| # binary files can be found (similar to ./configure --prefix in autoconf). |
| # * artifact.os: The operating system name using the same convention as |
| # https://github.com/trustin/os-maven-plugin for ${os.detected.name}. |
| # Practically speaking for Kudu at the time of writing, that is either |
| # 'linux' or 'osx'. |
| # * artifact.arch: The target system architecture using the same convention as |
| # os-maven-plugin for ${os.detected.arch}. Practically speaking for Kudu, at |
| # the time of writing this will always be set to x86_64. |
| # |
| # Example: |
| # |
| # $ cat META-INF/apache-kudu-test-binary.properties |
| # format.version=1 |
| # artifact.version=1.8.0 |
| # artifact.prefix=kudu-binary-1.8.0-linux-x86_64 |
| # artifact.os=linux |
| # artifact.arch=x86_64 |
| # |
| ################################################################################ |
| set -e |
| |
| SOURCE_ROOT=$(cd $(dirname $0)/../..; pwd) |
| BUILD_ROOT=$SOURCE_ROOT/build/mini-cluster |
| MINI_CLUSTER_SRCDIR=$SOURCE_ROOT/build-support/mini-cluster |
| TARGETS="kudu kudu-tserver kudu-master" |
| |
| # Remove leftovers from previous releases/builds: rename the existing directory. |
| # To skip this step, set the MINI_CLUSTER_NO_FRESH_BUILD environment variable. |
| if [ -d "$BUILD_ROOT" ]; then |
| if [ -n "$MINI_CLUSTER_NO_FRESH_BUILD" ]; then |
| echo "WARNING: using existing build directory" |
| else |
| suffix=$(date "+%Y%m%d.%H%M%S") |
| echo "Moving existing $BUILD_ROOT into $BUILD_ROOT.$suffix" |
| mv $BUILD_ROOT $BUILD_ROOT.$suffix |
| fi |
| fi |
| |
| THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty} |
| cd $SOURCE_ROOT |
| if [ -n "$NO_REBUILD_THIRDPARTY" ]; then |
| echo Skipping thirdparty because NO_REBUILD_THIRDPARTY is not empty |
| else |
| echo Building thirdparty... >&2 |
| $SOURCE_ROOT/build-support/enable_devtoolset.sh \ |
| $THIRDPARTY_DIR/build-if-necessary.sh |
| fi |
| |
| mkdir -p $BUILD_ROOT |
| cd $BUILD_ROOT |
| |
| MACOS="" |
| # TODO(mpercy): What's a better way to detect macOS? |
| if [ $(uname) == "Darwin" ]; then |
| MACOS=1 |
| fi |
| |
| EXTRA_CMAKE_FLAGS="" |
| if [ -n "$MACOS" ]; then |
| # TODO(mpercy): Consider using pkg-config to support building with MacPorts. |
| EXTRA_CMAKE_FLAGS="$EXTRA_CMAKE_FLAGS -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl" |
| # TODO(mpercy): Is it even possible to build Kudu with gcc/g++ on macOS? |
| export CC=clang |
| export CXX=clang++ |
| fi |
| |
| rm -rf CMakeCache.txt CMakeFiles |
| |
| # We want a fast build with a small total output size, so we need to build in |
| # release mode with dynamic linking so that all of the target executables can |
| # use the same shared objects for their dependencies. Since we may not |
| # distribute chronyc/chronyd in kudu-binaries JAR due to licensing restrictions, |
| # the test harness is built to not rely on chronyd as NTP server for tests run |
| # with the mini-cluster. |
| echo Configuring Kudu... >&2 |
| $SOURCE_ROOT/build-support/enable_devtoolset.sh \ |
| $THIRDPARTY_DIR/installed/common/bin/cmake ../.. \ |
| -DNO_TESTS=1 -DNO_CHRONY=1 \ |
| -DCMAKE_BUILD_TYPE=RELEASE -DKUDU_LINK=dynamic $EXTRA_CMAKE_FLAGS |
| |
| echo Building Kudu... >&2 |
| NUM_PROCS=$(getconf _NPROCESSORS_ONLN) |
| make -j$NUM_PROCS $TARGETS |
| |
| # Relocate the binaries. |
| $MINI_CLUSTER_SRCDIR/relocate_binaries_for_mini_cluster.py $BUILD_ROOT $TARGETS |
| |
| ARTIFACT_NAME="$(ls -dF kudu-binary-* | egrep '.*/$' | sed 's#/##')" |
| if [ $(echo $ARTIFACT_NAME | wc -w) -ne 1 ]; then |
| echo "ERROR: this script can handle only single kudu-binary artifact, but" |
| echo " multiple kudu-binary-* directories found under $BUILD_ROOT:" |
| echo " $ARTIFACT_NAME" |
| echo " Remove extra kudu-binary-* directories" |
| exit 1 |
| fi |
| |
| # Strip everything to minimize the size of the tarball we generate. |
| echo Stripping symbols... |
| for file in $ARTIFACT_NAME/bin/*; do |
| strip $file |
| done |
| |
| # Stripping libraries on macOS is tricky, so skip it for now. |
| # TODO(mpercy): Deal with detecting signed libraries and indirect symbol table |
| # entries on macOS. |
| if [ -z "$MACOS" ]; then |
| find $ARTIFACT_NAME/lib -type f -exec strip {} \; |
| fi |
| |
| # Generate the properties file that allows us to find this archive on the |
| # classpath at runtime. See above for the specification. |
| PROP_DIR="META-INF" |
| PROP_FILE="$PROP_DIR/apache-kudu-test-binary.properties" |
| FORMAT_VERSION=1 |
| ARTIFACT_VERSION=$(cat $SOURCE_ROOT/version.txt) |
| ARTIFACT_OS="linux" |
| if [ $MACOS ]; then |
| ARTIFACT_OS="osx" |
| fi |
| ARTIFACT_ARCH=$(uname -m) |
| rm -rf $PROP_DIR |
| mkdir -p $PROP_DIR |
| cat <<EOF > $PROP_FILE |
| format.version=$FORMAT_VERSION |
| artifact.os=$ARTIFACT_OS |
| artifact.arch=$ARTIFACT_ARCH |
| artifact.prefix=$ARTIFACT_NAME |
| artifact.version=$ARTIFACT_VERSION |
| EOF |
| |
| # Include the basic legal files. |
| # Create a platform-specific NOTICE file. |
| JAR_NOTICE=$MINI_CLUSTER_SRCDIR/NOTICE-BINARY-JAR-LINUX.txt |
| if [ $MACOS ]; then |
| JAR_NOTICE=$MINI_CLUSTER_SRCDIR/NOTICE-BINARY-JAR-OSX.txt |
| fi |
| cat $SOURCE_ROOT/NOTICE.txt \ |
| $JAR_NOTICE \ |
| > $ARTIFACT_NAME/NOTICE.txt |
| |
| # Create a platform-specific LICENSE file. |
| JAR_LICENSE=$MINI_CLUSTER_SRCDIR/LICENSE-BINARY-JAR-LINUX.txt |
| if [ $MACOS ]; then |
| JAR_LICENSE=$MINI_CLUSTER_SRCDIR/LICENSE-BINARY-JAR-OSX.txt |
| fi |
| cat $SOURCE_ROOT/LICENSE.txt \ |
| $THIRDPARTY_DIR/LICENSE.txt \ |
| $JAR_LICENSE \ |
| > $ARTIFACT_NAME/LICENSE.txt |
| |
| # Include the web UI template files. |
| cp -Rp $SOURCE_ROOT/www $ARTIFACT_NAME/ |
| |
| # Include a basic warning. |
| cat <<EOF > $ARTIFACT_NAME/README.txt |
| This archive contains Kudu binaries for use in a "mini cluster" environment for |
| TESTING ONLY. |
| |
| The binaries in this archive should never be deployed to run an actual Kudu |
| service, whether in production or development, because many security-related |
| dependencies are copied from the build system and will not be patched when the |
| operating system on the runtime host is patched. |
| EOF |
| |
| echo "Running license check on artifact..." |
| $SOURCE_ROOT/build-support/mini-cluster/check-license.pl $ARTIFACT_NAME |
| |
| echo Creating archive... |
| ARTIFACT_FILE=$ARTIFACT_NAME.jar |
| jar cf $ARTIFACT_FILE $PROP_DIR/ $ARTIFACT_NAME/ |
| echo "Binary test artifact: $(pwd)/$ARTIFACT_FILE" |