| #!/bin/bash |
| # |
| # Builds a mod_pagespeed distribution. |
| # |
| # Usage: |
| # 1. [optional] Log into buildbot |
| # 2. ./build_release_platform.sh [-clean] $RELEASE $CHANNEL [patch_file] [tag] |
| # |
| # Where $RELEASE is either the 4 segment version number, e.g 0.10.21.2, or the |
| # word "trunk" and $CHANNEL is either "beta" or "stable". |
| # |
| # The optional "patch_file" will be applied after runing 'gclient sync'. |
| # Also note that if you re-run the same build command multiple times, |
| # pre-applied patches will remain. |
| # |
| # If tag is specified it will be used to pick what gets checked out, instead of |
| # $RELEASE; this is useful if applying an unreleased patch that changes the |
| # version number for a security release. |
| # |
| # Updating: |
| # If you make any changes to this script, please scp it up to the builbotd: |
| # scp build_release_platform.sh [IP-OF-CENTOS-BUILDBOT]: |
| # scp build_release_platform.sh [IP-OF-UBUNTU-BUILDBOT]: |
| # |
| # The builbots have hardlinks between /home/builbot/build_release_platform.sh |
| # and /var/chroot/[chroot-name]/home/buildbot/build_release_platform.sh, which |
| # means that when you update the copy in ~/build_release_platform.sh you're |
| # automatically updating the one in the chroot as well. |
| |
| |
| set -e # exit script if any command returns an error |
| set -u # exit the script if any variable is uninitialized |
| export VIRTUALBOX_TEST="VIRTUALBOX_TEST" # to skip some tests that fail in VM |
| |
| # Cleanup /var/html if desired before running. |
| if [ "$1" = "-clean" ]; then |
| clean=1 |
| shift |
| else |
| clean=0 |
| fi |
| |
| RELEASE=$1 |
| CHANNEL=$2 |
| TAG=$RELEASE |
| |
| do_patch="0" |
| |
| DOC_ROOT=/var/www |
| [ -d /var/www/html ] && DOC_ROOT=/var/www/html |
| |
| if [ -d $DOC_ROOT/do_not_modify -a \ |
| ! -L $DOC_ROOT/do_not_modify ] || \ |
| [ -d $DOC_ROOT/mod_pagespeed_example -a \ |
| ! -L $DOC_ROOT/mod_pagespeed_example ] || \ |
| [ -d $DOC_ROOT/mod_pagespeed_test -a \ |
| ! -L $DOC_ROOT/mod_pagespeed_test ]; then |
| if [ $clean -eq 0 ]; then |
| echo Stale directories in $DOC_ROOT exist. Clean them |
| echo yourself or specify -clean as first arg to let the script do that |
| echo for you. |
| exit 1 |
| fi |
| set -x |
| sudo rm -rf $DOC_ROOT/do_not_modify |
| sudo rm -rf $DOC_ROOT/mod_pagespeed_example |
| sudo rm -rf $DOC_ROOT/mod_pagespeed_test |
| else |
| set -x |
| fi |
| |
| # Apply optional patch-file to apply before building. To create a patch from a |
| # commit, do: |
| # git show [commit id] > patchfile |
| if [ $# -ge 3 ]; then |
| patch_file=$(readlink -f $3) |
| do_patch="1" |
| fi |
| |
| if [ $# -eq 4 ]; then |
| TAG=$4 |
| fi |
| |
| if [ -d ~/bin/depot_tools ]; then |
| cd ~/bin/depot_tools |
| git pull |
| else |
| mkdir -p ~/bin |
| cd ~/bin |
| git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git |
| fi |
| |
| PATH=~/bin/depot_tools:$PATH |
| |
| # Are we on CentOS or Ubuntu? |
| # TODO(jmarantz): make this work on RedHat. |
| # |
| # Note that the 'force' directives on the install commands are only |
| # used for this RPM/DEB build/install/test script, and are not |
| # exported as a recommended flow for users. |
| # |
| # If the force directives are not included, then the installation will |
| # fail if run a second time. |
| if grep -q CentOS /etc/issue; then |
| # We redirect stderr to stdout because on Centos5 "httpd -M" writes to stderr |
| # for some reason. |
| /usr/sbin/httpd -M 2>&1 | grep -q php5_module || \ |
| sudo yum install php php-mbstring |
| |
| EXT=rpm |
| INSTALL="rpm --install" |
| RESTART="./centos.sh apache_debug_restart" |
| TEST="./centos.sh enable_ports_and_file_access apache_vm_system_tests" |
| |
| # To build on Centos 5/6 we need gcc 4.8 from scientific linux. We can't |
| # export CC and CXX because some steps still use a literal "g++". But #$%^ |
| # devtoolset includes its own sudo, and we don't want that because it doesn't |
| # support -E, so rename it if it exists before updating PATH. |
| DEVTOOLSET_BIN=/opt/rh/devtoolset-2/root/usr/bin/ |
| if [ -e "$DEVTOOLSET_BIN/sudo" ]; then |
| sudo mv "$DEVTOOLSET_BIN/sudo" "$DEVTOOLSET_BIN/sudo.ignored" |
| fi |
| export PATH="$DEVTOOLSET_BIN:$PATH" |
| |
| echo We appear to be running on CentOS. Building rpm... |
| else |
| echo Making sure PHP is installed ... |
| apache2ctl -M | grep -q php5_module || \ |
| sudo apt-get install libapache2-mod-php5 |
| |
| EXT=deb |
| INSTALL="dpkg --install" |
| RESTART="./ubuntu.sh apache_debug_restart" |
| TEST="./ubuntu.sh apache_vm_system_tests" |
| |
| if [ -d /usr/lib/gcc-mozilla/bin ]; then |
| export PATH=/usr/lib/gcc-mozilla/bin:$PATH |
| fi |
| |
| echo We appear to NOT be running on CentOS. Building deb... |
| fi |
| |
| if [ $(uname -m) = x86_64 ]; then |
| BIT_SIZE_NAME=x64 |
| else |
| BIT_SIZE_NAME=ia32 |
| fi |
| |
| if [[ "$RELEASE" == 1.9.32.* || "$RELEASE" == 1.10.33.* ]]; then |
| # Without https://github.com/pagespeed/mod_pagespeed/commit/0e79a08 the font |
| # inlining tests will fail, and we never backported that fix to 1.9 or 1.10. |
| export DISABLE_FONT_API_TESTS=1 |
| fi |
| |
| build_dir="$HOME/build/$RELEASE/$BIT_SIZE_NAME" |
| release_dir="$HOME/release/$RELEASE/$BIT_SIZE_NAME" |
| log_dir="$build_dir/log" |
| rm -rf $log_dir |
| mkdir -p $log_dir |
| rm -rf "$release_dir" |
| mkdir -p "$release_dir" |
| |
| # Usage: |
| # check log_filename.log command args... |
| # |
| # The log file will placed in $log_dir, and will be 'tail'ed if the |
| # command fails. |
| function check() { |
| # We are explicitly checking error status here and tailing the log |
| # file so turn off auto-exit-on-error tempoararily. |
| set +e |
| log_filename="$log_dir/$1" |
| shift |
| echo "[$(date '+%k:%M:%S')] $@ >> $log_filename" |
| echo $@ >> "$log_filename" |
| $@ >> "$log_filename" 2>&1 |
| rc=$? |
| if [ $rc != 0 ]; then |
| echo '***' status is $rc |
| tail "$log_filename" |
| echo Failed at $(date) |
| exit 1 |
| fi |
| set -e |
| } |
| |
| # We do the building in build/ which generates all kinds of crap which |
| # we don't ship in our binaries, such as .o and .a files. |
| # |
| # We put what we want to ship into release/ so that it can be scp'd onto |
| # the signing server with only one password. |
| mkdir -p $build_dir |
| rm -rf "$build_dir/src" |
| cd $build_dir && git clone https://github.com/pagespeed/mod_pagespeed.git src |
| cd $build_dir/src |
| if [ "$TAG" != "trunk" ]; then # Just treat "trunk" as master. |
| git checkout "$TAG" |
| fi |
| |
| if [ "$EXT" = "rpm" ] ; then |
| # MANYLINUX1 is required for CentOS 5 (but probably not newer CentOS): |
| # https://github.com/grpc/grpc/issues/7147 |
| export CFLAGS='-DGPR_MANYLINUX1 -std=gnu99' |
| |
| # On the centos buildbot we need to patch the Makefile to make |
| # apache_debug_restart to a killall -9 httpd. |
| cd $build_dir/src |
| killall_patch=$(cat <<EOF |
| diff --git a/install/Makefile b/install/Makefile |
| index 376def9..29fa72e 100644 |
| --- a/install/Makefile |
| +++ b/install/Makefile |
| @@ -573,6 +573,7 @@ setup_test_machine : |
| >sudo \$(APACHE_CONTROL_PROGRAM) restart |
| |
| apache_debug_restart : |
| +>killall -9 httpd || echo "not killed" |
| >\$(APACHE_CONTROL_PROGRAM) restart |
| |
| apache_debug_stop : stop |
| EOF |
| ) |
| echo "$killall_patch" | tr '>' '\t' | git apply |
| fi |
| cd $build_dir |
| |
| check gclient.log \ |
| gclient config https://github.com/pagespeed/mod_pagespeed.git --unmanaged --name=src |
| check gclient.log gclient sync --force |
| |
| cd src |
| |
| # Neither buildbot is using a compiler recent enough to provide stdalign.h, |
| # which boringssl needs. Even on Centos 5's gcc 4.1 we do have a way to set |
| # alignment, though, so following |
| # https://sourceware.org/bugzilla/show_bug.cgi?id=19390 define alignas ourself |
| # and put it where boringssl can find it. |
| echo '#define alignas(x) __attribute__ ((aligned (x)))' > \ |
| third_party/boringssl/src/crypto/stdalign.h |
| |
| if [ $do_patch -eq "1" ]; then |
| echo Applying patch-file $patch_file |
| git apply $patch_file |
| |
| echo "Re-running gclient in case the patch touched DEPS" |
| check gclient.log gclient sync |
| fi |
| |
| # This is needed on the vms, but not on our workstations for some reason. |
| find $build_dir/src -name "*.sh" | xargs chmod +x |
| cd $build_dir |
| echo src/build/gyp_chromium -Dchannel=$CHANNEL |
| export AR_host=$build_dir/src/build/wrappers/ar.sh |
| check gyp_chromium.log python src/build/gyp_chromium -Dchannel=$CHANNEL |
| |
| cd src |
| # It would be better to have AR.target overridden at gyp time, but |
| # that functionality seems broken. |
| MODPAGESPEED_ENABLE_UPDATES=1 check build.log \ |
| make BUILDTYPE=Release AR.host=${AR_host} AR.target=${AR_host} V=1 \ |
| linux_package_$EXT mod_pagespeed_test pagespeed_automatic_test |
| |
| ls -l $PWD/out/Release/mod-pagespeed-${CHANNEL}* |
| mkdir -p "$release_dir" |
| mv $PWD/out/Release/mod-pagespeed-${CHANNEL}* "$release_dir" |
| |
| if [ "$EXT" = "rpm" ] ; then |
| export SSL_CERT_DIR=/etc/pki/tls/certs |
| export SSL_CERT_FILE=/etc/pki/tls/cert.pem |
| fi |
| |
| check unit_test.log out/Release/mod_pagespeed_test |
| check unit_test.log out/Release/pagespeed_automatic_test |
| |
| # Buildbots should have NOPASSWD set, so won't need to be prompted for sudo |
| # password. |
| |
| echo Purging old releases ... |
| if [ "$EXT" = "rpm" ] ; then |
| # rpm --erase only succeeds if all packages listed are installed, so we need |
| # to find which one is installed and only erase that. |
| rpm --query mod-pagespeed-stable mod-pagespeed-beta | \ |
| grep -v "is not installed" | \ |
| xargs --no-run-if-empty sudo rpm --erase |
| else |
| # dpkg --purge succeeds even if one or both of the packages is not installed. |
| sudo dpkg --purge mod-pagespeed-beta mod-pagespeed-stable |
| fi |
| |
| echo Installing release ... |
| check install.log sudo $INSTALL "$release_dir"/*.$EXT |
| |
| echo Test restart to make sure config file is valid ... |
| cd $build_dir/src/install |
| check install.log sudo -E $RESTART |
| |
| echo Testing release ... |
| check system_test.log sudo -E $TEST |
| |
| echo Copy the unstripped .so files to a safe place for easier debugging later. |
| NBITS=$(getconf LONG_BIT) |
| cp $build_dir/src/out/Release/libmod_pagespeed.so \ |
| "$release_dir"/unstripped_libmodpagespeed_${NBITS}_${EXT}.so |
| cp $build_dir/src/out/Release/libmod_pagespeed_ap24.so \ |
| "$release_dir"/unstripped_libmodpagespeed_ap24_${NBITS}_${EXT}.so |
| |
| # Because we now build on the build-bots which are not on the internal network, |
| # you need to manually pull the builds from ~/release on each server. |
| echo Build succeeded at $(date) |
| |
| # This doesn't necessarily need to be limited to CentOS, but we only need to |
| # build PSOL libraries on one system, and CentOS has the oldest GCC, so we |
| # build it there. |
| if [ "$EXT" = "rpm" -a "$CHANNEL" = "beta" ]; then |
| echo Building PSOL binaries ... |
| |
| for buildtype in Release Debug; do |
| cd $build_dir/src |
| check psol_build.log make BUILDTYPE=$buildtype \ |
| AR.host=${AR_host} AR.target=${AR_host} V=1 \ |
| mod_pagespeed_test pagespeed_automatic_test |
| |
| |
| if [[ "$RELEASE" == 1.9.32.* ]]; then |
| # On 1.9 (and earlier, but we don't build them anymore) automatic/ was in |
| # a different place. |
| automatic_dir=net/instaweb/automatic/ |
| else |
| automatic_dir=pagespeed/automatic/ |
| fi |
| |
| cd $automatic_dir |
| |
| # TODO(sligocki): Fix and use |
| # check psol_automatic_build.log |
| set +e |
| make MOD_PAGESPEED_ROOT=$build_dir/src BUILDTYPE=$buildtype \ |
| AR.host=${AR_host} AR.target=${AR_host} V=1 \ |
| CXXFLAGS="-DSERF_HTTPS_FETCHING=1" \ |
| all \ |
| >> psol_automatic_build.log 2>&1 |
| set -e |
| cd $build_dir/src |
| |
| BINDIR=$HOME/psol_release/$RELEASE/psol/lib/$buildtype/linux/$BIT_SIZE_NAME |
| mkdir -p $BINDIR/ |
| mv $automatic_dir/pagespeed_automatic.a $BINDIR/ |
| if [ "$buildtype" = "Release" ]; then |
| mv out/$buildtype/js_minify $BINDIR/pagespeed_js_minify |
| fi |
| |
| # Sync release binaries incrementally as they're built so we don't |
| # lose progress. |
| echo PSOL $buildtype build succeeded at $(date) |
| |
| # VMs are running low on disk space, so clean up between builds. |
| rm -rf out/$buildtype |
| done |
| fi |
| |
| |
| exit 0 |