build-support/python/make-mesos-native-egg

#!/bin/bash
# Dependencies: vagrant >= 1.5.0 (for vagrantcloud support), VirtualBox
# System Requirements: 4 GiB of RAM free.
set -u -e

MESOS_BASEURL=https://archive.apache.org/dist/mesos
BOX_RAM_MB=$((8 * 1024))  # at least 4 GiB, needed to link libmesos
BOX_CPUS=4

usage() {
  cat <<EOF
Usage: make-mesos-native-egg.sh TARGET_DISTRIBUTION MESOS_VERSION AURORA_3RDPARTY_ROOT

Build a mesos Python egg in a pristine Vagrant environment.

TARGET_DISTRIBUTION is one of (centos6, centos7, jessie64, trusty64, xenial64).
MESOS_VERSION is a released version of mesos available on archive.apache.org.
AURORA_3RDPARTY_ROOT is the path to a sparse checkout of the Aurora 3rdparty repo.
EOF
  exit ${1:-1}
}

setup_tempdir() {
  TMPDIR=$(mktemp -d -t make-mesos-eggs.XXXXX)
  cleanup() {
    cd "$TMPDIR"
    if [[ -f Vagrantfile ]]; then
      vagrant destroy -f
    fi
    cd /
    rm -frv "$TMPDIR"
  }
  # TODO(ksweeney): Use atexit.sh from mesos and its superior handling of this cleanup idiom.
  trap cleanup EXIT
}

# Create a new vagrant box
init_vagrant_box() {
  local box_name=$1
  cat > Vagrantfile <<EOF
Vagrant.require_version ">= 1.5.0"
Vagrant.configure("2") do |config|
  config.vm.box = "$box_name"
  config.vm.provider "virtualbox" do |vb|
    vb.memory = "$BOX_RAM_MB"
    vb.cpus = "$BOX_CPUS"
  end
end
EOF
}

fetch_and_build_mesos() {
  local mesos_version=$1

  cat <<EOF
      wget --progress=dot "$MESOS_BASEURL/$mesos_version/mesos-${mesos_version}.tar.gz"
      tar zxvf mesos-${mesos_version}.tar.gz
      cd mesos-$mesos_version
      ./configure --disable-java --enable-optimize
      export MAKEFLAGS="-j$BOX_CPUS"
      make
      find . -name '*.egg' -exec cp -v {} /vagrant \\;
EOF
}

DEBIAN_JESSIE64_DEPENDENCIES=(
  g++
  make
  libapr1-dev
  libcurl4-nss-dev
  libsasl2-dev
  libsvn-dev
  python-dev
  python-virtualenv
  zlib1g-dev
)
build_debian_jessie64() {
  local mesos_version=$1 output_basedir=$2
  local python_outdir=$output_basedir/debian/jessie64/python
  pushd "$TMPDIR"
    # Using bento debian box due to debian/jessie64 not having
    # guest additions installed. No guest additions = no /vagrant.
    init_vagrant_box bento/debian-8.6
    vagrant up
    vagrant ssh <<EOF
      set -e -u

      sudo apt-get update
      sudo apt-get -y install ${DEBIAN_JESSIE64_DEPENDENCIES[*]}
      $(fetch_and_build_mesos "$mesos_version")
EOF
    mkdir -pv "$python_outdir"
    cp -v mesos.executor*.egg "$python_outdir"
  popd
}

UBUNTU_TRUSTY64_DEPENDENCIES=(
  g++-4.8
  libapr1-dev
  libcurl4-nss-dev
  libsasl2-dev
  libsvn-dev
  python-dev
  python-virtualenv
  zlib1g-dev
)
build_ubuntu_trusty64() {
  local mesos_version=$1 output_basedir=$2
  local python_outdir=$output_basedir/ubuntu/trusty64/python
  pushd "$TMPDIR"
    init_vagrant_box ubuntu/trusty64
    vagrant up
    vagrant ssh <<EOF
      set -e -u

      sudo apt-get update
      sudo apt-get -y install ${UBUNTU_TRUSTY64_DEPENDENCIES[*]}
      $(fetch_and_build_mesos "$mesos_version")
EOF
    mkdir -pv "$python_outdir"
    cp -v mesos.executor*.egg "$python_outdir"
  popd
}

UBUNTU_XENIAL64_DEPENDENCIES=(
  g++
  libapr1-dev
  libcurl4-nss-dev
  libsasl2-dev
  libsvn-dev
  python-dev
  python-virtualenv
  zlib1g-dev
)
build_ubuntu_xenial64() {
  local mesos_version=$1 output_basedir=$2
  local python_outdir=$output_basedir/ubuntu/xenial64/python
  pushd "$TMPDIR"
    # Using bento box due to ubuntu/xenial64
    # not being configured properly for Vagrant
    init_vagrant_box bento/ubuntu-16.04
    vagrant up
    vagrant ssh <<EOF
      set -e -u

      sudo apt-get update
      sudo apt-get -y install ${UBUNTU_XENIAL64_DEPENDENCIES[*]}
      $(fetch_and_build_mesos "$mesos_version")
EOF
    mkdir -pv "$python_outdir"
    cp -v mesos.executor*.egg "$python_outdir"
  popd
}

CENTOS6_X86_64_DEPENDENCIES=(
  apr-devel
  cyrus-sasl-devel
  cyrus-sasl-md5
  devtoolset-2-binutils
  devtoolset-2-gcc
  devtoolset-2-gcc-c++
  libcurl-devel
  patch
  python27-python-devel
  subversion-devel
  zlib-devel
)
build_centos6() {
  local mesos_version=$1 output_basedir=$2
  local python_outdir=$output_basedir/centos/6/python
  pushd "$TMPDIR"
    init_vagrant_box bento/centos-6.7
    vagrant up
    vagrant ssh <<EOF
    set -e -u

    # We need Software Collections for Python 2.7 (we dropped support for 2.6).
    sudo yum -y install centos-release-SCL

    # Mesos requires gcc 4.8, which is available in devtools-2 on centos 6.
    # TODO(maxim): Consider using signed packages when/if available.
    sudo wget http://people.centos.org/tru/devtools-2/devtools-2.repo -O /etc/yum.repos.d/devtools-2.repo

    sudo yum -y install ${CENTOS6_X86_64_DEPENDENCIES[*]}
    scl enable devtoolset-2 bash
    scl enable python27 - <<EOS
        $(fetch_and_build_mesos "$mesos_version")
EOS
EOF
    mkdir -pv "$python_outdir"
    cp -v mesos.executor*.egg "$python_outdir"
  popd
}

CENTOS7_X86_64_DEPENDENCIES=(
  apr-devel
  cyrus-sasl-devel
  cyrus-sasl-md5
  gcc-c++
  libcurl-devel
  patch
  python-devel
  subversion-devel
  zlib-devel
)
build_centos7() {
  local mesos_version=$1 output_basedir=$2
  local python_outdir=$output_basedir/centos/7/python
  mkdir -pv "$python_outdir"
  pushd "$TMPDIR"
    init_vagrant_box bento/centos-7.1
    vagrant up
    vagrant ssh <<EOF
    set -e -u

    sudo yum -y install ${CENTOS7_X86_64_DEPENDENCIES[*]}
    $(fetch_and_build_mesos "$mesos_version")
EOF
    cp -v mesos.executor*.egg "$python_outdir"
  popd
}

main() {
  local target_distribution=$1
  local mesos_version=$2
  local output_basedir=$3

  if [[ ! -d "$output_basedir" ]]; then
    echo "'$output_basedir' is not a directory."
    echo
    usage 1
  fi
  setup_tempdir
  pushd "$TMPDIR"
    case "$target_distribution" in
      jessie64)
        build_debian_jessie64 "$mesos_version" "$output_basedir"
        ;;
      trusty64)
        build_ubuntu_trusty64 "$mesos_version" "$output_basedir"
        ;;
      xenial64)
        build_ubuntu_xenial64 "$mesos_version" "$output_basedir"
        ;;
      centos6)
        build_centos6 "$mesos_version" "$output_basedir"
        ;;
      centos7)
        build_centos7 "$mesos_version" "$output_basedir"
        ;;
      *)
        echo "Invalid distribution '$target_distribution'."
        echo
        usage 1
        ;;
    esac
  popd
}

if [[ ${1:-""} = -h ]]; then usage 0; elif [[ $# -ne 3 ]]; then usage 1; fi
main "$@"