Add a tool and environment for building Aurora packages.
Bugs closed: AURORA-1410
Reviewed at https://reviews.apache.org/r/37107/
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..849ddff
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+dist/
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e45f22f
--- /dev/null
+++ b/README.md
@@ -0,0 +1,33 @@
+## Packaging for Apache Aurora
+
+This repository maintains configuration and tooling for building binary
+distributions of [Apache Aurora](https://aurora.apache.org/).
+
+### Building a binary
+
+Binaries are built within Docker containers that provide the appropriate build
+environment for the target platform. You will need to have a working Docker
+installation before proceeding.
+
+1. Fetch a source distribution, such as an
+ [official one](https://aurora.apache.org/downloads/).
+
+2. Run the builder script, providing the distribution platform and the source
+ distribution archive you downloaded in (1). The example below will build
+ Aurora 0.9.0 debs for Ubuntu Trusty.
+
+ ```
+ ./build-artifact.sh builder/deb/ubuntu-trusty \
+ ~/Downloads/apache-aurora-0.9.0.tar.gz
+ ```
+
+When this completes, debs will be placed in `dist/builder/deb/ubuntu-trusty/`.
+
+### Adding a new distribution platform
+
+There are only two requirements for a 'builder' to satisfy:
+
+- a `Dockerfile` to provide the repeatable build environment
+- a `build.sh` script that creates artifacts
+
+Please see the makeup of other builders for examples.
\ No newline at end of file
diff --git a/build-artifact.sh b/build-artifact.sh
new file mode 100755
index 0000000..e6ea723
--- /dev/null
+++ b/build-artifact.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -eu
+
+print_available_builders() {
+ find builder -name Dockerfile | sed "s/\/Dockerfile$//"
+}
+
+if [[ $# -ne 2 ]]; then
+ echo "usage: $0 BUILDER RELEASE_TAR"
+ echo 'Where BUILDER is a builder directory in:'
+ print_available_builders
+ exit 1
+else
+ BUILDER_DIR=$1
+ RELEASE_TAR=$2
+fi
+
+IMAGE_NAME="aurora-$(basename $BUILDER_DIR)"
+docker build -t "$IMAGE_NAME" "$BUILDER_DIR"
+
+ARTIFACT_DIR="$(pwd)/dist/$BUILDER_DIR"
+mkdir -p $ARTIFACT_DIR
+docker run \
+ --rm \
+ -v "$(pwd)/specs:/specs:ro" \
+ -v "$(realpath $RELEASE_TAR):/src.tar.gz:ro" \
+ -v "$ARTIFACT_DIR:/dist" \
+ -t "$IMAGE_NAME" /build.sh
+
+echo 'Produced artifacts:'
+ls $ARTIFACT_DIR
diff --git a/builder/deb/ubuntu-trusty/Dockerfile b/builder/deb/ubuntu-trusty/Dockerfile
new file mode 100644
index 0000000..73f150b
--- /dev/null
+++ b/builder/deb/ubuntu-trusty/Dockerfile
@@ -0,0 +1,52 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM ubuntu:trusty
+
+WORKDIR /aurora
+ENV HOME /aurora
+ENV DEBIAN_FRONTEND noninteractive
+# TODO(wfarner): Provide external parameterization via env vars for versions.
+ENV THRIFT_DEB thrift-compiler_0.9.1_amd64.deb
+
+RUN apt-get update && apt-get -y install \
+ bison \
+ debhelper \
+ devscripts \
+ dpkg-dev \
+ curl \
+ git \
+ libapr1-dev \
+ libcurl4-openssl-dev \
+ libsvn-dev \
+ python-all-dev \
+ software-properties-common
+
+RUN add-apt-repository ppa:openjdk-r/ppa -y \
+ && apt-get update \
+ && apt-get install -y openjdk-8-jdk \
+ && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
+
+RUN curl -O http://people.apache.org/~jfarrell/thrift/0.9.1/contrib/deb/ubuntu/12.04/$THRIFT_DEB \
+ && dpkg --install $THRIFT_DEB \
+ && rm $THRIFT_DEB
+
+# Install gradle 2.5.
+RUN git clone --depth 1 https://github.com/benley/gradle-packaging \
+ && cd gradle-packaging \
+ && apt-get install -y ruby ruby-dev unzip wget \
+ && gem install fpm && ./gradle-mkdeb.sh 2.5 \
+ && dpkg -i gradle-2.5_2.5-2_all.deb \
+ && cd .. && rm -rf gradle-packaging
+
+ADD build.sh /build.sh
+ADD pants.ini /pants.ini
diff --git a/builder/deb/ubuntu-trusty/build.sh b/builder/deb/ubuntu-trusty/build.sh
new file mode 100755
index 0000000..6bee0e7
--- /dev/null
+++ b/builder/deb/ubuntu-trusty/build.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+mkdir /scratch
+cd /scratch
+
+tar --strip-components 1 -C . -xf /src.tar.gz
+
+cp -R /specs/debian .
+
+dch \
+ -b \
+ -v $(cat .auroraversion)~SNAPSHOT \
+ -u low --maintmaint \
+ "Apache Aurora package builder <dev@aurora.apache.org> $(date -R)"
+
+dpkg-buildpackage -uc -b -tc
+
+mv ../*.deb /dist
diff --git a/specs/debian/pants.ini b/builder/deb/ubuntu-trusty/pants.ini
similarity index 100%
rename from specs/debian/pants.ini
rename to builder/deb/ubuntu-trusty/pants.ini
diff --git a/builder/rpm/centos-7/Dockerfile b/builder/rpm/centos-7/Dockerfile
new file mode 100644
index 0000000..05da048
--- /dev/null
+++ b/builder/rpm/centos-7/Dockerfile
@@ -0,0 +1,37 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM centos:7
+
+RUN yum install -y \
+ make \
+ rpm-build \
+ yum-utils \
+ apr-devel \
+ cyrus-sasl-devel \
+ gcc \
+ gcc-c++ \
+ git \
+ java-1.8.0-openjdk-devel \
+ krb5-devel \
+ libcurl-devel \
+ patch \
+ python \
+ python-devel \
+ subversion-devel \
+ tar \
+ unzip \
+ wget \
+ zlib-devel
+
+ADD build.sh /build.sh
+ADD pants.ini /pants.ini
diff --git a/builder/rpm/centos-7/build.sh b/builder/rpm/centos-7/build.sh
new file mode 100755
index 0000000..9e8eae9
--- /dev/null
+++ b/builder/rpm/centos-7/build.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+mkdir -p /scratch/src
+cd /scratch
+
+tar --strip-components 1 -C src -xf /src.tar.gz
+
+cp -R /specs/rpm .
+cd rpm
+
+make srpm
+yum-builddep -y ../../../dist/rpmbuild/SRPMS/*
+make rpm
diff --git a/builder/rpm/centos-7/pants.ini b/builder/rpm/centos-7/pants.ini
new file mode 100644
index 0000000..00074cb
--- /dev/null
+++ b/builder/rpm/centos-7/pants.ini
@@ -0,0 +1,9 @@
+[DEFAULT]
+pants_cachedir: %(homedir)s/.pants.d
+
+[python-setup]
+download_cache: %(pants_cachedir)s/python/downloads
+install_cache: %(pants_cachedir)s/python/eggs
+
+[python-repos]
+repos: ['third_party/', 'https://svn.apache.org/repos/asf/incubator/aurora/3rdparty/centos/7/python/']
diff --git a/specs/debian/rules b/specs/debian/rules
index d981400..ef35a04 100755
--- a/specs/debian/rules
+++ b/specs/debian/rules
@@ -21,7 +21,7 @@
gradle = $(shell which gradle)
pants = $(CURDIR)/pants
-PANTS_CONFIG_OVERRIDE = $(CURDIR)/debian/pants.ini
+PANTS_CONFIG_OVERRIDE = /pants.ini
export PANTS_CONFIG_OVERRIDE
override_dh_auto_build: __gradle_build __pants_build
@@ -31,12 +31,12 @@
__pants_build:
mkdir -p third_party
- $(pants) binary src/main/python/apache/aurora/executor:thermos_executor
- $(pants) binary src/main/python/apache/aurora/kerberos:kaurora
- $(pants) binary src/main/python/apache/aurora/kerberos:kaurora_admin
+ $(pants) binary src/main/python/apache/aurora/executor/bin:thermos_executor
+ $(pants) binary src/main/python/apache/aurora/client/cli:kaurora
+ $(pants) binary src/main/python/apache/aurora/admin:kaurora_admin
$(pants) binary src/main/python/apache/aurora/tools:thermos
$(pants) binary src/main/python/apache/aurora/tools:thermos_observer
- $(pants) binary src/main/python/apache/thermos/runner:thermos_runner
+ $(pants) binary src/main/python/apache/thermos/bin:thermos_runner
build-support/embed_runner_in_executor.py
override_dh_installinit:
diff --git a/specs/rpm/Makefile b/specs/rpm/Makefile
index 1833a25..77605fe 100644
--- a/specs/rpm/Makefile
+++ b/specs/rpm/Makefile
@@ -10,20 +10,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-GIT_COMMIT ?= HEAD
-
-USING_GIT ?= $(shell git status >/dev/null 2>&1 && echo 'true' || echo 'false')
-ifeq ($(USING_GIT),true)
- VERSION_SUFFIX := .$(shell git rev-parse --short $(GIT_COMMIT))
-else
- VERSION_SUFFIX :=
-endif
-
CURRENT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-TOPDIR ?= $(CURRENT_DIR)/../../..
-DIST_DIR ?= $(TOPDIR)/dist
+TOPDIR ?= /scratch/src
+DIST_DIR ?= /dist
RPM_TOPDIR ?= $(DIST_DIR)
DATETIME ?= $(shell date +%Y%m%d%H%M)
@@ -39,7 +29,7 @@
all: release_rpm
nightly_version:
- $(eval AURORA_VERSION := $(AURORA_VERSION).$(DATETIME)$(VERSION_SUFFIX))
+ $(eval AURORA_VERSION := $(AURORA_VERSION).$(DATETIME))
release_version:
$(eval AURORA_VERSION := $(AURORA_VERSION))
@@ -53,20 +43,9 @@
mkdir -p $(RPM_TOPDIR)/rpmbuild/RPMS
mkdir -p $(RPM_TOPDIR)/rpmbuild/SOURCES
mkdir -p $(RPM_TOPDIR)/rpmbuild/SRPMS
+ cp /src.tar.gz $(SOURCE_TARGET)
-tar_source: mkdir
- if [[ "$(USING_GIT)" == 'true' ]]; then \
- cd `git rev-parse --show-toplevel` && git archive --format=tar --prefix=aurora/ $(GIT_COMMIT) | gzip > $(SOURCE_TARGET); \
- else \
- tmp_dir=`mktemp -d '/tmp/aurorabuild.XXXXXX'`; \
- mkdir $${tmp_dir}/aurora; \
- cp -r $(TOPDIR)/* $${tmp_dir}/aurora; \
- cd $${tmp_dir}; \
- tar cvzf $(SOURCE_TARGET) aurora; \
- rm -rf $${tmp_dir}; \
- fi
-
-srpm: tar_source
+srpm: mkdir
rpmbuild $(RPM_OPTS) \
--define "_topdir $(RPM_TOPDIR)" \
--define "_builddir %{_topdir}/rpmbuild/BUILD" \
@@ -79,7 +58,7 @@
--define "MESOS_VERSION $(MESOS_VERSION)" \
--nodeps -bs aurora.spec
-rpm: tar_source
+rpm: mkdir
rpmbuild $(RPM_OPTS) \
--define "_topdir $(RPM_TOPDIR)" \
--define "_builddir %{_topdir}/rpmbuild/BUILD" \
diff --git a/specs/rpm/aurora.spec b/specs/rpm/aurora.spec
index 5ec516f..e6fca4e 100644
--- a/specs/rpm/aurora.spec
+++ b/specs/rpm/aurora.spec
@@ -37,10 +37,6 @@
%global JAVA_VERSION 1.8.0
%endif
-%if %{?!MESOS_BASEURL:1}0
-%global MESOS_BASEURL https://archive.apache.org/dist/mesos
-%endif
-
%if %{?!MESOS_VERSION:1}0
%global MESOS_VERSION 0.22.0
%endif
@@ -141,7 +137,7 @@
%prep
-%setup -n %{name}
+%setup -n apache-%{name}-%{version}
%build
@@ -163,40 +159,27 @@
wget %{GRADLE_BASEURL}/gradle-%{GRADLE_VERSION}-bin.zip
unzip gradle-%{GRADLE_VERSION}-bin.zip
-# Creates Pants directory where we'll store our native Mesos Python eggs.
-mkdir -p .pants.d/python/eggs/
-
-# Builds mesos-native and mesos-interface eggs if not currently packaged.
-wget "%{MESOS_BASEURL}/%{MESOS_VERSION}/mesos-%{MESOS_VERSION}.tar.gz"
-tar xvzf mesos-%{MESOS_VERSION}.tar.gz
-pushd mesos-%{MESOS_VERSION}
-./configure --disable-java
-make
-find . -name '*.egg' -exec cp -v {} ../.pants.d/python/eggs/ \;
-popd
-
# Builds the Aurora scheduler.
./gradle-%{GRADLE_VERSION}/bin/gradle installDist
+# Configures pants to use our distributed platform-specific eggs.
+# This avoids building mesos to produce them.
+export PANTS_CONFIG_OVERRIDE=/pants.ini
+
# Builds Aurora client PEX binaries.
-./pants binary src/main/python/apache/aurora/kerberos:kaurora
-./pants binary src/main/python/apache/aurora/kerberos:kaurora_admin
+./pants binary src/main/python/apache/aurora/client/cli:kaurora
+mv dist/kaurora.pex dist/aurora.pex
+./pants binary src/main/python/apache/aurora/admin:kaurora_admin
+mv dist/kaurora_admin.pex dist/aurora_admin.pex
# Builds Aurora Thermos and GC executor PEX binaries.
-./pants binary src/main/python/apache/aurora/executor:thermos_executor
+./pants binary src/main/python/apache/aurora/executor/bin:thermos_executor
./pants binary src/main/python/apache/aurora/tools:thermos
-./pants binary src/main/python/apache/thermos/runner:thermos_runner
+./pants binary src/main/python/apache/thermos/bin:thermos_runner
./pants binary src/main/python/apache/aurora/tools:thermos_observer
# Packages the Thermos runner within the Thermos executor.
-python <<EOF
-import contextlib
-import zipfile
-with contextlib.closing(zipfile.ZipFile('dist/thermos_executor.pex', 'a')) as zf:
- zf.writestr('apache/aurora/executor/resources/__init__.py', '')
- zf.write('dist/thermos_runner.pex', 'apache/aurora/executor/resources/thermos_runner.pex')
-EOF
-
+build-support/embed_runner_in_executor.py
%install
rm -rf $RPM_BUILD_ROOT
@@ -224,10 +207,6 @@
install -m 755 dist/${pex_binary}.pex %{buildroot}%{_bindir}/${pex_binary}
done
-# Strip the "k" from Kerberized client binaries.
-mv %{buildroot}%{_bindir}/kaurora %{buildroot}%{_bindir}/aurora
-mv %{buildroot}%{_bindir}/kaurora_admin %{buildroot}%{_bindir}/aurora_admin
-
# Installs all support scripting.
%if 0%{?fedora} || 0%{?rhel} > 6
install -m 644 build-support/packaging/rpm/%{name}.service %{buildroot}%{_sysconfdir}/systemd/system/%{name}.service
