[FLINK-16880] Add tools and template Dockerfiles to facilitate release workflow
This closes #1.
diff --git a/README.md b/README.md
index 2879e5c..c755abc 100644
--- a/README.md
+++ b/README.md
@@ -5,6 +5,32 @@
These Dockerfiles are maintained by the Apache Flink community.
+Workflow for new releases
+-------------------------
+
+### Release workflow
+
+When a new release of Flink Stateful Functions is available, the Dockerfiles in this repo should be updated.
+
+Updating the Dockerfiles involves 3 steps:
+
+1. Add the GPG key ID of the key used to sign the new release to the `gpg_keys.txt` file.
+ * Entry format: <statefun_version>=<signing_key>, e.g., 2.0.0=1C1E2394D3194E1944613488F320986D35C33D6A
+ * Commit this change with message `[release] Add GPG key for x.y.z release`.
+2. Run `add-version.sh` with the appropriate arguments (`-s statefun-version -f flink-version`)
+ * e.g. `./add-version.sh -s 2.0.0 -f 1.10.0`
+ * Commit these changes with message `[release] Update Dockerfiles for x.y.z release`.
+
+A pull request can then be opened on this repo with the changes.
+
+### Release checklist
+
+- [ ] The GPG key ID of the key used to sign the new release has been added to `gpg_keys.txt` and
+ committed with the message `[release] Add GPG key for x.y.z release`
+- [ ] `./add-version.sh -s x.y.z -f a.b.c` has been run, and the new Dockerfiles committed with
+ the message `[release] Update Dockerfiles for x.y.z release`
+- [ ] A pull request with the above changes has been opened on this repo and merged
+
License
-------
diff --git a/add-version.sh b/add-version.sh
new file mode 100755
index 0000000..78e5bb9
--- /dev/null
+++ b/add-version.sh
@@ -0,0 +1,77 @@
+#!/bin/bash -e
+
+# Use this script to rebuild the Dockerfiles and all variants for a particular
+# release. Before running this, you must first delete the existing release
+# directory.
+#
+# TODO: to conform with other similar setups, this likely needs to become
+# "update.sh" and be taught how to derive the latest version (e.g. 1.2.0) from
+# a given release (e.g. 1.2) and assemble a .travis.yml file dynamically.
+#
+# See other repos (e.g. httpd, cassandra) for update.sh examples.
+
+function usage() {
+ echo >&2 "usage: $0 -s statefun-version -f flink-version"
+}
+
+function error() {
+ local msg="$1"
+ if [ -n "$2" ]; then
+ local code="$2"
+ else
+ local code=1
+ fi
+ echo >&2 "$msg"
+ exit "$code"
+}
+
+statefun_version= # Like 2.0.0
+flink_version= # Like 1.10.0
+
+while getopts s:f:h arg; do
+ case "$arg" in
+ s)
+ statefun_version=$OPTARG
+ ;;
+ f)
+ flink_version=$OPTARG
+ ;;
+ h)
+ usage
+ exit 0
+ ;;
+ \?)
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+if [ -z "$statefun_version" ] || [ -z "$flink_version" ]; then
+ usage
+ exit 1
+fi
+
+gpg_key=`grep "$statefun_version" gpg_keys.txt | cut -d '=' -f2`
+
+if [ -z "$gpg_key" ]; then
+ error "Missing GPG key ID in gpg_keys.txt file for release $statefun_version"
+fi
+
+if [ -d "$statefun_version" ]; then
+ error "Directory $statefun_version already exists; delete before continuing"
+fi
+
+echo -n >&2 "Generating Dockerfiles..."
+dir="$statefun_version"
+mkdir "$dir"
+
+cp -r template/* "$dir"
+
+sed \
+ -e "s/%%STATEFUN_VERSION%%/$statefun_version/" \
+ -e "s/%%FLINK_VERSION%%/$flink_version/" \
+ -e "s/%%GPG_KEY%%/$gpg_key/" \
+ "template/Dockerfile" > "$dir/Dockerfile"
+
+echo >&2 " done."
\ No newline at end of file
diff --git a/gpg_keys.txt b/gpg_keys.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/gpg_keys.txt
diff --git a/template/Dockerfile b/template/Dockerfile
new file mode 100644
index 0000000..2e41900
--- /dev/null
+++ b/template/Dockerfile
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM flink:%%FLINK_VERSION%%
+
+ENV STATEFUN_VERSION=%%STATEFUN_VERSION%% \
+ GPG_KEY=%%GPG_KEY%%
+
+ENV ROLE worker
+ENV MASTER_HOST localhost
+ENV STATEFUN_HOME /opt/statefun
+ENV STATEFUN_MODULES $STATEFUN_HOME/modules
+
+# Cleanup flink-lib
+RUN rm -fr $FLINK_HOME/lib/flink-table*jar
+
+# Copy our distriubtion template
+COPY flink-distribution/ $FLINK_HOME/
+
+# Install Stateful Functions dependencies in Flink lib
+ENV DIST_JAR_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-distribution/${STATEFUN_VERSION}/statefun-flink-distribution-${STATEFUN_VERSION}.jar \
+ DIST_ASC_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-distribution/${STATEFUN_VERSION}/statefun-flink-distribution-${STATEFUN_VERSION}.jar.asc \
+ CORE_JAR_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-core/${STATEFUN_VERSION}/statefun-flink-core-${STATEFUN_VERSION}.jar \
+ CORE_ASC_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-core/${STATEFUN_VERSION}/statefun-flink-core-${STATEFUN_VERSION}.jar.asc
+
+RUN set -ex; \
+ wget -nv -O statefun-flink-distribution.jar "$DIST_JAR_URL"; \
+ wget -nv -O statefun-flink-distribution.jar.asc "$DIST_ASC_URL"; \
+ wget -nv -O statefun-flink-core.jar "$CORE_JAR_URL"; \
+ wget -nv -O statefun-flink-core.jar.asc "$CORE_ASC_URL"; \
+ \
+ export GNUPGHOME="$(mktemp -d)"; \
+ for server in ha.pool.sks-keyservers.net $(shuf -e \
+ hkp://p80.pool.sks-keyservers.net:80 \
+ keyserver.ubuntu.com \
+ hkp://keyserver.ubuntu.com:80 \
+ pgp.mit.edu) ; do \
+ gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+ done && \
+ gpg --batch --verify statefun-flink-distribution.jar.asc statefun-flink-distribution.jar; \
+ gpg --batch --verify statefun-flink-core.jar.asc statefun-flink-core.jar; \
+ gpgconf --kill all; \
+ rm -rf "$GNUPGHOME" statefun-flink-distribution.jar.asc statefun-flink-core.jar.asc; \
+ \
+ mkdir -p $FLINK_HOME/lib; \
+ mv statefun-flink-distribution.jar $FLINK_HOME/lib; \
+ mv statefun-flink-core.jar $FLINK_HOME/lib;
+
+# add user modules
+USER root
+
+RUN mkdir -p $STATEFUN_MODULES && \
+ useradd --system --home-dir $STATEFUN_HOME --uid=9998 --gid=flink statefun && \
+ chown -R statefun:flink $STATEFUN_HOME && \
+ chmod -R g+rw $STATEFUN_HOME
+
+# entry point
+ADD docker-entry-point.sh /docker-entry-point.sh
+
+ENTRYPOINT ["/docker-entry-point.sh"]
diff --git a/template/docker-entry-point.sh b/template/docker-entry-point.sh
new file mode 100755
index 0000000..4e502b4
--- /dev/null
+++ b/template/docker-entry-point.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#
+# Role types
+#
+WORKER="worker"
+MASTER="master"
+
+#
+# Environment
+#
+FLINK_HOME=${FLINK_HOME:-"/opt/flink/bin"}
+ROLE=${ROLE:-"worker"}
+MASTER_HOST=${MASTER_HOST:-"localhost"}
+
+#
+# Start a service depending on the role.
+#
+if [[ "${ROLE}" == "${WORKER}" ]]; then
+ #
+ # start the TaskManager (worker role)
+ #
+ exec ${FLINK_HOME}/bin/taskmanager.sh start-foreground \
+ -Djobmanager.rpc.address=${MASTER_HOST}
+
+elif [[ "${ROLE}" == "${MASTER}" ]]; then
+ #
+ # start the JobManager (master role) with our predefined job.
+ #
+ exec $FLINK_HOME/bin/standalone-job.sh \
+ start-foreground \
+ -Djobmanager.rpc.address=${MASTER_HOST} \
+ "$@"
+else
+ #
+ # unknown role
+ #
+ echo "unknown role ${ROLE}"
+ exit 1
+fi
diff --git a/template/flink-distribution/bin/flink-console.sh b/template/flink-distribution/bin/flink-console.sh
new file mode 100755
index 0000000..16f97ad
--- /dev/null
+++ b/template/flink-distribution/bin/flink-console.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This file was taken from Apache Flink, and modified to include another entry point
+
+# Start a Flink service as a console application. Must be stopped with Ctrl-C
+# or with SIGTERM by kill or the controlling process.
+USAGE="Usage: flink-console.sh (taskexecutor|zookeeper|historyserver|standalonesession|standalonejob|statefun) [args]"
+
+SERVICE=$1
+ARGS=("${@:2}") # get remaining arguments as array
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/config.sh
+
+case ${SERVICE} in
+ (taskexecutor)
+ CLASS_TO_RUN=org.apache.flink.runtime.taskexecutor.TaskManagerRunner
+ ;;
+
+ (historyserver)
+ CLASS_TO_RUN=org.apache.flink.runtime.webmonitor.history.HistoryServer
+ ;;
+
+ (zookeeper)
+ CLASS_TO_RUN=org.apache.flink.runtime.zookeeper.FlinkZooKeeperQuorumPeer
+ ;;
+
+ (standalonesession)
+ CLASS_TO_RUN=org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint
+ ;;
+
+ (standalonejob)
+ CLASS_TO_RUN=org.apache.flink.container.entrypoint.StandaloneJobClusterEntryPoint
+ ;;
+
+ (statefun)
+ CLASS_TO_RUN=org.apache.flink.statefun.flink.launcher.StatefulFunctionsClusterEntryPoint
+ ;;
+
+ (*)
+ echo "Unknown service '${SERVICE}'. $USAGE."
+ exit 1
+ ;;
+esac
+
+FLINK_TM_CLASSPATH=`constructFlinkClassPath`
+
+log_setting=("-Dlog4j.configuration=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlogback.configurationFile=file:${FLINK_CONF_DIR}/logback-console.xml")
+
+JAVA_VERSION=$(${JAVA_RUN} -version 2>&1 | sed 's/.*version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
+
+# Only set JVM 8 arguments if we have correctly extracted the version
+if [[ ${JAVA_VERSION} =~ ${IS_NUMBER} ]]; then
+ if [ "$JAVA_VERSION" -lt 18 ]; then
+ JVM_ARGS="$JVM_ARGS -XX:MaxPermSize=256m"
+ fi
+fi
+
+echo "Starting $SERVICE as a console application on host $HOSTNAME."
+exec $JAVA_RUN ${JVM_ARGS} ${FLINK_ENV_JAVA_OPTS} "${log_setting[@]}" -classpath "`manglePathList "$FLINK_TM_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS"`" ${CLASS_TO_RUN} "${ARGS[@]}"
diff --git a/template/flink-distribution/bin/standalone-job.sh b/template/flink-distribution/bin/standalone-job.sh
new file mode 100755
index 0000000..e9def43
--- /dev/null
+++ b/template/flink-distribution/bin/standalone-job.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This file was taken from Apache Flink, and modified to include another entry point
+
+# Start/stop a Flink JobManager.
+USAGE="Usage: standalone-job.sh ((start|start-foreground))|stop [args]"
+
+STARTSTOP=$1
+ENTRY_POINT_NAME="statefun"
+
+if [[ ${STARTSTOP} != "start" ]] && [[ ${STARTSTOP} != "start-foreground" ]] && [[ ${STARTSTOP} != "stop" ]]; then
+ echo ${USAGE}
+ exit 1
+fi
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/config.sh
+
+# Startup parameters
+ARGS=("--configDir" "${FLINK_CONF_DIR}" "${@:2}")
+
+if [[ ${STARTSTOP} == "start" ]] || [[ ${STARTSTOP} == "start-foreground" ]]; then
+ if [ ! -z "${FLINK_JM_HEAP_MB}" ] && [ "${FLINK_JM_HEAP}" == 0 ]; then
+ echo "used deprecated key \`${KEY_JOBM_MEM_MB}\`, please replace with key \`${KEY_JOBM_MEM_SIZE}\`"
+ else
+ flink_jm_heap_bytes=$(parseBytes ${FLINK_JM_HEAP})
+ FLINK_JM_HEAP_MB=$(getMebiBytes ${flink_jm_heap_bytes})
+ fi
+
+ if [[ ! ${FLINK_JM_HEAP_MB} =~ $IS_NUMBER ]] || [[ "${FLINK_JM_HEAP_MB}" -lt "0" ]]; then
+ echo "[ERROR] Configured memory size is not a valid value. Please set '${KEY_JOBM_MEM_SIZE}' in ${FLINK_CONF_FILE}."
+ exit 1
+ fi
+
+ if [ "${FLINK_JM_HEAP_MB}" -gt "0" ]; then
+ export JVM_ARGS="$JVM_ARGS -Xms"$FLINK_JM_HEAP_MB"m -Xmx"$FLINK_JM_HEAP_MB"m"
+ fi
+
+ # Add cluster entry point specific JVM options
+ export FLINK_ENV_JAVA_OPTS="${FLINK_ENV_JAVA_OPTS} ${FLINK_ENV_JAVA_OPTS_JM}"
+fi
+
+if [[ $STARTSTOP == "start-foreground" ]]; then
+ exec "${FLINK_BIN_DIR}"/flink-console.sh ${ENTRY_POINT_NAME} "${ARGS[@]}"
+else
+ "${FLINK_BIN_DIR}"/flink-daemon.sh ${STARTSTOP} ${ENTRY_POINT_NAME} "${ARGS[@]}"
+fi
diff --git a/template/flink-distribution/conf/flink-conf.yaml b/template/flink-distribution/conf/flink-conf.yaml
new file mode 100644
index 0000000..574ccb0
--- /dev/null
+++ b/template/flink-distribution/conf/flink-conf.yaml
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This file is the base for the Apache Flink configuration
+
+#==============================================================================
+# Configurations strictly required by Stateful Functions. Do not change.
+#==============================================================================
+
+classloader.parent-first-patterns.additional: org.apache.flink.statefun;org.apache.kafka;com.google.protobuf
+execution.checkpointing.max-concurrent-checkpoints: 1
+jobmanager.scheduler: legacy
+
+#==============================================================================
+# Recommended configurations. Users may change according to their needs.
+#==============================================================================
+
+state.backend: rocksdb
+state.backend.rocksdb.timer-service.factory: ROCKSDB
+state.checkpoints.dir: file:///checkpoint-dir
+state.backend.incremental: true
+taskmanager.memory.process.size: 4g
