[FLINK-17367] Add script for building with configurable download URL
diff --git a/.gitignore b/.gitignore
index c34e36b..feb151f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
.*.swp
-.idea
\ No newline at end of file
+.idea
+dev
\ No newline at end of file
diff --git a/1.10/scala_2.11-debian/Dockerfile b/1.10/scala_2.11-debian/Dockerfile
index 5c84be8..f7b5f6d 100644
--- a/1.10/scala_2.11-debian/Dockerfile
+++ b/1.10/scala_2.11-debian/Dockerfile
@@ -46,7 +46,8 @@
# Configure Flink version
ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.10.0/flink-1.10.0-bin-scala_2.11.tgz \
FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.10.0/flink-1.10.0-bin-scala_2.11.tgz.asc \
- GPG_KEY=BB137807CEFBE7DD2616556710B12A1F89C115E8
+ GPG_KEY=BB137807CEFBE7DD2616556710B12A1F89C115E8 \
+ SKIP_GPG=false
# Prepare environment
ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@
# Install Flink
RUN set -ex; \
wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
- wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
\
- export GNUPGHOME="$(mktemp -d)"; \
- for server in ha.pool.sks-keyservers.net $(shuf -e \
- hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com \
- hkp://keyserver.ubuntu.com:80 \
- pgp.mit.edu) ; do \
- gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
- done && \
- gpg --batch --verify flink.tgz.asc flink.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ if [ "$SKIP_GPG" = "false" ]; then \
+ wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+ export GNUPGHOME="$(mktemp -d)"; \
+ for server in ha.pool.sks-keyservers.net $(shuf -e \
+ hkp://p80.pool.sks-keyservers.net:80 \
+ keyserver.ubuntu.com \
+ hkp://keyserver.ubuntu.com:80 \
+ pgp.mit.edu) ; do \
+ gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+ done && \
+ gpg --batch --verify flink.tgz.asc flink.tgz; \
+ gpgconf --kill all; \
+ rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ fi; \
\
tar -xf flink.tgz --strip-components=1; \
rm flink.tgz; \
diff --git a/1.10/scala_2.12-debian/Dockerfile b/1.10/scala_2.12-debian/Dockerfile
index c8e1c09..cf22764 100644
--- a/1.10/scala_2.12-debian/Dockerfile
+++ b/1.10/scala_2.12-debian/Dockerfile
@@ -46,7 +46,8 @@
# Configure Flink version
ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.10.0/flink-1.10.0-bin-scala_2.12.tgz \
FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.10.0/flink-1.10.0-bin-scala_2.12.tgz.asc \
- GPG_KEY=BB137807CEFBE7DD2616556710B12A1F89C115E8
+ GPG_KEY=BB137807CEFBE7DD2616556710B12A1F89C115E8 \
+ SKIP_GPG=false
# Prepare environment
ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@
# Install Flink
RUN set -ex; \
wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
- wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
\
- export GNUPGHOME="$(mktemp -d)"; \
- for server in ha.pool.sks-keyservers.net $(shuf -e \
- hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com \
- hkp://keyserver.ubuntu.com:80 \
- pgp.mit.edu) ; do \
- gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
- done && \
- gpg --batch --verify flink.tgz.asc flink.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ if [ "$SKIP_GPG" = "false" ]; then \
+ wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+ export GNUPGHOME="$(mktemp -d)"; \
+ for server in ha.pool.sks-keyservers.net $(shuf -e \
+ hkp://p80.pool.sks-keyservers.net:80 \
+ keyserver.ubuntu.com \
+ hkp://keyserver.ubuntu.com:80 \
+ pgp.mit.edu) ; do \
+ gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+ done && \
+ gpg --batch --verify flink.tgz.asc flink.tgz; \
+ gpgconf --kill all; \
+ rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ fi; \
\
tar -xf flink.tgz --strip-components=1; \
rm flink.tgz; \
diff --git a/1.9/scala_2.11-debian/Dockerfile b/1.9/scala_2.11-debian/Dockerfile
index 8a82570..8f71da8 100644
--- a/1.9/scala_2.11-debian/Dockerfile
+++ b/1.9/scala_2.11-debian/Dockerfile
@@ -46,7 +46,8 @@
# Configure Flink version
ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.9.3/flink-1.9.3-bin-scala_2.11.tgz \
FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.9.3/flink-1.9.3-bin-scala_2.11.tgz.asc \
- GPG_KEY=6B6291A8502BA8F0913AE04DDEB95B05BF075300
+ GPG_KEY=6B6291A8502BA8F0913AE04DDEB95B05BF075300 \
+ SKIP_GPG=false
# Prepare environment
ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@
# Install Flink
RUN set -ex; \
wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
- wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
\
- export GNUPGHOME="$(mktemp -d)"; \
- for server in ha.pool.sks-keyservers.net $(shuf -e \
- hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com \
- hkp://keyserver.ubuntu.com:80 \
- pgp.mit.edu) ; do \
- gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
- done && \
- gpg --batch --verify flink.tgz.asc flink.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ if [ "$SKIP_GPG" = "false" ]; then \
+ wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+ export GNUPGHOME="$(mktemp -d)"; \
+ for server in ha.pool.sks-keyservers.net $(shuf -e \
+ hkp://p80.pool.sks-keyservers.net:80 \
+ keyserver.ubuntu.com \
+ hkp://keyserver.ubuntu.com:80 \
+ pgp.mit.edu) ; do \
+ gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+ done && \
+ gpg --batch --verify flink.tgz.asc flink.tgz; \
+ gpgconf --kill all; \
+ rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ fi; \
\
tar -xf flink.tgz --strip-components=1; \
rm flink.tgz; \
diff --git a/1.9/scala_2.12-debian/Dockerfile b/1.9/scala_2.12-debian/Dockerfile
index dc30005..095e9d2 100644
--- a/1.9/scala_2.12-debian/Dockerfile
+++ b/1.9/scala_2.12-debian/Dockerfile
@@ -46,7 +46,8 @@
# Configure Flink version
ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.9.3/flink-1.9.3-bin-scala_2.12.tgz \
FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.9.3/flink-1.9.3-bin-scala_2.12.tgz.asc \
- GPG_KEY=6B6291A8502BA8F0913AE04DDEB95B05BF075300
+ GPG_KEY=6B6291A8502BA8F0913AE04DDEB95B05BF075300 \
+ SKIP_GPG=false
# Prepare environment
ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@
# Install Flink
RUN set -ex; \
wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
- wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
\
- export GNUPGHOME="$(mktemp -d)"; \
- for server in ha.pool.sks-keyservers.net $(shuf -e \
- hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com \
- hkp://keyserver.ubuntu.com:80 \
- pgp.mit.edu) ; do \
- gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
- done && \
- gpg --batch --verify flink.tgz.asc flink.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ if [ "$SKIP_GPG" = "false" ]; then \
+ wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+ export GNUPGHOME="$(mktemp -d)"; \
+ for server in ha.pool.sks-keyservers.net $(shuf -e \
+ hkp://p80.pool.sks-keyservers.net:80 \
+ keyserver.ubuntu.com \
+ hkp://keyserver.ubuntu.com:80 \
+ pgp.mit.edu) ; do \
+ gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+ done && \
+ gpg --batch --verify flink.tgz.asc flink.tgz; \
+ gpgconf --kill all; \
+ rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ fi; \
\
tar -xf flink.tgz --strip-components=1; \
rm flink.tgz; \
diff --git a/Dockerfile-debian.template b/Dockerfile-debian.template
index 0a206e6..7222314 100644
--- a/Dockerfile-debian.template
+++ b/Dockerfile-debian.template
@@ -46,7 +46,8 @@
# Configure Flink version
ENV FLINK_TGZ_URL=%%BINARY_DOWNLOAD_URL%% \
FLINK_ASC_URL=%%ASC_DOWNLOAD_URL%% \
- GPG_KEY=%%GPG_KEY%%
+ GPG_KEY=%%GPG_KEY%% \
+ CHECK_GPG=%%CHECK_GPG%%
# Prepare environment
ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@
# Install Flink
RUN set -ex; \
wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
- wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
\
- export GNUPGHOME="$(mktemp -d)"; \
- for server in ha.pool.sks-keyservers.net $(shuf -e \
- hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com \
- hkp://keyserver.ubuntu.com:80 \
- pgp.mit.edu) ; do \
- gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
- done && \
- gpg --batch --verify flink.tgz.asc flink.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ if [ "$CHECK_GPG" = "true" ]; then \
+ wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+ export GNUPGHOME="$(mktemp -d)"; \
+ for server in ha.pool.sks-keyservers.net $(shuf -e \
+ hkp://p80.pool.sks-keyservers.net:80 \
+ keyserver.ubuntu.com \
+ hkp://keyserver.ubuntu.com:80 \
+ pgp.mit.edu) ; do \
+ gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+ done && \
+ gpg --batch --verify flink.tgz.asc flink.tgz; \
+ gpgconf --kill all; \
+ rm -rf "$GNUPGHOME" flink.tgz.asc; \
+ fi; \
\
tar -xf flink.tgz --strip-components=1; \
rm flink.tgz; \
diff --git a/add-custom.sh b/add-custom.sh
new file mode 100755
index 0000000..c863f3f
--- /dev/null
+++ b/add-custom.sh
@@ -0,0 +1,49 @@
+#!/bin/bash -e
+
+# Use this script to build the Dockerfiles against an arbitrary
+# Flink distribution.
+# This is exlusively for development purposes.
+
+source "$(dirname "$0")"/common.sh
+
+function usage() {
+ echo >&2 "usage: $0 -u binary-download-url [-n name]"
+}
+
+binary_download_url=
+name=custom
+
+while getopts u:n:h arg; do
+ case "$arg" in
+ u)
+ binary_download_url=$OPTARG
+ ;;
+ n)
+ name=$OPTARG
+ ;;
+ h)
+ usage
+ exit 0
+ ;;
+ \?)
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+if [ -z "${binary_download_url}" ]; then
+ usage
+ exit 1
+fi
+
+mkdir -p "dev"
+
+echo -n >&2 "Generating Dockerfiles..."
+for source_variant in "${SOURCE_VARIANTS[@]}"; do
+ dir="dev/${name}-${source_variant}"
+ rm -rf "${dir}"
+
+ generate "${dir}" "${binary_download_url}" "" "" false ${source_variant}
+done
+echo >&2 " done."
diff --git a/add-version.sh b/add-version.sh
index 779113b..329ed70 100755
--- a/add-version.sh
+++ b/add-version.sh
@@ -10,6 +10,8 @@
#
# See other repos (e.g. httpd, cassandra) for update.sh examples.
+source "$(dirname "$0")"/common.sh
+
function usage() {
echo >&2 "usage: $0 -r flink-release -f flink-version"
}
@@ -57,7 +59,6 @@
fi
# Defaults, can vary between versions
-source_variants=( debian )
scala_versions=( 2.11 2.12 )
gpg_key=
@@ -95,28 +96,8 @@
mkdir "$flink_release"
-function generate() {
- dir=$1
- binary_download_url=$2
- asc_download_url=$3
- gpg_key=$4
- source_variant=$5
-
- mkdir "$dir"
- cp docker-entrypoint.sh "$dir/docker-entrypoint.sh"
-
- # '&' has special semantics in sed replacement patterns
- escaped_binary_download_url=$(echo "$binary_download_url" | sed 's/&/\\\&/')
-
- sed \
- -e "s,%%BINARY_DOWNLOAD_URL%%,${escaped_binary_download_url}," \
- -e "s,%%ASC_DOWNLOAD_URL%%,$asc_download_url," \
- -e "s/%%GPG_KEY%%/$gpg_key/" \
- "Dockerfile-$source_variant.template" > "$dir/Dockerfile"
-}
-
echo -n >&2 "Generating Dockerfiles..."
-for source_variant in "${source_variants[@]}"; do
+for source_variant in "${SOURCE_VARIANTS[@]}"; do
for scala_version in "${scala_versions[@]}"; do
dir="$flink_release/scala_${scala_version}-${source_variant}"
@@ -126,7 +107,7 @@
# Not all mirrors have the .asc files
flink_asc_url=https://www.apache.org/dist/${flink_url_file_path}.asc
- generate "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} ${source_variant}
+ generate "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} true ${source_variant}
done
done
echo >&2 " done."
diff --git a/common.sh b/common.sh
new file mode 100644
index 0000000..0232930
--- /dev/null
+++ b/common.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -e
+
+# Defaults, can vary between versions
+export SOURCE_VARIANTS=( debian )
+
+function generate() {
+ dir=$1
+ binary_download_url=$2
+ asc_download_url=$3
+ gpg_key=$4
+ check_gpg=$5
+ source_variant=$6
+
+ mkdir "$dir"
+ cp docker-entrypoint.sh "$dir/docker-entrypoint.sh"
+
+ # '&' has special semantics in sed replacement patterns
+ escaped_binary_download_url=$(echo "$binary_download_url" | sed 's/&/\\\&/')
+
+ sed \
+ -e "s,%%BINARY_DOWNLOAD_URL%%,${escaped_binary_download_url}," \
+ -e "s,%%ASC_DOWNLOAD_URL%%,$asc_download_url," \
+ -e "s/%%GPG_KEY%%/$gpg_key/" \
+ -e "s/%%CHECK_GPG%%/${check_gpg}/" \
+ "Dockerfile-$source_variant.template" > "$dir/Dockerfile"
+}
