Add a container for building releases
This container is similar to the existing build-release-candidate
container, but is simplified to remove all logic around publishing and
signing. Instead, this builds all artifacts in a similar environment to
one that artifacts will be created in the future (e.g. we know use
Ubuntu). This will allow us to more easily locally build artifacts using
this container and compare them against release candidates to verify
reproducibility.
DAFFODIL-2971
diff --git a/containers/build-release/Dockerfile b/containers/build-release/Dockerfile
new file mode 100644
index 0000000..b5ac6e3
--- /dev/null
+++ b/containers/build-release/Dockerfile
@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# To make for reproducibility, this should match the version used by GitHub
+# actions CI release candidate workflow
+FROM docker.io/library/ubuntu:22.04
+
+ENV TZ=UTC
+
+# Install/configure dependencies. Note that these are not pinned to any
+# specific version, so buidls might different depending on when this Docker
+# image is built. Note that the sbt and yarn versions don't really matter all
+# too much since they are just used to bootstrap--the daffodil projects specify
+# the actual versions of yarn/sbt to use/download when building. We also need
+# to install SBT separtely since we need to install dependencies like curl and
+# gpg to verify SBT signature
+RUN \
+ dpkg --add-architecture i386 && \
+ apt-get update && \
+ apt-get install -qy \
+ clang \
+ curl \
+ git \
+ gpg \
+ libmxml-dev \
+ llvm \
+ npm \
+ openjdk-8-jdk-headless \
+ rpm \
+ unzip \
+ wine32 \
+ winetricks && \
+ echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" > /etc/apt/sources.list.d/sbt.list && \
+ curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | \
+ gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --import && \
+ chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg && \
+ apt-get update && \
+ apt-get install -y \
+ sbt && \
+ npm --no-update-notifier install --global yarn@1.22.19 node@20 && \
+ winetricks -q dotnet40
+
+# WIX is a Microsoft Windows Installer creation tool kit.
+#
+# Install wix, including changes to allow WiX to run in wine on Linux. See
+# src/wix_wine.sh for more details on why we need to do this and how it works
+#
+# Updating WIX should be done only if there is a specific need (for security, or other compelling reason)
+# because it is likely things will break and the release scripts/process will have to adapt.
+# The WIX version 3.11.2 is hard coded into these script lines as tokens wix3112rtm and wix311.
+#
+# In order to ensure we are downloading and using the exact WIX binary we have tested and trust
+# we verify the sha512 is the same as the one expected. This protects us from if someone
+# was to get github credentials allowing them to change the wix binaries on github.
+# If WIX is updated to a newer version, this sha512 will need to be recomputed.
+RUN \
+ WIXSHA=6fd961c85e1e6adafb99ef50c9659e3eb83c84ecaf49f523e556788845b206e1857aba2c39409405d4cda1df9b30a552ce5aab808be5f8368a37a447d78d1a05 && \
+ curl -sS -L https://github.com/wixtoolset/wix3/releases/download/wix3112rtm/wix311-binaries.zip -o wix311-binaries.zip && \
+ echo "$WIXSHA wix311-binaries.zip" | sha512sum --quiet -c - && \
+ mkdir /opt/wix && \
+ unzip -q wix311-binaries.zip -d /opt/wix/ && \
+ rm wix311-binaries.zip
+RUN mv /opt/wix/candle.exe /opt/wix/real-candle.exe
+RUN mv /opt/wix/light.exe /opt/wix/real-light.exe
+COPY src/wix_wine.sh /opt/wix/candle.exe
+COPY src/wix_wine.sh /opt/wix/light.exe
+
+# We expect users to mount the project repo to /root/project, so we change that
+# to our working directory so the daffodil-build-release command is run from
+# the repo directory
+WORKDIR "/project/"
+
+# Install and set the entrypoint
+COPY src/daffodil-build-release /usr/bin/
+ENTRYPOINT ["/usr/bin/daffodil-build-release"]
diff --git a/containers/build-release/README.md b/containers/build-release/README.md
new file mode 100644
index 0000000..6176cd5
--- /dev/null
+++ b/containers/build-release/README.md
@@ -0,0 +1,46 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+## Daffodil Build Release Container
+
+Daffodil release artifacts are built using GitHub actions. This container can
+be used to build those same artifacts for testing or verifying reproducibility.
+This can be used for all Daffodil projects, including the Daffodil VS Code
+Extension and the SBT plugin.
+
+To build or update the build release container image:
+
+ podman build -t daffodil-build-release containers/build-release/
+
+To use the container image to build a release, run the following from the root
+of the project git repository, replacing `<REPO_DIR>` with the path to the git
+repository of the project to build and `<ARTIFACT_DIR>` with the directory
+where you want the artifacts saved:
+
+ podman run -it --rm \
+ --hostname daffodil.build \
+ --volume <REPO_DIR>:/project:O \
+ --volume <ARTIFACT_DIR>:/artifacts \
+ daffodil-build-release
+
+Note that the `<REPO_DIR>` volume uses the `:O` suffix so that changes to the
+repository inside the container do not affect the repository outside the
+container.
+
+When run, the container will ask for an optional pre-release label (e.g. rc1)
+and the project to build. The resulting artifacts will be written to the
+`<ARTIFACT_DIR>/release/` directory.
diff --git a/containers/build-release/src/daffodil-build-release b/containers/build-release/src/daffodil-build-release
new file mode 100755
index 0000000..22dc3e2
--- /dev/null
+++ b/containers/build-release/src/daffodil-build-release
@@ -0,0 +1,113 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+export WIX=/opt/wix/
+export LANG=en_US.UTF-8
+export CC=clang
+export AR=llvm-ar
+export SOURCE_DATE_EPOCH=$(git show --no-patch --format=%ct HEAD)
+
+echo "Select a project: "
+select PROJECT in daffodil daffodil-sbt daffodil-vscode; do
+ case $PROJECT in
+ "daffodil" | "daffodil-sbt")
+ PROJECT_VERSION=$(grep 'version :=' build.sbt | cut -d\" -f2)
+ break
+ ;;
+ "daffodil-vscode")
+ PROJECT_VERSION=$(grep '"version"' package.json | cut -d\" -f4)
+ break
+ ;;
+ *)
+ echo "unknown project: $PROJECT" >&2
+ exit 1
+ ;;
+ esac
+done
+echo
+
+read -p "Pre-release label (e.g. rc1 to rc99) or empty for final release: " PRE_RELEASE_LABEL
+echo
+
+# apppend the pre-release label if it is not empty/all whitepsace
+if [[ ! $PRE_RELEASE_LABEL =~ ^[\s]*$ ]]
+then
+ RELEASE_VERSION=$PROJECT_VERSION-$PRE_RELEASE_LABEL
+else
+ RELEASE_VERSION=$PROJECT_VERSION
+fi
+
+echo "==== Building artifacts for $PROJECT $RELEASE_VERSION ===="
+RELEASE_DIR=/artifacts/release
+DIST_DIR=$RELEASE_DIR/asf-dist/$RELEASE_VERSION
+MAVEN_DIR=$RELEASE_DIR/maven-local
+
+mkdir -p ~/.sbt/1.0
+echo "ThisBuild / publishTo := Some(MavenCache(\"maven-local\", file(\"$MAVEN_DIR\")))" >> ~/.sbt/1.0/build.sbt
+
+echo "==== Building source artifact ===="
+mkdir -p $DIST_DIR/src/
+git archive --format=zip --prefix=apache-$PROJECT-$PROJECT_VERSION-src/ HEAD > $DIST_DIR/src/apache-$PROJECT-$PROJECT_VERSION-src.zip
+
+echo "==== Building binary artifacts ===="
+
+case $PROJECT in
+ "daffodil")
+ mkdir -p $DIST_DIR/bin
+ sbt \
+ +compile \
+ +publish \
+ daffodil-cli/Rpm/packageBin \
+ daffodil-cli/Windows/packageBin \
+ daffodil-cli/Universal/packageBin \
+ daffodil-cli/Universal/packageZipTarball
+
+ cp daffodil-cli/target/universal/apache-daffodil-*.tgz $DIST_DIR/bin/
+ cp daffodil-cli/target/universal/apache-daffodil-*.zip $DIST_DIR/bin/
+ cp daffodil-cli/target/rpm/RPMS/noarch/apache-daffodil-*.rpm $DIST_DIR/bin/
+ MSI_NAME=$(basename $DIST_DIR/bin/*.zip .zip).msi
+ cp daffodil-cli/target/windows/Daffodil.msi $DIST_DIR/bin/$MSI_NAME
+ chmod -x $DIST_DIR/bin/$MSI_NAME
+ ;;
+
+ "daffodil-sbt")
+ sbt \
+ "^compile" \
+ "^publish"
+ ;;
+
+ "daffodil-vscode")
+ mkdir -p $DIST_DIR/bin/
+ yarn package
+ cp *.vsix $DIST_DIR/bin/
+ ;;
+
+esac
+
+echo "==== Calculating Checksums ===="
+
+for i in $DIST_DIR/*/
+do
+ pushd $i > /dev/null
+ for file in *
+ do
+ sha512sum --binary $file > $file.sha512
+ done
+ popd &> /dev/null
+done
diff --git a/containers/build-release/src/wix_wine.sh b/containers/build-release/src/wix_wine.sh
new file mode 100755
index 0000000..d63e6c7
--- /dev/null
+++ b/containers/build-release/src/wix_wine.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We use sbt-native-packager plugin to build a Windows MSI installer.
+# Unfortunately, that uses WiX toolset which requires Windows. It can be run
+# under Wine, but the plugin cannot be easily configured to use Wine. This
+# script is one part of getting that to work and is setup within the container
+# environment as needed.
+
+
+# We run a few wine commands, let's disable all wine debug information since it
+# is interpreted by sbt as errors and makes the output look like something
+# failed.
+export WINEDEBUG=-all
+
+# Create initial wine config, redirecting stderr to stdout. The command outputs
+# debug message to stderr, which SBT makes look like an actual error.
+winecfg 2>&1
+
+# The sbt native-packager plugin executes the $WIX/{candle,light}.exe
+# executables to build the Daffodil MSI. The problem is that those are Windows
+# executables and so can't be directly executed in the Linux container. To get
+# around this, the container Dockerfile copies the $WIX/{candle,light}.exe
+# files to $WIX/real-{candle,light}.exe and installs this script to
+# $WIX/{candle,light}.exe. This way, when the sbt plugin executes
+# $WIX/{candle,light}.exe, it's actually executing this script, which figures
+# out how it was executed (either as candle.exe or light.exe) and redirects the
+# execution to the real-{candle,light}.exe file using wine.
+
+
+# Figure out what was originally executed, and prepend "real-" to it. This is
+# what to execute with wine
+REAL_CMD=real-$(basename "$0")
+
+# The paths passed as arguments to this script by the plugin are all absolute
+# Linux style paths. For arguments that look like a path (i.e. starts with a
+# forward slash), use winepath to convert them to a Windows style path that
+# wine applications can understand. Leave all other arguments unmodified.
+i=0
+NEWARGS=()
+for VAR in "$@"
+do
+ if [[ $VAR == /* ]]
+ then
+ NEWARGS[$i]=$(winepath -w "$VAR")
+ else
+ NEWARGS[$i]=$VAR
+ fi
+ ((++i))
+done
+
+# Uncomment to tell bash to output the wine command we are about to execute,
+# helpful for debugging when something goes wrong with wine
+# set -x
+
+# Execute wine with the real WiX command and modified arguments
+wine $WIX/$REAL_CMD "${NEWARGS[@]}"
