chore(release): add ASF release scripts for source tarballs (#76)
diff --git a/dev/release/README.md b/dev/release/README.md
new file mode 100644
index 0000000..16c985d
--- /dev/null
+++ b/dev/release/README.md
@@ -0,0 +1,160 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Release Process
+
+This guide is for maintainers creating release candidates of Apache
+DataFusion Java and running the release process.
+
+## Release Prerequisites
+
+### Add a git remote for the `apache` repo
+
+The instructions below assume the upstream git repo
+`git@github.com:apache/datafusion-java.git` is configured as the remote
+named `apache`:
+
+```shell
+git remote add apache git@github.com:apache/datafusion-java.git
+```
+
+### Add your GPG public key to the SVN `KEYS` file
+
+If you will be releasing the final tarball, your GPG public key must be
+present in:
+
+- https://dist.apache.org/repos/dist/dev/datafusion/KEYS
+- https://dist.apache.org/repos/dist/release/datafusion/KEYS
+
+See https://infra.apache.org/release-signing.html#generate for generating
+keys.
+
+Committers can add signing keys via Subversion using their ASF account:
+
+```shell
+svn co https://dist.apache.org/repos/dist/dev/datafusion
+cd datafusion
+(gpg --list-sigs "Your Name" && gpg --armor --export "Your Name") >> KEYS
+svn commit KEYS -m "Add key for Your Name"
+```
+
+## Release Process: Step by Step
+
+As part of the Apache governance model, official releases consist of
+signed source tarballs approved by the PMC.
+
+### 1. Pick a Release Candidate (RC) number
+
+Pick numbers in sequential order, with `1` for `rc1`, `2` for `rc2`, and
+so on.
+
+### 2. Create the Git tag for the Release Candidate
+
+Release tags look like `0.1.0`, and release candidate tags look like
+`0.1.0-rc1`.
+
+```shell
+git fetch apache
+git tag 0.1.0-rc1 apache/main
+git push apache 0.1.0-rc1
+```
+
+### 3. Create, sign, and upload the tarball
+
+Run `create-tarball.sh` with the version and RC number:
+
+```shell
+./dev/release/create-tarball.sh 0.1.0 1
+```
+
+The script:
+
+1. Creates and uploads the release candidate artifacts to the
+ [`dist/dev/datafusion`](https://dist.apache.org/repos/dist/dev/datafusion)
+ location on the Apache SVN distribution server, under
+ `apache-datafusion-java-<version>-rc<rc>/`.
+2. Prints an email template to send to `dev@datafusion.apache.org` for
+ the release vote.
+
+### 4. Vote on the Release Candidate
+
+Send the email printed by the script to `dev@datafusion.apache.org`.
+
+The vote must remain open for at least 72 hours and requires at least
+three PMC `+1` votes and no `-1` votes to pass.
+
+### Verifying Release Candidates
+
+`dev/release/verify-release-candidate.sh` downloads the candidate
+tarball, verifies its signature and checksums, and builds and tests the
+source distribution:
+
+```shell
+./dev/release/verify-release-candidate.sh 0.1.0 1
+```
+
+The script requires `curl`, `git`, `gpg`, a C toolchain, `java`, and
+either `shasum` or `sha256sum`/`sha512sum` to be on the `PATH`. It will
+install a temporary Rust toolchain into the sandbox directory.
+
+### 5. If the Vote Passes: Promote the Tarball
+
+Move artifacts from the `dev` SVN area to the `release` area using
+`release-tarball.sh`:
+
+```shell
+./dev/release/release-tarball.sh 0.1.0 1
+```
+
+### 6. Create the Release Git Tag
+
+Tag the same release candidate commit with the final release tag:
+
+```shell
+git checkout 0.1.0-rc1
+git tag 0.1.0
+git push apache 0.1.0
+```
+
+### 7. Add the Release to Apache Reporter
+
+Add the release at https://reporter.apache.org/addrelease.html?datafusion
+so it appears in the project's board report.
+
+### 8. Delete Old RCs and Releases
+
+See the ASF documentation on [when to
+archive](https://www.apache.org/legal/release-policy.html#when-to-archive).
+
+Release candidates should be deleted once the release is published:
+
+```shell
+svn ls https://dist.apache.org/repos/dist/dev/datafusion | grep datafusion-java
+svn delete -m "delete old DataFusion Java RC" \
+ https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-java-0.1.0-rc1/
+```
+
+Only the latest release should remain in the `release` SVN area. Delete
+older releases after publishing a new one:
+
+```shell
+svn ls https://dist.apache.org/repos/dist/release/datafusion | grep datafusion-java
+svn delete -m "delete old DataFusion Java release" \
+ https://dist.apache.org/repos/dist/release/datafusion/datafusion-java-0.1.0
+```
diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py
new file mode 100755
index 0000000..a7e0a99
--- /dev/null
+++ b/dev/release/check-rat-report.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import fnmatch
+import re
+import sys
+import xml.etree.ElementTree as ET
+
+if len(sys.argv) != 3:
+ sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" %
+ sys.argv[0])
+ sys.exit(1)
+
+exclude_globs_filename = sys.argv[1]
+xml_filename = sys.argv[2]
+
+globs = [line.strip() for line in open(exclude_globs_filename, "r")]
+
+tree = ET.parse(xml_filename)
+root = tree.getroot()
+resources = root.findall('resource')
+
+all_ok = True
+for r in resources:
+ approvals = r.findall('license-approval')
+ if not approvals or approvals[0].attrib['name'] == 'true':
+ continue
+ clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
+ excluded = False
+ for g in globs:
+ if fnmatch.fnmatch(clean_name, g):
+ excluded = True
+ break
+ if not excluded:
+ sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % (
+ clean_name, r.attrib['name'], approvals[0].attrib['name']))
+ all_ok = False
+
+if not all_ok:
+ sys.exit(1)
+
+print('OK')
+sys.exit(0)
diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh
new file mode 100755
index 0000000..f227197
--- /dev/null
+++ b/dev/release/create-tarball.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Adapted from https://github.com/apache/datafusion/blob/main/dev/release/create-tarball.sh
+
+# This script creates a signed tarball in
+# dev/dist/apache-datafusion-java-<version>-rc<rc>/apache-datafusion-java-<version>.tar.gz
+# and uploads it to the "dev" area of the dist.apache.datafusion repository,
+# then prints an email template for the dev@datafusion.apache.org list for a
+# formal vote.
+#
+# See dev/release/README.md for full release instructions.
+#
+# Requirements:
+#
+# 1. gpg setup for signing and have uploaded your public
+# signature to https://pgp.mit.edu/ and the project KEYS file.
+#
+# 2. Logged into the Apache SVN server with the appropriate
+# credentials.
+#
+# Based in part on 02-source.sh from apache/arrow.
+
+set -e
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
+
+if [ "$#" -ne 2 ]; then
+ echo "Usage: $0 <version> <rc>"
+ echo "ex. $0 0.1.0 1"
+ exit
+fi
+
+version=$1
+rc=$2
+tag="${version}-rc${rc}"
+
+release=apache-datafusion-java-${version}
+distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc}
+tarname=${release}.tar.gz
+tarball=${distdir}/${tarname}
+url="https://dist.apache.org/repos/dist/dev/datafusion/${release}-rc${rc}"
+
+echo "Attempting to create ${tarball} from tag ${tag}"
+release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag})
+
+if [ -z "$release_hash" ]; then
+ echo "Cannot continue: unknown git tag: ${tag}"
+ exit 1
+fi
+
+echo "Draft email for dev@datafusion.apache.org mailing list"
+echo ""
+echo "---------------------------------------------------------"
+cat <<MAIL
+To: dev@datafusion.apache.org
+Subject: [VOTE] Release Apache DataFusion Java ${version} RC${rc}
+Hi,
+
+I would like to propose a release of Apache DataFusion Java version ${version}.
+
+This release candidate is based on commit: ${release_hash} [1]
+The proposed release tarball and signatures are hosted at [2].
+The changelog is located at [3].
+
+Please download, verify checksums and signatures, run the unit tests, and vote
+on the release. The vote will be open for at least 72 hours.
+
+Only votes from PMC members are binding, but all members of the community are
+encouraged to test the release and vote with "(non-binding)".
+
+The standard verification procedure is documented at
+https://github.com/apache/datafusion-java/blob/main/dev/release/README.md#verifying-release-candidates.
+
+[ ] +1 Release this as Apache DataFusion Java ${version}
+[ ] +0
+[ ] -1 Do not release this as Apache DataFusion Java ${version} because...
+
+Here is my vote:
+
++1
+
+[1]: https://github.com/apache/datafusion-java/tree/${release_hash}
+[2]: ${url}
+[3]: https://github.com/apache/datafusion-java/blob/${release_hash}/dev/changelog/${version}.md
+MAIL
+echo "---------------------------------------------------------"
+
+
+# create <tarball> containing the files in git at $release_hash
+# the files in the tarball are prefixed with apache-datafusion-java-<version>/
+mkdir -p ${distdir}
+(cd "${SOURCE_TOP_DIR}" && git archive ${release_hash} --prefix ${release}/ | gzip > ${tarball})
+
+echo "Signing tarball and creating checksums"
+gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
+# create checksums with relative path of tarball so that they can be verified
+# with a command such as
+# shasum --check apache-datafusion-java-0.1.0.tar.gz.sha512
+(cd ${distdir} && shasum -a 256 ${tarname}) > ${tarball}.sha256
+(cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512
+
+
+echo "Uploading to datafusion dist/dev to ${url}"
+svn co --depth=empty https://dist.apache.org/repos/dist/dev/datafusion ${SOURCE_TOP_DIR}/dev/dist
+svn add ${distdir}
+svn ci -m "Apache DataFusion Java ${version} RC${rc}" ${distdir}
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
new file mode 100644
index 0000000..81d83e8
--- /dev/null
+++ b/dev/release/rat_exclude_files.txt
@@ -0,0 +1,17 @@
+*.gitignore
+*.dockerignore
+.github/pull_request_template.md
+.github/ISSUE_TEMPLATE/bug_report.yml
+.github/ISSUE_TEMPLATE/feature_request.yml
+.gitmodules
+.mvn/wrapper/maven-wrapper.properties
+mvnw
+mvnw.cmd
+native/Cargo.lock
+dev/release/rat_exclude_files.txt
+docs/source/_static/**
+docs/source/conf.py
+rust-toolchain
+target/*
+**/target/*
+tpch-data/*
diff --git a/dev/release/release-tarball.sh b/dev/release/release-tarball.sh
new file mode 100755
index 0000000..de266f6
--- /dev/null
+++ b/dev/release/release-tarball.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Adapted from https://github.com/apache/datafusion/blob/main/dev/release/release-tarball.sh
+
+# This script copies a tarball from the "dev" area of the
+# dist.apache.datafusion repository to the "release" area.
+#
+# It should only be run after the release has been approved by the Apache
+# DataFusion PMC.
+#
+# See dev/release/README.md for full release instructions.
+#
+# Based in part on post-01-upload.sh from apache/arrow.
+
+set -e
+set -u
+
+if [ "$#" -ne 2 ]; then
+ echo "Usage: $0 <version> <rc-num>"
+ echo "ex. $0 0.1.0 1"
+ exit
+fi
+
+version=$1
+rc=$2
+
+read -r -p "Proceed to release tarball for ${version}-rc${rc}? [y/N]: " answer
+answer=${answer:-no}
+if [ "${answer}" != "y" ]; then
+ echo "Cancelled tarball release!"
+ exit 1
+fi
+
+tmp_dir=tmp-apache-datafusion-java-dist
+
+echo "Recreate temporary directory: ${tmp_dir}"
+rm -rf ${tmp_dir}
+mkdir -p ${tmp_dir}
+
+echo "Clone dev dist repository"
+svn \
+ co \
+ https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-java-${version}-rc${rc} \
+ ${tmp_dir}/dev
+
+echo "Clone release dist repository"
+svn co https://dist.apache.org/repos/dist/release/datafusion ${tmp_dir}/release
+
+echo "Copy ${version}-rc${rc} to release working copy"
+release_version=datafusion-java-${version}
+mkdir -p ${tmp_dir}/release/${release_version}
+cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/
+svn add ${tmp_dir}/release/${release_version}
+
+echo "Commit release"
+svn ci -m "Apache DataFusion Java ${version}" ${tmp_dir}/release
+
+echo "Clean up"
+rm -rf ${tmp_dir}
+
+echo "Success! The release is available here:"
+echo " https://dist.apache.org/repos/dist/release/datafusion/${release_version}"
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
new file mode 100755
index 0000000..e486adc
--- /dev/null
+++ b/dev/release/verify-release-candidate.sh
@@ -0,0 +1,175 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Adapted from https://github.com/apache/datafusion/blob/main/dev/release/verify-release-candidate.sh
+
+# Check that required dependencies are installed
+check_dependencies() {
+ local missing_deps=0
+ local required_deps=("curl" "git" "gpg" "cc" "java")
+
+ # Either shasum or sha256sum/sha512sum are required
+ local has_sha_tools=0
+
+ for dep in "${required_deps[@]}"; do
+ if ! command -v $dep &> /dev/null; then
+ echo "Error: $dep is not installed or not in PATH"
+ missing_deps=1
+ fi
+ done
+
+ if command -v shasum &> /dev/null; then
+ has_sha_tools=1
+ elif command -v sha256sum &> /dev/null && command -v sha512sum &> /dev/null; then
+ has_sha_tools=1
+ else
+ echo "Error: Neither shasum nor sha256sum/sha512sum are installed or in PATH"
+ missing_deps=1
+ fi
+
+ if [ $missing_deps -ne 0 ]; then
+ echo "Please install missing dependencies and try again"
+ exit 1
+ fi
+}
+
+case $# in
+ 2) VERSION="$1"
+ RC_NUMBER="$2"
+ ;;
+ *) echo "Usage: $0 X.Y.Z RC_NUMBER"
+ exit 1
+ ;;
+esac
+
+set -e
+set -x
+set -o pipefail
+
+check_dependencies
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+DATAFUSION_JAVA_DIST_URL='https://dist.apache.org/repos/dist/dev/datafusion'
+
+download_dist_file() {
+ curl \
+ --silent \
+ --show-error \
+ --fail \
+ --location \
+ --remote-name $DATAFUSION_JAVA_DIST_URL/$1
+}
+
+download_rc_file() {
+ download_dist_file apache-datafusion-java-${VERSION}-rc${RC_NUMBER}/$1
+}
+
+import_gpg_keys() {
+ download_dist_file KEYS
+ gpg --import KEYS
+}
+
+if type shasum >/dev/null 2>&1; then
+ sha256_verify="shasum -a 256 -c"
+ sha512_verify="shasum -a 512 -c"
+else
+ sha256_verify="sha256sum -c"
+ sha512_verify="sha512sum -c"
+fi
+
+fetch_archive() {
+ local dist_name=$1
+ download_rc_file ${dist_name}.tar.gz
+ download_rc_file ${dist_name}.tar.gz.asc
+ download_rc_file ${dist_name}.tar.gz.sha256
+ download_rc_file ${dist_name}.tar.gz.sha512
+ verify_dir_artifact_signatures
+}
+
+verify_dir_artifact_signatures() {
+ # verify the signature and the checksums of each artifact
+ find . -name '*.asc' | while read sigfile; do
+ artifact=${sigfile/.asc/}
+ gpg --verify $sigfile $artifact || exit 1
+
+ # go into the directory because the checksum files contain only the
+ # basename of the artifact
+ pushd $(dirname $artifact)
+ base_artifact=$(basename $artifact)
+ ${sha256_verify} $base_artifact.sha256 || exit 1
+ ${sha512_verify} $base_artifact.sha512 || exit 1
+ popd
+ done
+}
+
+setup_tempdir() {
+ cleanup() {
+ if [ "${TEST_SUCCESS}" = "yes" ]; then
+ rm -fr "${DATAFUSION_JAVA_TMPDIR}"
+ else
+ echo "Failed to verify release candidate. See ${DATAFUSION_JAVA_TMPDIR} for details."
+ fi
+ }
+
+ if [ -z "${DATAFUSION_JAVA_TMPDIR:-}" ]; then
+ # clean up automatically if DATAFUSION_JAVA_TMPDIR is not defined
+ DATAFUSION_JAVA_TMPDIR=$(mktemp -d -t "$1.XXXXX")
+ trap cleanup EXIT
+ else
+ # don't clean up automatically
+ mkdir -p "${DATAFUSION_JAVA_TMPDIR}"
+ fi
+}
+
+test_source_distribution() {
+ # install rust toolchain in a similar fashion like test-miniconda
+ export RUSTUP_HOME=$PWD/test-rustup
+ export CARGO_HOME=$PWD/test-rustup
+
+ curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
+
+ export PATH=$RUSTUP_HOME/bin:$PATH
+ source $RUSTUP_HOME/env
+
+ # raises on any formatting errors
+ rustup component add rustfmt
+ (cd native && cargo fmt --all -- --check)
+
+ # build native + JVM and run the full test suite
+ make test
+}
+
+TEST_SUCCESS=no
+
+setup_tempdir "datafusion-java-${VERSION}"
+echo "Working in sandbox ${DATAFUSION_JAVA_TMPDIR}"
+cd ${DATAFUSION_JAVA_TMPDIR}
+
+dist_name="apache-datafusion-java-${VERSION}"
+import_gpg_keys
+fetch_archive ${dist_name}
+tar xf ${dist_name}.tar.gz
+pushd ${dist_name}
+ test_source_distribution
+popd
+
+TEST_SUCCESS=yes
+echo 'Release candidate looks good!'
+exit 0
