| #!/usr/bin/env bash |
| |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| trap cleanup EXIT |
| |
| # Source in utils. |
| SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| # shellcheck source=SCRIPTDIR/release-util.sh |
| . "$SELF/release-util.sh" |
| |
| # Print usage and exit. |
| function exit_with_usage { |
| cat <<'EOF' |
| Usage: release-build.sh <tag|publish-dist|publish-snapshot|publish-release> |
| Creates release deliverables from a tag or commit. |
| Argument: one of 'tag', 'publish-dist', 'publish-snapshot', or 'publish-release' |
| tag Prepares for release on specified git branch: Set release version, |
| update CHANGES and RELEASENOTES, create release tag, |
| increment version for ongoing dev, and publish to Apache git repo. |
| publish-dist Build and publish distribution packages (tarballs) to Apache dist repo |
| publish-snapshot Build and publish maven artifacts snapshot release to Apache snapshots repo |
| publish-release Build and publish maven artifacts release to Apache release repo, and |
| construct vote email from template |
| |
| All other inputs are environment variables. Please use do-release-docker.sh or |
| do-release.sh to set up the needed environment variables. This script, release-build.sh, |
| is not intended to be called stand-alone, and such use is untested. The env variables used are: |
| |
| Used for 'tag' and 'publish' stages: |
| PROJECT - The project to build. No default. |
| RELEASE_VERSION - Version used in pom files for release (e.g. 2.1.2) |
| Required for 'tag'; defaults for 'publish' to the version in pom at GIT_REF |
| RELEASE_TAG - Name of release tag (e.g. 2.1.2RC0), also used by |
| publish-dist as package version name in dist directory path |
| ASF_USERNAME - Username of ASF committer account |
| ASF_PASSWORD - Password of ASF committer account |
| DRY_RUN - 1:true (default), 0:false. If "1", does almost all the work, but doesn't actually |
| publish anything to upstream source or object repositories. It defaults to "1", so if you want |
| to actually publish you have to set '-f' (force) flag in do-release.sh or do-release-docker.sh. |
| |
| Used only for 'tag': |
| YETUS_HOME - installation location for Apache Yetus |
| GIT_NAME - Name to use with git |
| GIT_EMAIL - E-mail address to use with git |
| GIT_BRANCH - Git branch on which to make release. Tag is always placed at HEAD of this branch. |
| NEXT_VERSION - Development version after release (e.g. 2.1.3-SNAPSHOT) |
| |
| Used only for 'publish': |
| GIT_REF - Release tag or commit to build from (defaults to $RELEASE_TAG; only need to |
| separately define GIT_REF if RELEASE_TAG is not actually present as a tag at publish time) |
| If both RELEASE_TAG and GIT_REF are undefined it will default to HEAD of master. |
| GPG_KEY - GPG key id (usually email addr) used to sign release artifacts |
| REPO - Set to full path of a directory to use as maven local repo (dependencies cache) |
| to avoid re-downloading dependencies for each stage. It is automatically set if you |
| request full sequence of stages (tag, publish-dist, publish-release) in do-release.sh. |
| |
| For example: |
| $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh publish-dist |
| EOF |
| exit 1 |
| } |
| |
| set -e |
| |
| function cleanup { |
| # If REPO was set, then leave things be. Otherwise if we defined a repo clean it out. |
| if [[ -z "${REPO}" ]] && [[ -n "${MAVEN_LOCAL_REPO}" ]]; then |
| log "Cleaning up temp repo in '${MAVEN_LOCAL_REPO}'. Set REPO to reuse downloads." >&2 |
| rm -f "${MAVEN_SETTINGS_FILE}" &> /dev/null || true |
| rm -rf "${MAVEN_LOCAL_REPO}" &> /dev/null || true |
| fi |
| } |
| |
| if [ $# -ne 1 ]; then |
| exit_with_usage |
| fi |
| |
| if [[ "$1" == "-h" ]]; then |
| exit_with_usage |
| fi |
| |
| if [[ "$*" == *"help"* ]]; then |
| exit_with_usage |
| fi |
| |
| init_locale |
| init_java |
| init_mvn |
| init_python |
| # Print out subset of perl version (used in git hooks and japi-compliance-checker) |
| perl --version | grep 'This is' |
| |
| rm -rf "${PROJECT}" |
| |
| if is_debug; then |
| set -x # detailed logging during action |
| fi |
| |
| if [[ "$1" == "tag" ]]; then |
| init_yetus |
| # for 'tag' stage |
| set -o pipefail |
| check_get_passwords ASF_PASSWORD |
| check_needed_vars PROJECT RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH |
| if [ -z "${GIT_REPO}" ]; then |
| check_needed_vars ASF_USERNAME ASF_PASSWORD |
| fi |
| git_clone_overwrite |
| |
| # 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified |
| # $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*' |
| # projects share the same HBASE jira name. To make this work, by convention, the HBASE jira "Fix Version" |
| # field values have the sub-project name pre-pended, as in "hbase-operator-tools-1.0.0". |
| # So, here we prepend the project name to the version, but only for the hbase sub-projects. |
| jira_fix_version="${RELEASE_VERSION}" |
| shopt -s nocasematch |
| if [[ "${PROJECT}" =~ ^hbase- ]]; then |
| jira_fix_version="${PROJECT}-${RELEASE_VERSION}" |
| fi |
| shopt -u nocasematch |
| update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}" |
| |
| cd "${PROJECT}" |
| |
| git config user.name "$GIT_NAME" |
| git config user.email "$GIT_EMAIL" |
| git config user.signingkey "${GPG_KEY}" |
| |
| # Create release version |
| maven_set_version "$RELEASE_VERSION" |
| find . -name pom.xml -exec git add {} \; |
| git add RELEASENOTES.md CHANGES.md |
| |
| git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" |
| log "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH" |
| git tag -s -m "Via create-release" "$RELEASE_TAG" |
| |
| # Create next version |
| maven_set_version "$NEXT_VERSION" |
| find . -name pom.xml -exec git add {} \; |
| git commit -s -m "Preparing development version $NEXT_VERSION" |
| |
| if ! is_dry_run; then |
| # Push changes |
| git push origin "$RELEASE_TAG" |
| git push origin "HEAD:$GIT_BRANCH" |
| cd .. |
| rm -rf "${PROJECT}" |
| else |
| cd .. |
| mv "${PROJECT}" "${PROJECT}.tag" |
| log "Dry run: Clone with version changes and tag available as ${PROJECT}.tag in the output directory." |
| fi |
| exit 0 |
| fi |
| |
| ### Below is for 'publish-*' stages ### |
| check_get_passwords ASF_PASSWORD |
| check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD GPG_KEY |
| |
| # Commit ref to checkout when building |
| BASE_DIR=$(pwd) |
| GIT_REF=${GIT_REF:-master} |
| if [[ "$PROJECT" =~ ^hbase ]]; then |
| RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/hbase" |
| else |
| RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/${PROJECT}" |
| fi |
| |
| # in case of dry run, enable publish steps to chain from tag step |
| if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then |
| ln -s "${PROJECT}.tag" "${PROJECT}" |
| else |
| git_clone_overwrite |
| fi |
| cd "${PROJECT}" |
| git checkout "$GIT_REF" |
| git_hash="$(git rev-parse --short HEAD)" |
| GIT_LONG_HASH="$(git rev-parse HEAD)" |
| log "Checked out ${PROJECT} at ${GIT_REF} commit $git_hash" |
| |
| if [ -z "${RELEASE_VERSION}" ]; then |
| RELEASE_VERSION="$(maven_get_version)" |
| fi |
| |
| # This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins |
| # machines by explicitly calling /usr/sbin/lsof. Please see SPARK-22377 and the discussion |
| # in its pull request. |
| LSOF=lsof |
| if ! hash $LSOF 2>/dev/null; then |
| LSOF=/usr/sbin/lsof |
| fi |
| |
| package_version_name="$RELEASE_TAG" |
| if [ -z "$package_version_name" ]; then |
| package_version_name="${RELEASE_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" |
| fi |
| |
| git clean -d -f -x |
| cd .. |
| if [[ "$PROJECT" =~ ^hbase- ]]; then |
| DEST_DIR_NAME="${PROJECT}-${package_version_name}" |
| else |
| DEST_DIR_NAME="$package_version_name" |
| fi |
| |
| if [[ "$1" == "publish-dist" ]]; then |
| # Source and binary tarballs |
| log "Packaging release source tarballs" |
| make_src_release "${PROJECT}" "${RELEASE_VERSION}" |
| |
| # we do not have binary tarballs for hbase-thirdparty |
| if [[ "${PROJECT}" != "hbase-thirdparty" ]]; then |
| make_binary_release "${PROJECT}" "${RELEASE_VERSION}" |
| fi |
| |
| svn_target="svn-${PROJECT}" |
| svn co --depth=empty "$RELEASE_STAGING_LOCATION" "$svn_target" |
| rm -rf "${svn_target:?}/${DEST_DIR_NAME}" |
| mkdir -p "$svn_target/${DEST_DIR_NAME}" |
| |
| log "Copying release tarballs" |
| cp "${PROJECT}"-*.tar.* "$svn_target/${DEST_DIR_NAME}/" |
| cp "${PROJECT}/CHANGES.md" "$svn_target/${DEST_DIR_NAME}/" |
| cp "${PROJECT}/RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/" |
| shopt -s nocasematch |
| # Generate api report only if project is hbase for now. |
| if [ "${PROJECT}" == "hbase" ]; then |
| # This script usually reports an errcode along w/ the report. |
| generate_api_report "./${PROJECT}" "${API_DIFF_TAG}" "${GIT_REF}" || true |
| cp api*.html "$svn_target/${DEST_DIR_NAME}/" |
| fi |
| shopt -u nocasematch |
| |
| log "svn add" |
| svn add "$svn_target/${DEST_DIR_NAME}" |
| |
| if ! is_dry_run; then |
| cd "$svn_target" |
| svn ci --username "$ASF_USERNAME" --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $package_version_name" --no-auth-cache |
| cd .. |
| rm -rf "$svn_target" |
| else |
| mv "$svn_target/${DEST_DIR_NAME}" "${svn_target}_${DEST_DIR_NAME}.dist" |
| log "Dry run: svn-managed 'dist' directory with release tarballs, CHANGES.md and RELEASENOTES.md available as $(pwd)/${svn_target}_${DEST_DIR_NAME}.dist" |
| rm -rf "$svn_target" |
| fi |
| log "svn ci done" |
| |
| exit 0 |
| fi |
| |
| if [[ "$1" == "publish-snapshot" ]]; then |
| ( |
| cd "${PROJECT}" |
| mvn_log="${BASE_DIR}/mvn_deploy_snapshot.log" |
| log "Publishing snapshot to nexus" |
| maven_deploy snapshot "$mvn_log" |
| if ! is_dry_run; then |
| log "Snapshot artifacts successfully published to repo." |
| rm "$mvn_log" |
| else |
| log "Dry run: Snapshot artifacts successfully built, but not published due to dry run." |
| fi |
| ) |
| exit $? |
| fi |
| |
| if [[ "$1" == "publish-release" ]]; then |
| ( |
| cd "${PROJECT}" |
| mvn_log="${BASE_DIR}/mvn_deploy_release.log" |
| log "Staging release in nexus" |
| maven_deploy release "$mvn_log" |
| declare staged_repo_id="dryrun-no-repo" |
| if ! is_dry_run; then |
| staged_repo_id=$(grep -o "Closing staging repository with ID .*" "$mvn_log" \ |
| | sed -e 's/Closing staging repository with ID "\([^"]*\)"./\1/') |
| log "Release artifacts successfully published to repo ${staged_repo_id}" |
| rm "$mvn_log" |
| else |
| log "Dry run: Release artifacts successfully built, but not published due to dry run." |
| fi |
| # Dump out email to send. Where we find vote.tmpl depends |
| # on where this script is run from |
| PROJECT_TEXT="${PROJECT//-/ }" #substitute like 's/-/ /g' |
| export PROJECT_TEXT |
| eval "echo \"$(< "${SELF}/vote.tmpl")\"" |tee "${BASE_DIR}/vote.txt" |
| ) |
| exit $? |
| fi |
| |
| set +x # done with detailed logging |
| cd .. |
| rm -rf "${PROJECT}" |
| log "ERROR: expects to be called with 'tag', 'publish-dist', 'publish-release', or 'publish-snapshot'" >&2 |
| exit_with_usage |