| #!/bin/bash |
| |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| # fail immediately |
| set -o errexit |
| set -o nounset |
| # print command before executing |
| #set -o xtrace |
| |
| CURR_DIR=`pwd` |
| if [[ `basename $CURR_DIR` != "scripts" ]] ; then |
| echo "You have to call the script from the scripts/ dir" |
| exit 1 |
| fi |
| |
| REDIRECT=' > /dev/null 2>&1' |
| if [[ $# -lt 1 ]]; then |
| echo "This script will validate source release candidate published in dist for apache hudi" |
| echo "You can set 3 args to this script: release version is mandatory, while rest two are optional. Default release_type is \"dev\". Or you can set to \"release\"" |
| echo "--release=\${CURRENT_RELEASE_VERSION}" |
| echo "--rc_num=\${RC_NUM}" |
| echo "--release_type=release" |
| exit |
| else |
| for param in "$@" |
| do |
| if [[ $param =~ --release\=([0-9]\.[0-9]*\.[0-9]) ]]; then |
| RELEASE_VERSION=${BASH_REMATCH[1]} |
| fi |
| if [[ $param =~ --rc_num\=([0-9]*) ]]; then |
| RC_NUM=${BASH_REMATCH[1]} |
| fi |
| if [[ $param =~ --release_type\="release" ]]; then |
| RELEASE_TYPE="release" |
| fi |
| if [[ $param =~ --verbose ]]; then |
| REDIRECT="" |
| fi |
| done |
| fi |
| |
| if [ "$(uname)" == "Darwin" ]; then |
| SHASUM="shasum -a 512" |
| else |
| SHASUM="sha512sum" |
| fi |
| |
| if [ -z ${RC_NUM+x} ]; then |
| RC_NUM=-1 |
| fi |
| |
| if [ -z ${RELEASE_TYPE+x} ]; then |
| RELEASE_TYPE=dev |
| fi |
| |
| # Get to a scratch dir |
| RELEASE_TOOL_DIR=`pwd` |
| WORK_DIR=/tmp/validation_scratch_dir_001 |
| rm -rf $WORK_DIR |
| mkdir $WORK_DIR |
| pushd $WORK_DIR |
| |
| # Checkout dist repo |
| LOCAL_SVN_DIR=local_svn_dir |
| ROOT_SVN_URL=https://dist.apache.org/repos/dist/ |
| REPO_TYPE=${RELEASE_TYPE} |
| #RELEASE_REPO=release |
| HUDI_REPO=hudi |
| |
| if [ $RC_NUM == -1 ]; then |
| ARTIFACT_SUFFIX=${RELEASE_VERSION} |
| else |
| ARTIFACT_SUFFIX=${RELEASE_VERSION}-rc${RC_NUM} |
| fi |
| |
| |
| rm -rf $LOCAL_SVN_DIR |
| mkdir $LOCAL_SVN_DIR |
| cd $LOCAL_SVN_DIR |
| |
| echo "Downloading from svn co ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO}" |
| |
| (bash -c "svn co ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO} $REDIRECT") || (echo -e "\t\t Unable to checkout ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO} to $REDIRECT. Please run with --verbose to get details\n" && exit -1) |
| |
| echo "Validating hudi-${ARTIFACT_SUFFIX} with release type \"${REPO_TYPE}\"" |
| cd ${HUDI_REPO}/hudi-${ARTIFACT_SUFFIX} |
| $SHASUM hudi-${ARTIFACT_SUFFIX}.src.tgz > got.sha512 |
| |
| echo "Checking Checksum of Source Release" |
| diff -u hudi-${ARTIFACT_SUFFIX}.src.tgz.sha512 got.sha512 |
| echo -e "\t\tChecksum Check of Source Release - [OK]\n" |
| |
| # Download KEYS file |
| curl https://dist.apache.org/repos/dist/release/hudi/KEYS > ../KEYS |
| |
| # GPG Check |
| echo "Checking Signature" |
| (bash -c "gpg --import ../KEYS $REDIRECT" && bash -c "gpg --verify hudi-${ARTIFACT_SUFFIX}.src.tgz.asc hudi-${ARTIFACT_SUFFIX}.src.tgz $REDIRECT" && echo -e "\t\tSignature Check - [OK]\n") || (echo -e "\t\tSignature Check - [FAILED] - Run with --verbose to get details\n" && exit -1) |
| |
| # Untar |
| (bash -c "tar -zxf hudi-${ARTIFACT_SUFFIX}.src.tgz $REDIRECT") || (echo -e "\t\t Unable to untar hudi-${ARTIFACT_SUFFIX}.src.tgz . Please run with --verbose to get details\n" && exit -1) |
| cd hudi-${ARTIFACT_SUFFIX} |
| |
| ### BEGIN: Binary Files Check |
| echo "Checking for binary files in source release" |
| numBinaryFiles=`find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -va 'application/json' | grep -va 'text/' | grep -va 'application/xml' | grep -va 'application/json' | wc -l | sed -e s'/ //g'` |
| |
| if [ "$numBinaryFiles" -gt "0" ]; then |
| echo -e "There were non-text files in source release. Please check below\n" |
| find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -va 'application/json' | grep -va 'text/' | grep -va 'application/xml' |
| exit -1 |
| fi |
| echo -e "\t\tNo Binary Files in Source Release? - [OK]\n" |
| ### END: Binary Files Check |
| |
| ### Checking for DISCLAIMER |
| echo "Checking for DISCLAIMER" |
| disclaimerFile="./DISCLAIMER" |
| if [ -f "$disclaimerFile" ]; then |
| echo "DISCLAIMER file should not be present " |
| exit -1 |
| fi |
| echo -e "\t\tDISCLAIMER file exists ? [OK]\n" |
| |
| ### Checking for LICENSE and NOTICE |
| echo "Checking for LICENSE and NOTICE" |
| licenseFile="./LICENSE" |
| noticeFile="./NOTICE" |
| if [ ! -f "$licenseFile" ]; then |
| echo "License file missing" |
| exit -1 |
| fi |
| echo -e "\t\tLicense file exists ? [OK]" |
| |
| if [ ! -f "$noticeFile" ]; then |
| echo "Notice file missing" |
| exit -1 |
| fi |
| echo -e "\t\tNotice file exists ? [OK]\n" |
| |
| ### Licensing Check |
| echo "Performing custom Licensing Check " |
| numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data'| grep -v '.commit' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l` |
| if [ "$numfilesWithNoLicense" -gt "0" ]; then |
| echo "There were some source files that did not have Apache License" |
| find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" |
| exit -1 |
| fi |
| echo -e "\t\tLicensing Check Passed [OK]\n" |
| |
| ### Checking for RAT |
| echo "Running RAT Check" |
| (bash -c "mvn apache-rat:check $REDIRECT") || (echo -e "\t\t Rat Check Failed. Please run with --verbose to get details\n" && exit -1) |
| echo -e "\t\tRAT Check Passed [OK]\n" |
| |
| popd |