ci/lib/templates.lib.txt

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

(@ load("@ytt:data", "data") -@)
(@ load("functions.lib.yml",
        "gci_label_value",
        "image_family_name")
-@)

(@ def google_variables(): -@)
GOOGLE_PROJECT=(@=data.values.google.project @)
GOOGLE_ZONE=(@=data.values.google.zone @)
(@- end @)


(@ def instance_variables(): -@)
INSTANCE_USER=build
INSTANCE_DIR=$(pwd)/instance

if [ ! -d "${INSTANCE_DIR}" ]; then
  echo "${INSTANCE_DIR} not found."
  exit 1
fi
(@- end @)

(@ def remote_functions(): -@)
(@=instance_variables() @)

SSH_OPTIONS=${SSH_OPTIONS:-"-o StrictHostKeyChecking=no -o PasswordAuthentication=no"}

ssh_key_file=${INSTANCE_DIR}/identity
if [ ! -r "${ssh_key_file}" ]; then
  echo "${ssh_key_file} not readable."
  exit 1
fi

instance_file=${INSTANCE_DIR}/instance.sh
if [ ! -r "${instance_file}" ]; then
  echo "${instance_file} not readable."
  exit 1
fi

external_ip=$(source ${instance_file} && echo -n ${networkInterfaces_accessConfigs_natIP})

function remote_shell {
  ssh ${SSH_OPTIONS} -i ${ssh_key_file} ${INSTANCE_USER}@${external_ip} "$@"
}

function remote_download {
  scp ${SSH_OPTIONS} -i ${ssh_key_file} -q -r "${INSTANCE_USER}@${external_ip}:${1}" "$2"
}

function remote_download_directory {
  ssh ${SSH_OPTIONS} -i ${ssh_key_file} ${INSTANCE_USER}@${external_ip} tar -C $(dirname ${1}) -czf - $(basename ${1}) | tar -C ${2} -zxf -
}

function remote_upload {
  scp ${SSH_OPTIONS} -i ${ssh_key_file} -q -r "$1" "${INSTANCE_USER}@${external_ip}:${2}"
}

(@- end @)

(@ def ctest_bash_task(path, timeout=300, parallel=6): -@)
set -ueo pipefail
(@= remote_functions() @)
remote_shell cmake -E chdir (@= path @) ctest -C ${CMAKE_CONFIG} -j(@= str(parallel) @) --timeout=(@= str(timeout) @) --output-on-failure --repeat until-pass:4 --schedule-random
(@- end @)

(@ def clang_tidy_bash_task(): -@)
set -ueo pipefail
export GEODE_HOME=$(pwd)/geode
log=$(mktemp)
cd build
cmake ../source -DCMAKE_EXPORT_COMPILE_COMMANDS=ON >>${log} 2>&1 || (cat ${log}; exit 1)
cmake --build dependencies --parallel $(nproc) >>${log} 2>&1 || (cat ${log}; exit 1)
run-clang-tidy -j $(nproc) -quiet
(@- end @)

(@ def clang_format_bash_task(): -@)
set -ueo pipefail
export GEODE_HOME=$(pwd)/geode
log=$(mktemp)
cd build
cmake ../source -DCMAKE_EXPORT_COMPILE_COMMANDS=ON >>${log} 2>&1 || (cat ${log}; exit 1)
jq -r '.[].file' compile_commands.json | sort | uniq | grep -v $(pwd) | xargs clang-format --dry-run -Werror
(@- end @)

(@ def rat_check_bash_task(): -@)
set -ueo pipefail
export GEODE_HOME=$(pwd)/geode
log=$(mktemp)
cd build
cmake ../source -DUSE_RAT=ON >>${log} 2>&1 || (cat ${log}; exit 1)
cmake --build . --target rat-check
(@- end @)

(@ def packer_build_bash_task(build): -@)
set -ueo pipefail
cd source/packer
packer build -only=googlecompute \
    -var-file=default.json \
    -var pipeline=(@= gci_label_value(data.values.pipeline.name) @) \
    -var version=(@= data.values.pipeline.version @) \
    -var pre=(@= data.values.pipeline.pre @) \
    -var owner=(@= gci_label_value(data.values.github.owner) @) \
    -var repository=(@= gci_label_value(data.values.github.repository) @) \
    -var branch=(@= gci_label_value(data.values.repository.branch) @) \
    -var image_name_prefix=(@= image_family_name(build.image_family)[0:51] @) \
    -var image_family=(@= image_family_name(build.image_family) @) \
    -var googlecompute_project=(@=  data.values.google.project @) \
    -var googlecompute_zone=(@= data.values.google.zone @) \
    (@= build.image_family @).json
(@- end @)

(@ def instance_create_bash_task(pipeline_name, build_config): -@)
set -ueo pipefail
(@= google_variables() @)
(@= instance_variables() @)
ssh_key_file=${INSTANCE_DIR}/identity
ssh_pubkey_file=${ssh_key_file}.pub
ssh-keygen -m pem  -t rsa -f ${ssh_key_file} -C "${INSTANCE_USER}" -N '' <<< y
ssh_pubkey=$(cat ${ssh_pubkey_file})
ssh_keys_file=${INSTANCE_DIR}/ssh_keys_file
echo "${INSTANCE_USER}:${ssh_pubkey}" > ${ssh_keys_file}
instance_name=build-$(cat /proc/sys/kernel/random/uuid)
image_name=$(cat image/name)
time_to_live=$(( $(date +%s) + ( 4 * 60 * 60 ) ))
instance_file=${INSTANCE_DIR}/instance.sh
gcloud compute instances create ${instance_name} \
    --format='config[export](name,networkInterfaces[0].accessConfigs[0].natIP)' \
    --project=${GOOGLE_PROJECT} \
    --zone=${GOOGLE_ZONE} \
    --subnet=default \
    --machine-type=e2-standard-16 \
    --boot-disk-size=200GB \
    --boot-disk-type=pd-standard \
    --boot-disk-device-name=${instance_name} \
    --image-project=${GOOGLE_PROJECT} \
    --image=${image_name} \
    --metadata-from-file ssh-keys=${ssh_keys_file} \
    --labels=time-to-live=${time_to_live},pipeline-name=(@= pipeline_name @),build-config=(@= build_config @) \
    > ${instance_file}
(@= remote_functions() @)
SSH_OPTIONS="${SSH_OPTIONS} -o ConnectTimeout=10"
echo "Waiting for ssh on ${instance_name} to be ready."
console_file=$(mktemp)
console_next=0
while ! remote_shell echo ready 2>/dev/null ; do
  gcloud compute instances get-serial-port-output ${instance_name} \
      --start ${console_next} \
      --project=${GOOGLE_PROJECT} \
      --zone=${GOOGLE_ZONE} \
      --format='value[separator="
"](next,contents)' \
      > ${console_file}
  tmp_next=$(head -n 1 ${console_file})
  if (( tmp_next != console_next )); then
    console_next=${tmp_next}
    sed '1d;s/\x1b\[[0-9;]*[JH]//g' ${console_file}
  fi
done
(@- end @)

(@ def instance_delete_bash_task(): -@)
set -ueo pipefail
(@= google_variables() @)
(@= instance_variables() @)

instance_file=${INSTANCE_DIR}/instance.sh
instance_name=$(source ${instance_file} && echo -n ${name})

gcloud compute instances delete ${instance_name} \
    --project=${GOOGLE_PROJECT} \
    --zone=${GOOGLE_ZONE} \
    --delete-disks=all \
    --quiet
(@- end @)

(@ def download_build_bash_task(): -@)
set -ueo pipefail
(@= remote_functions() @)
remote_download_directory build .
(@- end @)

(@ def build_bash_task(): -@)
set -ueo pipefail
(@= remote_functions() @)
pushd source
git_url=$(git remote get-url origin)
git_rev=$(git rev-parse HEAD)
popd
version=$(cat version/number)
builddate=$(date "+%Y-%m-%d")
remote_shell cmake -E make_directory build
remote_shell cmake -E time cmake -E chdir build cmake ../source ${CMAKE_CONFIGURE_FLAGS} \
    -DCMAKE_BUILD_TYPE=${CMAKE_CONFIG} \
    -DGEODE_ROOT=../geode \
    -DPRODUCT_VERSION=${version} \
    -DPRODUCT_BUILDDATE=${builddate} \
    -DPRODUCT_SOURCE_REVISION=${git_rev} \
    -DPRODUCT_SOURCE_REPOSITORY=${git_url}
remote_shell cmake -E time cmake --build build --config ${CMAKE_CONFIG} -- ${CMAKE_BUILD_FLAGS}
remote_shell cmake -E time cmake --build build --config ${CMAKE_CONFIG} --target docs -- ${CMAKE_BUILD_FLAGS}
remote_shell cmake -E time cmake -E chdir build cpack -C ${CMAKE_CONFIG} -G "${CPACK_GENERATORS}" | tee cpack.out
packages=$(awk '/^CPack: - package: / {print $4}' cpack.out)
for package in ${packages}; do
  remote_download ${package} package/
done
checksums=$(awk '/^CPack: - checksum file: / {print $5}' cpack.out)
for checksum in ${checksums}; do
  remote_download ${checksum} package/
done
(@- end @)

(@ def upload_source_bash_task(): -@)
set -ueo pipefail
(@= remote_functions() @)
remote_upload source .
(@- end @)


(@ def upload_geode_bash_task(): -@)
set -ueo pipefail
(@= remote_functions() @)
geode_version=$(cat geode-latest/version)
geode_name="apache-geode-${geode_version}"
geode_artifact="${geode_name}.tgz"
remote_upload geode-latest/${geode_artifact} .
remote_shell cmake -E tar xvf ${geode_artifact}
remote_shell cmake -E rename ${geode_name} geode
(@- end @)

(@ def extract_geode_bash_task(): -@)
set -ueo pipefail
if (cmp -s geode-latest/version geode/version); then
 exit 0
fi
geode_version=$(cat geode-latest/version)
geode_name="apache-geode-${geode_version}"
geode_artifact="${geode_name}.tgz"
tar xvf geode-latest/${geode_artifact} --directory=geode --strip-components=1
unzip -p geode/lib/geode-core-${geode_version}.jar org/apache/geode/internal/GemFireVersion.properties
cp geode-latest/version geode/version
(@- end @)

(@ def is_source_from_pr_testable_bash_task(): -@)
# Cribbed shamelessly from geode/ci/scripts/shared_utilities.sh
is_source_from_pr_testable() {
  if [[ $# -ne 3 ]]; then
    >&2 echo "Invalid args. Try ${0} \"<repo_path>\" \"<list of exclusion dirs>\""
    exit 1
  fi
  local repo_dir="${1}"
  if [[ ! -d "${repo_dir}" ]]; then
    # If the repo_dir does not exist, assume call from non-PR
    return 0;
  fi

  pushd "${repo_dir}" 2>&1 >> /dev/null
    local base_dir=$(git rev-parse --show-toplevel)
    local github_pr_dir="${base_dir}/.git/resource"
    pushd ${base_dir} 2>&1 >> /dev/null
      local return_code=0
      if [ -d "${github_pr_dir}" ]; then
        # Modify this path list with directories to exclude
        local exclude_dirs="${2}"
        for d in $(echo ${exclude_dirs}); do
          local exclude_pathspec="${exclude_pathspec} :(exclude,glob)${d}/**"
        done
        local exclude_files="${3}"
        for f in "${exclude_files}"; do
          local exclude_pathspec="${exclude_pathspec} :(exclude,glob)${f}"
        done
        pushd ${base_dir} &> /dev/null
          local files=$(git diff --name-only $(cat "${github_pr_dir}/base_sha") $(cat "${github_pr_dir}/head_sha") -- . $(echo ${exclude_pathspec}))
        popd &> /dev/null
        if [[ -z "${files}" ]]; then
          >&2 echo "Code changes are from CI only"
          return_code=1
        else
          >&2 echo "real code change here!"
        fi
      else
        >&2 echo "repo is not from a PR"
      fi
    popd 2>&1 >> /dev/null
  popd 2>&1 >> /dev/null
  return ${return_code}
}

exclude_dirs="ci packer docker tools"
exclude_files="*/*.md"
is_source_from_pr_testable "source" "${exclude_dirs}" "${exclude_files}" && exit 0 || exit 1

(@- end @)