| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| (@ load("@ytt:data", "data") -@) |
| (@ load("functions.lib.yml", |
| "gci_label_value", |
| "image_family_name") |
| -@) |
| |
| (@ def google_variables(): -@) |
| GOOGLE_PROJECT=(@=data.values.google.project @) |
| GOOGLE_ZONE=(@=data.values.google.zone @) |
| (@- end @) |
| |
| |
| (@ def instance_variables(): -@) |
| INSTANCE_USER=build |
| INSTANCE_DIR=$(pwd)/instance |
| |
| if [ ! -d "${INSTANCE_DIR}" ]; then |
| echo "${INSTANCE_DIR} not found." |
| exit 1 |
| fi |
| (@- end @) |
| |
| (@ def remote_functions(): -@) |
| (@=instance_variables() @) |
| |
| SSH_OPTIONS=${SSH_OPTIONS:-"-o StrictHostKeyChecking=no -o PasswordAuthentication=no"} |
| |
| ssh_key_file=${INSTANCE_DIR}/identity |
| if [ ! -r "${ssh_key_file}" ]; then |
| echo "${ssh_key_file} not readable." |
| exit 1 |
| fi |
| |
| instance_file=${INSTANCE_DIR}/instance.sh |
| if [ ! -r "${instance_file}" ]; then |
| echo "${instance_file} not readable." |
| exit 1 |
| fi |
| |
| external_ip=$(source ${instance_file} && echo -n ${networkInterfaces_accessConfigs_natIP}) |
| |
| function remote_shell { |
| ssh ${SSH_OPTIONS} -i ${ssh_key_file} ${INSTANCE_USER}@${external_ip} "$@" |
| } |
| |
| function remote_download { |
| scp ${SSH_OPTIONS} -i ${ssh_key_file} -q -r "${INSTANCE_USER}@${external_ip}:${1}" "$2" |
| } |
| |
| function remote_download_directory { |
| ssh ${SSH_OPTIONS} -i ${ssh_key_file} ${INSTANCE_USER}@${external_ip} tar -C $(dirname ${1}) -czf - $(basename ${1}) | tar -C ${2} -zxf - |
| } |
| |
| function remote_upload { |
| scp ${SSH_OPTIONS} -i ${ssh_key_file} -q -r "$1" "${INSTANCE_USER}@${external_ip}:${2}" |
| } |
| |
| (@- end @) |
| |
| (@ def run_cpp_unit_tests(): -@) |
| set -ueo pipefail |
| (@= remote_functions() @) |
| remote_shell cmake -E chdir build/cppcache/test ctest -C ${CMAKE_CONFIG} -j8 --timeout=500 --output-on-failure --schedule-random |
| (@- end @) |
| |
| (@ def run_cpp_integration_tests(): -@) |
| set -ueo pipefail |
| (@= remote_functions() @) |
| remote_shell cmake -E chdir build/cppcache/integration/test ctest -C ${CMAKE_CONFIG} -j6 --timeout=500 --output-on-failure --repeat until-pass:4 --schedule-random |
| (@- end @) |
| |
| (@ def run_cpp_legacy_integration_tests(): -@) |
| set -ueo pipefail |
| (@= remote_functions() @) |
| remote_shell cmake -E chdir build/cppcache/integration-test ctest -C ${CMAKE_CONFIG} -j4 --timeout=500 --output-on-failure --repeat until-pass:4 --schedule-random |
| (@- end @) |
| |
| (@ def run_net_unit_tests(): -@) |
| set -ueo pipefail |
| (@= remote_functions() @) |
| remote_shell cmake -E chdir build/clicache/test2 ../packages/xunit.runner.console.2.4.0/tools/net452/xunit.console.exe ${CMAKE_CONFIG}/Apache.Geode.Tests2.dll -parallel all |
| (@- end @) |
| |
| (@ def run_net_integration_tests(): -@) |
| set -ueo pipefail |
| (@= remote_functions() @) |
| remote_shell cmake -E chdir build/clicache/integration-test2 ../packages/xunit.runner.console.2.4.0/tools/net452/xunit.console.exe ${CMAKE_CONFIG}/Apache.Geode.IntegrationTests2.dll -verbose -maxthreads 6 |
| (@- end @) |
| |
| (@ def run_net_legacy_integration_tests(): -@) |
| set -ueo pipefail |
| (@= remote_functions() @) |
| remote_shell cmake -E chdir build/clicache/integration-test ctest -C ${CMAKE_CONFIG} -j6 --timeout=1000 --output-on-failure --repeat until-pass:4 |
| (@- end @) |
| |
| |
| (@ def clang_tidy_bash_task(): -@) |
| set -ueo pipefail |
| export GEODE_HOME=$(pwd)/geode/expanded/apache-geode-$(cat geode/version) |
| log=$(mktemp) |
| cd build |
| cmake ../source -DCMAKE_EXPORT_COMPILE_COMMANDS=ON >>${log} 2>&1 || (cat ${log}; exit 1) |
| cmake --build dependencies --parallel $(nproc) >>${log} 2>&1 || (cat ${log}; exit 1) |
| run-clang-tidy -j $(nproc) -quiet |
| (@- end @) |
| |
| (@ def clang_format_bash_task(): -@) |
| set -ueo pipefail |
| export GEODE_HOME=$(pwd)/geode/expanded/apache-geode-$(cat geode/version) |
| log=$(mktemp) |
| cd build |
| cmake ../source -DCMAKE_EXPORT_COMPILE_COMMANDS=ON >>${log} 2>&1 || (cat ${log}; exit 1) |
| jq -r '.[].file' compile_commands.json | sort | uniq | grep -v $(pwd) | xargs clang-format --dry-run -Werror |
| (@- end @) |
| |
| (@ def rat_check_bash_task(): -@) |
| set -ueo pipefail |
| export GEODE_HOME=$(pwd)/geode/expanded/apache-geode-$(cat geode/version) |
| log=$(mktemp) |
| cd build |
| cmake ../source -DUSE_RAT=ON >>${log} 2>&1 || (cat ${log}; exit 1) |
| cmake --build . --target rat-check |
| (@- end @) |
| |
| (@ def packer_build_bash_task(build): -@) |
| set -ueo pipefail |
| cd source/packer |
| packer build -only=googlecompute \ |
| -var-file=default.json \ |
| -var pipeline=(@= gci_label_value(data.values.pipeline.name) @) \ |
| -var owner=(@= gci_label_value(data.values.github.owner) @) \ |
| -var repository=(@= gci_label_value(data.values.github.repository) @) \ |
| -var branch=(@= gci_label_value(data.values.repository.branch) @) \ |
| -var image_name_prefix=(@= image_family_name(build.image_family)[0:51] @) \ |
| -var image_family=(@= image_family_name(build.image_family) @) \ |
| -var googlecompute_project=(@= data.values.google.project @) \ |
| -var googlecompute_zone=(@= data.values.google.zone @) \ |
| (@= build.image_family @).json |
| (@- end @) |
| |
| (@ def instance_create_bash_task(pipeline_name, build_config): -@) |
| set -ueo pipefail |
| (@= google_variables() @) |
| (@= instance_variables() @) |
| ssh_key_file=${INSTANCE_DIR}/identity |
| ssh_pubkey_file=${ssh_key_file}.pub |
| ssh-keygen -m pem -t rsa -f ${ssh_key_file} -C "${INSTANCE_USER}" -N '' <<< y |
| ssh_pubkey=$(cat ${ssh_pubkey_file}) |
| ssh_keys_file=${INSTANCE_DIR}/ssh_keys_file |
| echo "${INSTANCE_USER}:${ssh_pubkey}" > ${ssh_keys_file} |
| instance_name=build-$(cat /proc/sys/kernel/random/uuid) |
| image_name=$(cat image/name) |
| time_to_live=$(( $(date +%s) + ( 4 * 60 * 60 ) )) |
| instance_file=${INSTANCE_DIR}/instance.sh |
| gcloud compute instances create ${instance_name} \ |
| --format='config[export](name,networkInterfaces[0].accessConfigs[0].natIP)' \ |
| --project=${GOOGLE_PROJECT} \ |
| --zone=${GOOGLE_ZONE} \ |
| --subnet=default \ |
| --machine-type=e2-standard-16 \ |
| --boot-disk-size=200GB \ |
| --boot-disk-type=pd-standard \ |
| --boot-disk-device-name=${instance_name} \ |
| --image-project=${GOOGLE_PROJECT} \ |
| --image=${image_name} \ |
| --metadata-from-file ssh-keys=${ssh_keys_file} \ |
| --labels=time-to-live=${time_to_live},pipeline-name=(@= pipeline_name @),build-config=(@= build_config @) \ |
| > ${instance_file} |
| (@= remote_functions() @) |
| SSH_OPTIONS="${SSH_OPTIONS} -o ConnectTimeout=10" |
| echo "Waiting for ssh on ${instance_name} to be ready." |
| console_file=$(mktemp) |
| console_next=0 |
| while ! remote_shell echo ready 2>/dev/null ; do |
| gcloud compute instances get-serial-port-output ${instance_name} \ |
| --start ${console_next} \ |
| --project=${GOOGLE_PROJECT} \ |
| --zone=${GOOGLE_ZONE} \ |
| --format='value[separator=" |
| "](next,contents)' \ |
| > ${console_file} |
| tmp_next=$(head -n 1 ${console_file}) |
| if (( tmp_next != console_next )); then |
| console_next=${tmp_next} |
| sed '1d;s/\x1b\[[0-9;]*[JH]//g' ${console_file} |
| fi |
| done |
| (@- end @) |
| |
| (@ def instance_delete_bash_task(): -@) |
| set -ueo pipefail |
| (@= google_variables() @) |
| (@= remote_functions() @) |
| |
| instance_name=$(source ${instance_file} && echo -n ${name}) |
| |
| gcloud compute instances delete ${instance_name} \ |
| --project=${GOOGLE_PROJECT} \ |
| --zone=${GOOGLE_ZONE} \ |
| --delete-disks=all \ |
| --quiet |
| (@- end @) |
| |
| (@ def download_build_bash_task(): -@) |
| set -ueo pipefail |
| (@= remote_functions() @) |
| remote_download_directory build . |
| (@- end @) |
| |
| (@ def build_bash_task(): -@) |
| set -ueo pipefail |
| (@= remote_functions() @) |
| pushd source |
| git_url=$(git remote get-url origin) |
| git_rev=$(git rev-parse HEAD) |
| popd |
| version=$(cat version/number) |
| builddate=$(date "+%Y-%m-%d") |
| remote_upload source . |
| remote_shell cmake -E make_directory build |
| remote_shell cmake -E time cmake -E chdir build cmake ../source ${CMAKE_CONFIGURE_FLAGS} \ |
| -DCMAKE_BUILD_TYPE=${CMAKE_CONFIG} \ |
| -DPRODUCT_VERSION=${version} \ |
| -DPRODUCT_BUILDDATE=${builddate} \ |
| -DPRODUCT_SOURCE_REVISION=${git_rev} \ |
| -DPRODUCT_SOURCE_REPOSITORY=${git_url} |
| remote_shell cmake -E time cmake --build build --config ${CMAKE_CONFIG} -- ${CMAKE_BUILD_FLAGS} |
| remote_shell cmake -E time cmake --build build --config ${CMAKE_CONFIG} --target docs -- ${CMAKE_BUILD_FLAGS} |
| remote_shell cmake -E time cmake -E chdir build cpack -C ${CMAKE_CONFIG} -G "${CPACK_GENERATORS}" | tee cpack.out |
| packages=$(awk '/^CPack: - package: / {print $4}' cpack.out) |
| for package in ${packages}; do |
| remote_download ${package} package/ |
| done |
| checksums=$(awk '/^CPack: - checksum file: / {print $5}' cpack.out) |
| for checksum in ${checksums}; do |
| remote_download ${checksum} package/ |
| done |
| (@- end @) |