| #!/usr/bin/env bash |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| |
| # shell function library for Bookkeeper CI builds |
| |
| # lists all available functions in this tool |
| function ci_list_functions() { |
| declare -F | awk '{print $NF}' | sort | egrep '^ci_' | sed 's/^ci_//' |
| } |
| |
| # prints thread dumps for all running JVMs |
| # used in CI when a job gets cancelled because of a job timeout |
| function ci_print_thread_dumps() { |
| for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do |
| echo "----------------------- pid $java_pid -----------------------" |
| cat /proc/$java_pid/cmdline | xargs -0 echo |
| jcmd $java_pid Thread.print -l |
| jcmd $java_pid GC.heap_info |
| done |
| return 0 |
| } |
| |
| # copies test reports into test-reports and surefire-reports directory |
| # subsequent runs of tests might overwrite previous reports. This ensures that all test runs get reported. |
| function ci_move_test_reports() { |
| ( |
| if [ -n "${GITHUB_WORKSPACE}" ]; then |
| cd "${GITHUB_WORKSPACE}" |
| mkdir -p test-reports |
| mkdir -p surefire-reports |
| fi |
| # aggregate all junit xml reports in a single directory |
| if [ -d test-reports ]; then |
| # copy test reports to single directory, rename duplicates |
| find . -path '*/target/surefire-reports/junitreports/TEST-*.xml' -print0 | xargs -0 -r -n 1 mv -t test-reports --backup=numbered |
| # rename possible duplicates to have ".xml" extension |
| ( |
| for f in test-reports/*~; do |
| mv -- "$f" "${f}.xml" |
| done 2>/dev/null |
| ) || true |
| fi |
| # aggregate all surefire-reports in a single directory |
| if [ -d surefire-reports ]; then |
| ( |
| find . -type d -path '*/target/surefire-reports' -not -path './surefire-reports/*' | |
| while IFS=$'\n' read -r directory; do |
| echo "Copying reports from $directory" |
| target_dir="surefire-reports/${directory}" |
| if [ -d "$target_dir" ]; then |
| # rotate backup directory names *~3 -> *~2, *~2 -> *~3, *~1 -> *~2, ... |
| ( command ls -vr1d "${target_dir}~"* 2> /dev/null | awk '{print "mv "$0" "substr($0,0,length-1)substr($0,length,1)+1}' | sh ) || true |
| # backup existing target directory, these are the results of the previous test run |
| mv "$target_dir" "${target_dir}~1" |
| fi |
| # copy files |
| cp -R --parents "$directory" surefire-reports |
| # remove the original directory |
| rm -rf "$directory" |
| done |
| ) |
| fi |
| ) |
| } |
| |
| # Finds fastest up-to-date ubuntu mirror based on download speed |
| function ci_find_fast_ubuntu_mirror() { |
| local ubuntu_release=${1:-"$(lsb_release -c 2>/dev/null | cut -f2 || echo "jammy")"} |
| local ubuntu_arch=${2:-"$(dpkg --print-architecture 2>/dev/null || echo "amd64")"} |
| { |
| # choose mirrors that are up-to-date by checking the Last-Modified header for |
| { |
| # randomly choose up to 10 mirrors using http:// protocol |
| # (https isn't supported in docker containers that don't have ca-certificates installed) |
| curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10 |
| # also consider Azure's Ubuntu mirror |
| echo http://azure.archive.ubuntu.com/ubuntu/ |
| } | xargs -I {} sh -c "ubuntu_release=$ubuntu_release ubuntu_arch=$ubuntu_arch;"'echo "$(curl -m 5 -sI {}dists/${ubuntu_release}/Contents-${ubuntu_arch}.gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }' |
| } | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \ |
| |sort -g -r |head -1| awk '{ print $2 }' |
| } |
| |
| function ci_pick_ubuntu_mirror() { |
| echo "Choosing fastest up-to-date ubuntu mirror based on download speed..." |
| UBUNTU_MIRROR=$(ci_find_fast_ubuntu_mirror) |
| if [ -z "$UBUNTU_MIRROR" ]; then |
| # fallback to no mirror |
| UBUNTU_MIRROR="http://archive.ubuntu.com/ubuntu/" |
| UBUNTU_SECURITY_MIRROR="http://security.ubuntu.com/ubuntu/" |
| else |
| UBUNTU_SECURITY_MIRROR="${UBUNTU_MIRROR}" |
| fi |
| echo "Picked '$UBUNTU_MIRROR'." |
| # set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables |
| # that can be used by docker builds |
| export UBUNTU_MIRROR |
| export UBUNTU_SECURITY_MIRROR |
| # make environment variables available for later GitHub Actions steps |
| if [ -n "$GITHUB_ENV" ]; then |
| echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV |
| echo "UBUNTU_SECURITY_MIRROR=$UBUNTU_SECURITY_MIRROR" >> $GITHUB_ENV |
| fi |
| } |
| |
| if [ -z "$1" ]; then |
| echo "usage: $0 [ci_tool_function_name]" |
| echo "Available ci tool functions:" |
| ci_list_functions |
| exit 1 |
| fi |
| ci_function_name="ci_$1" |
| shift |
| |
| if [[ "$(LC_ALL=C type -t $ci_function_name)" == "function" ]]; then |
| eval "$ci_function_name" "$@" |
| else |
| echo "Invalid ci tool function" |
| echo "Available ci tool functions:" |
| ci_list_functions |
| exit 1 |
| fi |