| #!/usr/bin/env bash |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| |
| # shell function library for Pulsar CI builds |
| |
| SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" |
| |
| set -e |
| set -o pipefail |
| |
| ARTIFACT_RETENTION_DAYS="${ARTIFACT_RETENTION_DAYS:-3}" |
| |
| # lists all available functions in this tool |
| function ci_list_functions() { |
| declare -F | awk '{print $NF}' | sort | grep -E '^ci_' | sed 's/^ci_//' |
| } |
| |
| # prints thread dumps for all running JVMs |
| # used in CI when a job gets cancelled because of a job timeout |
| function ci_print_thread_dumps() { |
| for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do |
| echo "----------------------- pid $java_pid -----------------------" |
| cat /proc/$java_pid/cmdline | xargs -0 echo |
| jcmd $java_pid Thread.print -l |
| jcmd $java_pid GC.heap_info |
| done |
| return 0 |
| } |
| |
| # runs maven |
| function _ci_mvn() { |
| mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ |
| "$@" |
| } |
| |
| # runs OWASP Dependency Check for all projects |
| function ci_dependency_check() { |
| _ci_mvn -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl '!pulsar-client-tools-test' "$@" |
| } |
| |
| function ci_pick_ubuntu_mirror() { |
| echo "Choosing fastest up-to-date ubuntu mirror based on download speed..." |
| UBUNTU_MIRROR=$({ |
| # choose mirrors that are up-to-date by checking the Last-Modified header for |
| { |
| # randomly choose up to 10 mirrors using http:// protocol |
| # (https isn't supported in docker containers that don't have ca-certificates installed) |
| curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10 |
| # also consider Azure's Ubuntu mirror |
| echo http://azure.archive.ubuntu.com/ubuntu/ |
| } | xargs -I {} sh -c 'echo "$(curl -m 5 -sI {}dists/$(lsb_release -c | cut -f2)-security/Contents-$(dpkg --print-architecture).gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }' |
| } | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \ |
| |sort -g -r |head -1| awk '{ print $2 }') |
| if [ -z "$UBUNTU_MIRROR" ]; then |
| # fallback to full mirrors list |
| UBUNTU_MIRROR="mirror://mirrors.ubuntu.com/mirrors.txt" |
| fi |
| OLD_MIRROR=$(cat /etc/apt/sources.list | grep '^deb ' | head -1 | awk '{ print $2 }') |
| echo "Picked '$UBUNTU_MIRROR'. Current mirror is '$OLD_MIRROR'." |
| if [[ "$OLD_MIRROR" != "$UBUNTU_MIRROR" ]]; then |
| sudo sed -i "s|$OLD_MIRROR|$UBUNTU_MIRROR|g" /etc/apt/sources.list |
| sudo apt-get update |
| fi |
| # set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables |
| # that can be used by docker builds |
| export UBUNTU_MIRROR |
| export UBUNTU_SECURITY_MIRROR=$UBUNTU_MIRROR |
| # make environment variables available for later GitHub Actions steps |
| if [ -n "$GITHUB_ENV" ]; then |
| echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV |
| echo "UBUNTU_SECURITY_MIRROR=$UBUNTU_SECURITY_MIRROR" >> $GITHUB_ENV |
| fi |
| } |
| |
| # installs a tool executable if it's not found on the PATH |
| function ci_install_tool() { |
| local tool_executable=$1 |
| local tool_package=${2:-$1} |
| if ! command -v $tool_executable &>/dev/null; then |
| if [[ "$GITHUB_ACTIONS" == "true" ]]; then |
| echo "::group::Installing ${tool_package}" |
| sudo apt-get -y install ${tool_package} >/dev/null || { |
| echo "Installing the package failed. Switching the ubuntu mirror and retrying..." |
| ci_pick_ubuntu_mirror |
| # retry after picking the ubuntu mirror |
| sudo apt-get -y install ${tool_package} |
| } |
| echo '::endgroup::' |
| else |
| fail "$tool_executable wasn't found on PATH. You should first install $tool_package with your package manager." |
| fi |
| fi |
| } |
| |
| # outputs the given message to stderr and exits the shell script |
| function fail() { |
| echo "$*" >&2 |
| exit 1 |
| } |
| |
| # saves a given image (1st parameter) to the GitHub Actions Artifacts with the given name (2nd parameter) |
| function ci_docker_save_image_to_github_actions_artifacts() { |
| local image=$1 |
| local artifactname="${2}.zst" |
| ci_install_tool pv |
| echo "::group::Saving docker image ${image} with name ${artifactname} in GitHub Actions Artifacts" |
| # delete possible previous artifact that might exist when re-running |
| timeout 1m gh-actions-artifact-client.js delete "${artifactname}" &>/dev/null || true |
| docker save ${image} | zstd | pv -ft -i 5 | pv -Wbaf -i 5 | timeout 20m gh-actions-artifact-client.js upload --retentionDays=$ARTIFACT_RETENTION_DAYS "${artifactname}" |
| echo "::endgroup::" |
| } |
| |
| # loads a docker image from the GitHub Actions Artifacts with the given name (1st parameter) |
| function ci_docker_load_image_from_github_actions_artifacts() { |
| local artifactname="${1}.zst" |
| ci_install_tool pv |
| echo "::group::Loading docker image from name ${artifactname} in GitHub Actions Artifacts" |
| timeout 20m gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | unzstd | docker load |
| echo "::endgroup::" |
| } |
| |
| # loads and extracts a zstd (.tar.zst) compressed tar file from the GitHub Actions Artifacts with the given name (1st parameter) |
| function ci_restore_tar_from_github_actions_artifacts() { |
| local artifactname="${1}.tar.zst" |
| ci_install_tool pv |
| echo "::group::Restoring tar from name ${artifactname} in GitHub Actions Artifacts to $PWD" |
| timeout 5m gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | tar -I zstd -xf - |
| echo "::endgroup::" |
| } |
| |
| # stores a given command (with full arguments, specified after 1st parameter) output to GitHub Actions Artifacts with the given name (1st parameter) |
| function ci_store_tar_to_github_actions_artifacts() { |
| local artifactname="${1}.tar.zst" |
| shift |
| if [[ "$GITHUB_ACTIONS" == "true" ]]; then |
| ci_install_tool pv |
| echo "::group::Storing $1 tar command output to name ${artifactname} in GitHub Actions Artifacts" |
| # delete possible previous artifact that might exist when re-running |
| timeout 1m gh-actions-artifact-client.js delete "${artifactname}" &>/dev/null || true |
| "$@" | pv -ft -i 5 | pv -Wbaf -i 5 | timeout 10m gh-actions-artifact-client.js upload --retentionDays=$ARTIFACT_RETENTION_DAYS "${artifactname}" |
| echo "::endgroup::" |
| else |
| local artifactfile="$(mktemp -t artifact.XXXX)" |
| echo "Storing output for debugging in $artifactfile" |
| "$@" | pv -ft -i 5 | pv -Wbaf -i 5 > $artifactfile |
| fi |
| } |
| |
| # copies test reports into test-reports and surefire-reports directory |
| # subsequent runs of tests might overwrite previous reports. This ensures that all test runs get reported. |
| function ci_move_test_reports() { |
| ( |
| if [ -n "${GITHUB_WORKSPACE}" ]; then |
| cd "${GITHUB_WORKSPACE}" |
| mkdir -p test-reports |
| mkdir -p surefire-reports |
| fi |
| # aggregate all junit xml reports in a single directory |
| if [ -d test-reports ]; then |
| # copy test reports to single directory, rename duplicates |
| find . -path '*/target/surefire-reports/junitreports/TEST-*.xml' -print0 | xargs -0 -r -n 1 mv -t test-reports --backup=numbered |
| # rename possible duplicates to have ".xml" extension |
| ( |
| for f in test-reports/*~; do |
| mv -- "$f" "${f}.xml" |
| done 2>/dev/null |
| ) || true |
| fi |
| # aggregate all surefire-reports in a single directory |
| if [ -d surefire-reports ]; then |
| ( |
| find . -type d -path '*/target/surefire-reports' -not -path './surefire-reports/*' | |
| while IFS=$'\n' read -r directory; do |
| echo "Copying reports from $directory" |
| target_dir="surefire-reports/${directory}" |
| if [ -d "$target_dir" ]; then |
| # rotate backup directory names *~3 -> *~2, *~2 -> *~3, *~1 -> *~2, ... |
| ( command ls -vr1d "${target_dir}~"* 2> /dev/null | awk '{print "mv "$0" "substr($0,0,length-1)substr($0,length,1)+1}' | sh ) || true |
| # backup existing target directory, these are the results of the previous test run |
| mv "$target_dir" "${target_dir}~1" |
| fi |
| # copy files |
| cp -R --parents "$directory" surefire-reports |
| # remove the original directory |
| rm -rf "$directory" |
| done |
| ) |
| fi |
| ) |
| } |
| |
| function ci_check_ready_to_test() { |
| if [[ -z "$GITHUB_EVENT_PATH" ]]; then |
| >&2 echo "GITHUB_EVENT_PATH isn't set" |
| return 1 |
| fi |
| |
| PR_JSON_URL=$(jq -r '.pull_request.url' "${GITHUB_EVENT_PATH}") |
| echo "Refreshing $PR_JSON_URL..." |
| PR_JSON=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "${PR_JSON_URL}") |
| |
| if printf "%s" "${PR_JSON}" | jq -e '.draft | select(. == true)' &> /dev/null; then |
| echo "PR is draft." |
| elif ! ( printf "%s" "${PR_JSON}" | jq -e '.mergeable | select(. == true)' &> /dev/null ); then |
| echo "PR isn't mergeable." |
| else |
| # check ready-to-test label |
| if printf "%s" "${PR_JSON}" | jq -e '.labels[] | .name | select(. == "ready-to-test")' &> /dev/null; then |
| echo "Found ready-to-test label." |
| return 0 |
| else |
| echo "There is no ready-to-test label on the PR." |
| fi |
| |
| # check if the PR has been approved |
| PR_NUM=$(jq -r '.pull_request.number' "${GITHUB_EVENT_PATH}") |
| REPO_FULL_NAME=$(jq -r '.repository.full_name' "${GITHUB_EVENT_PATH}") |
| REPO_NAME=$(basename "${REPO_FULL_NAME}") |
| REPO_OWNER=$(dirname "${REPO_FULL_NAME}") |
| # use graphql query to find out reviewDecision |
| PR_REVIEW_DECISION=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" -X POST -d '{"query": "query { repository(name: \"'${REPO_NAME}'\", owner: \"'${REPO_OWNER}'\") { pullRequest(number: '${PR_NUM}') { reviewDecision } } }"}' https://api.github.com/graphql |jq -r '.data.repository.pullRequest.reviewDecision') |
| echo "Review decision for PR #${PR_NUM} in repository ${REPO_OWNER}/${REPO_NAME} is ${PR_REVIEW_DECISION}" |
| if [[ "$PR_REVIEW_DECISION" == "APPROVED" ]]; then |
| return 0 |
| fi |
| fi |
| |
| FORK_REPO_URL=$(jq -r '.pull_request.head.repo.html_url' "$GITHUB_EVENT_PATH") |
| PR_BRANCH_LABEL=$(jq -r '.pull_request.head.label' "$GITHUB_EVENT_PATH") |
| PR_BASE_BRANCH=$(jq -r '.pull_request.base.ref' "$GITHUB_EVENT_PATH") |
| PR_URL=$(jq -r '.pull_request.html_url' "$GITHUB_EVENT_PATH") |
| FORK_PR_TITLE_URL_ENCODED=$(printf "%s" "${PR_JSON}" | jq -r '"[run-tests] " + .title | @uri') |
| FORK_PR_BODY_URL_ENCODED=$(jq -n -r "\"This PR is for running tests for upstream PR ${PR_URL}.\n\n<!-- Before creating this PR, please ensure that the fork $FORK_REPO_URL is up to date with https://github.com/apache/pulsar -->\" | @uri") |
| if [[ "$PR_BASE_BRANCH" != "master" ]]; then |
| sync_non_master_fork_docs=$(cat <<EOF |
| \\$('\n') |
| If ${FORK_REPO_URL}/tree/${PR_BASE_BRANCH} is missing, you must sync the branch ${PR_BASE_BRANCH} on the command line. |
| \`\`\` |
| git fetch https://github.com/apache/pulsar ${PR_BASE_BRANCH} |
| git push ${FORK_REPO_URL} FETCH_HEAD:refs/heads/${PR_BASE_BRANCH} |
| \`\`\` |
| EOF |
| ) |
| else |
| sync_non_master_fork_docs="" |
| fi |
| |
| >&2 tee -a "$GITHUB_STEP_SUMMARY" <<EOF |
| |
| # Instructions for proceeding with the pull request: |
| |
| apache/pulsar pull requests should be first tested in your own fork since the apache/pulsar CI based on |
| GitHub Actions has constrained resources and quota. GitHub Actions provides separate quota for |
| pull requests that are executed in a forked repository. |
| |
| 1. Go to ${FORK_REPO_URL}/tree/${PR_BASE_BRANCH} and ensure that your ${PR_BASE_BRANCH} branch is up to date |
| with https://github.com/apache/pulsar \\ |
| [Sync your fork if it's behind.](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork)${sync_non_master_fork_docs} |
| 2. Open a pull request to your own fork. You can use this link to create the pull request in |
| your own fork: |
| [Create PR in fork for running tests](${FORK_REPO_URL}/compare/${PR_BASE_BRANCH}...${PR_BRANCH_LABEL}?expand=1&title=${FORK_PR_TITLE_URL_ENCODED}&body=${FORK_PR_BODY_URL_ENCODED}) |
| 3. Edit the description of the pull request ${PR_URL} and add the link to the PR that you opened to your own fork |
| so that the reviewer can verify that tests pass in your own fork. |
| 4. Ensure that tests pass in your own fork. Your own fork will be used to run the tests during the PR review |
| and any changes made during the review. You as a PR author are responsible for following up on test failures. |
| Please report any flaky tests as new issues at https://github.com/apache/pulsar/issues |
| after checking that the flaky test isn't already reported. |
| 5. When the PR is approved, it will be possible to restart the Pulsar CI workflow within apache/pulsar |
| repository by adding a comment "/pulsarbot rerun-failure-checks" to the PR. |
| An alternative for the PR approval is to add a ready-to-test label to the PR. This can be done |
| by Apache Pulsar committers. |
| 6. When tests pass on the apache/pulsar side, the PR can be merged by a Apache Pulsar Committer. |
| |
| If you have any trouble you can get support in multiple ways: |
| * by sending email to the [dev mailing list](mailto:dev@pulsar.apache.org) ([subscribe](mailto:dev-subscribe@pulsar.apache.org)) |
| * on the [#contributors channel on Pulsar Slack](https://apache-pulsar.slack.com/channels/contributors) ([join](https://pulsar.apache.org/community#section-discussions)) |
| * in apache/pulsar [GitHub discussions Q&A](https://github.com/apache/pulsar/discussions/categories/q-a) |
| |
| EOF |
| return 1 |
| } |
| |
| ci_snapshot_pulsar_maven_artifacts() { |
| ( |
| if [ -n "$GITHUB_WORKSPACE" ]; then |
| cd "$GITHUB_WORKSPACE" |
| else |
| fail "This script can only be run in GitHub Actions" |
| fi |
| mkdir -p target |
| find $HOME/.m2/repository/org/apache/pulsar -name "*.jar" > /tmp/provided_pulsar_maven_artifacts |
| ) |
| } |
| |
| ci_upload_unittest_coverage_files() { |
| _ci_upload_coverage_files unittest "$1" |
| } |
| |
| ci_upload_inttest_coverage_files() { |
| _ci_upload_coverage_files_inttest inttest "$1" integration-tests |
| } |
| |
| ci_upload_systest_coverage_files() { |
| _ci_upload_coverage_files_inttest systest "$1" system-tests |
| } |
| |
| _ci_upload_coverage_files_inttest() { |
| local testtype="$1" |
| local testgroup="$2" |
| local job_name="$3" |
| local store_deps=0 |
| local firsttestgroup="$(_ci_list_testgroups_with_coverage "${job_name}" | head -1)" |
| if [[ "${firsttestgroup}" == "${testgroup}" ]]; then |
| store_deps=1 |
| fi |
| _ci_upload_coverage_files "${testtype}" "${testgroup}" $store_deps |
| } |
| |
| _ci_upload_coverage_files() { |
| ( |
| testtype="$1" |
| testgroup="$2" |
| store_deps="${3:-1}" |
| echo "::group::Uploading $testtype coverage files" |
| if [ -n "$GITHUB_WORKSPACE" ]; then |
| cd "$GITHUB_WORKSPACE" |
| else |
| fail "This script can only be run in GitHub Actions" |
| fi |
| |
| if [ ! -f /tmp/provided_pulsar_maven_artifacts ]; then |
| fail "It is necessary to run '$0 snapshot_pulsar_maven_artifacts' before running any tests." |
| fi |
| |
| set -x |
| |
| local classpathFile="target/classpath_${testtype}_${testgroup}" |
| |
| local execFiles=$(find . -path "*/target/jacoco*.exec" -printf "%P\n") |
| if [[ -n "$execFiles" ]]; then |
| # create temp file |
| local completeClasspathFile=$(mktemp -t tmp.classpath.XXXX) |
| |
| ci_install_tool xmlstarlet |
| |
| local projects=$({ |
| for execFile in $execFiles; do |
| echo $(dirname "$(dirname "$execFile")") |
| done |
| } | sort | uniq) |
| |
| # iterate the projects |
| for project in $projects; do |
| local artifactId=$(xmlstarlet sel -t -m _:project -v _:artifactId -n $project/pom.xml) |
| # find the test scope classpath for the project |
| mvn -f $project/pom.xml -DincludeScope=test -Dscan=false dependency:build-classpath -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \ |
| | sed 's/:/\n/g' | { grep 'org/apache/pulsar' || true; } \ |
| | { tee -a $completeClasspathFile || true; } > target/classpath_$artifactId || true |
| done |
| |
| cat $completeClasspathFile | sort | uniq > $classpathFile |
| # delete temp file |
| rm $completeClasspathFile |
| |
| # upload target/jacoco*.exec, target/classes and any dependent jar files that were built during the unit test execution |
| # transform jacoco exec filenames by appending "_${testtype}_${testgroup}" to the filename part to make the files unique |
| # so that they don't get overridden when all files are extracted to the same working directory before merging |
| ( |
| cd / |
| ci_store_tar_to_github_actions_artifacts coverage_and_deps_${testtype}_${testgroup} \ |
| tar -I zstd -cPf - \ |
| --transform="flags=r;s|\\(/jacoco.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \ |
| --transform="flags=r;s|\\(/tmp/jacocoDir/.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \ |
| --exclude="*/META-INF/bundled-dependencies/*" \ |
| $GITHUB_WORKSPACE/target/classpath_* \ |
| $(find "$GITHUB_WORKSPACE" -path "*/target/jacoco*.exec" -printf "%p\n%h/classes\n" | sort | uniq) \ |
| $([ -d /tmp/jacocoDir ] && echo "/tmp/jacocoDir" ) \ |
| $([[ $store_deps -eq 1 ]] && { cat $GITHUB_WORKSPACE/$classpathFile | sort | uniq | { grep -v -Fx -f /tmp/provided_pulsar_maven_artifacts || true; }; } || true) |
| ) |
| fi |
| echo "::endgroup::" |
| ) |
| } |
| |
| ci_restore_unittest_coverage_files() { |
| _ci_restore_coverage_files unittest unit-tests |
| } |
| |
| ci_restore_inttest_coverage_files() { |
| _ci_restore_coverage_files inttest integration-tests |
| } |
| |
| ci_restore_systest_coverage_files() { |
| _ci_restore_coverage_files systest system-tests |
| } |
| |
| _ci_restore_coverage_files() { |
| ( |
| test_type="$1" |
| job_name="$2" |
| cd / |
| for testgroup in $(_ci_list_testgroups_with_coverage "${job_name}"); do |
| ci_restore_tar_from_github_actions_artifacts coverage_and_deps_${test_type}_${testgroup} || true |
| done |
| ) |
| } |
| |
| _ci_list_testgroups_with_coverage() { |
| local job_name="$1" |
| yq e ".jobs.${job_name}.strategy.matrix.include.[] | select(.no_coverage != true) | .group" "$GITHUB_WORKSPACE/.github/workflows/pulsar-ci.yaml" |
| } |
| |
| ci_delete_unittest_coverage_files() { |
| _ci_delete_coverage_files unittest unit-tests |
| } |
| |
| ci_delete_inttest_coverage_files() { |
| _ci_delete_coverage_files inttest integration-tests |
| } |
| |
| ci_delete_systest_coverage_files() { |
| _ci_delete_coverage_files systest system-tests |
| } |
| |
| _ci_delete_coverage_files() { |
| ( |
| test_type="$1" |
| job_name="$2" |
| for testgroup in $(yq e ".jobs.${job_name}.strategy.matrix.include.[] | select(.no_coverage != true) | .group" "$GITHUB_WORKSPACE/.github/workflows/pulsar-ci.yaml"); do |
| timeout 1m gh-actions-artifact-client.js delete coverage_and_deps_${test_type}_${testgroup}.tar.zst || true |
| done |
| ) |
| } |
| |
| # creates an aggregated jacoco xml report for all projects that contain a target/jacoco*.exec file |
| # |
| # the default maven jacoco report has multiple problems: |
| # - by default, jacoco:report goal will only report coverage for the current project. it is not suitable for Pulsar's |
| # unit tests that test production code that resides in multiple modules. |
| # - there's jacoco:report-aggregate that is supposed to resolve this. It has 2 issues: |
| # - 0.8.8 version doesn't yet support the required "includeCurrentProject" feature |
| # - the dependent projects must be built as part of the same mvn execution and belong to the same maven "reactor" |
| # - this isn't compatible with the way how Pulsar CI builds in "Build and License check" job and reuses |
| # the build results to run unit tests. |
| # |
| # This solution resolves the problem by using the Jacoco command line tool to generate the report. |
| # It assumes that all projects that contain a target/jacoco*.exec file will also contain compiled classfiles. |
| ci_create_test_coverage_report() { |
| echo "::group::Create test coverage report" |
| if [ -n "$GITHUB_WORKSPACE" ]; then |
| cd "$GITHUB_WORKSPACE" |
| else |
| cd "$SCRIPT_DIR/.." |
| fi |
| |
| local execFiles=$(find . -path "*/target/jacoco*.exec" -size +10c -printf "%P\n" ) |
| if [[ -n "$execFiles" ]]; then |
| mkdir -p /tmp/jacocoDir |
| if [ ! -f /tmp/jacocoDir/jacococli.jar ]; then |
| local jacoco_version=$(mvn help:evaluate -Dscan=false -Dexpression=jacoco-maven-plugin.version -q -DforceStdout) |
| curl -sL -o /tmp/jacocoDir/jacococli.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/${jacoco_version}/org.jacoco.cli-${jacoco_version}-nodeps.jar" |
| fi |
| |
| ci_install_tool xmlstarlet |
| # create mapping from project directory to project artifactId |
| local projectToArtifactIdMapping=$(find -name pom.xml -printf "%P\n" |xargs -I{} bash -c 'echo -n "$(dirname $1) "; xmlstarlet sel -t -m _:project -v _:artifactId -n $1' -- {}) |
| |
| # create temp files |
| local completeClasspathFile=$(mktemp -t tmp.classpath.XXXX) |
| local filterArtifactsFile=$(mktemp -t tmp.artifacts.XXXX) |
| local classesDir=$(mktemp -d -t tmp.classes.XXXX) |
| local sourcefilesFile=$(mktemp -t tmp.sources.XXXX) |
| |
| local projects=$({ |
| for execFile in $execFiles; do |
| echo $(dirname "$(dirname "$execFile")") |
| done |
| } | sort | uniq) |
| |
| # projects that aren't considered as production code and their own src/main/java source code shouldn't be analysed |
| local excludeProjectsPattern="testmocks|testclient|buildtools" |
| |
| # iterate projects |
| for project in $projects; do |
| local artifactId="$(printf "%s" "$projectToArtifactIdMapping" | grep -F "$project " | cut -d' ' -f2)" |
| if [[ -d "$project/target/classes" && -d "$project/src/main/java" ]]; then |
| mkdir -p "$classesDir/$project" |
| cp -Rl "$project/target/classes" "$classesDir/$project" |
| echo "$project/src/main/java" >> $sourcefilesFile |
| fi |
| echo "/$artifactId/" >> $filterArtifactsFile |
| if [[ -n "$(echo "$project" | grep -v -E "$excludeProjectsPattern")" ]]; then |
| if [ -f "target/classpath_$artifactId" ]; then |
| echo "Found cached classpath for $artifactId." |
| cat "target/classpath_$artifactId" >> $completeClasspathFile |
| else |
| echo "Resolving classpath for $project..." |
| # find the test scope classpath for the project |
| mvn -f $project/pom.xml -DincludeScope=test -Dscan=false dependency:build-classpath -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \ |
| | sed 's/:/\n/g' | { grep 'org/apache/pulsar' || true; } \ |
| >> $completeClasspathFile || true |
| fi |
| else |
| echo "Skipping analysing of $project" |
| fi |
| done |
| |
| # delete any possible embedded jar files in the classes directory |
| find "$classesDir" -name "*.jar" -print -delete |
| |
| local excludeJarsPattern="bouncy-castle-bc|tests|$excludeProjectsPattern" |
| |
| local classfilesArgs="--classfiles $({ |
| { |
| for classpathEntry in $(cat $completeClasspathFile | { grep -v -f $filterArtifactsFile || true; } | sort | uniq | { grep -v -E "$excludeJarsPattern" || true; }); do |
| if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 2>/dev/null)" ]]; then |
| # file must be processed by removing META-INF/bundled-dependencies |
| local jartempfile=$(mktemp -t jarfile.XXXX --suffix=.jar) |
| cp $classpathEntry $jartempfile |
| zip -q -d $jartempfile 'META-INF/bundled-dependencies/*' &> /dev/null |
| echo $jartempfile |
| else |
| echo $classpathEntry |
| fi |
| done |
| } |
| echo $classesDir |
| } | tr '\n' ':' | sed -e 's/:$//' -e 's/:/ --classfiles /g')" |
| |
| local sourcefilesArgs="--sourcefiles $({ |
| # find the source file folders for the pulsar .jar files that are on the classpath |
| for artifactId in $(cat $completeClasspathFile | sort | uniq | { grep -v -E "$excludeJarsPattern" || true; } | perl -p -e 's|.*/org/apache/pulsar/([^/]*)/.*|$1|'); do |
| local project="$(printf "%s" "$projectToArtifactIdMapping" | { grep $artifactId || true; } | cut -d' ' -f1)" |
| if [[ -n "$project" && -d "$project/src/main/java" ]]; then |
| echo "$project/src/main/java" |
| fi |
| done |
| cat $sourcefilesFile |
| } | tr '\n' ':' | sed -e 's/:$//' -e 's/:/ --sourcefiles /g')" |
| |
| rm $completeClasspathFile $filterArtifactsFile $sourcefilesFile |
| |
| set -x |
| mkdir -p target/jacoco_test_coverage_report/html |
| java -jar /tmp/jacocoDir/jacococli.jar report $execFiles \ |
| $classfilesArgs \ |
| --encoding UTF-8 --name "Apache Pulsar test coverage" \ |
| $sourcefilesArgs \ |
| --xml target/jacoco_test_coverage_report/jacoco.xml \ |
| --html target/jacoco_test_coverage_report/html \ |
| --csv target/jacoco_test_coverage_report/jacoco.csv |
| set +x |
| |
| rm -rf "$classesDir" |
| fi |
| echo "::endgroup::" |
| } |
| |
| # creates a jacoco xml report of the jacoco exec files produced in docker containers which have /tmp/jacocoDir mounted as /jacocoDir |
| # this is used to calculate test coverage for the apache/pulsar code that is run inside the containers in integration tests |
| # and system tests |
| ci_create_inttest_coverage_report() { |
| echo "::group::Create int test coverage in containers report" |
| if [[ -n "$(find /tmp/jacocoDir -name "*.exec" -print -quit)" ]]; then |
| cd "$GITHUB_WORKSPACE" |
| echo "Creating coverage report to target/jacoco_inttest_coverage_report" |
| set -x |
| mkdir -p target/jacoco_inttest_coverage_report |
| # install jacococli.jar command line tool |
| if [ ! -f /tmp/jacocoDir/jacococli.jar ]; then |
| local jacoco_version=$(mvn help:evaluate -Dexpression=jacoco-maven-plugin.version -Dscan=false -q -DforceStdout) |
| curl -sL -o /tmp/jacocoDir/jacococli.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/${jacoco_version}/org.jacoco.cli-${jacoco_version}-nodeps.jar" |
| fi |
| # extract the Pulsar jar files from the docker image that was used to run the tests in the docker containers |
| # the class files used to produce the jacoco exec files are needed in the xml report generation |
| if [ ! -d /tmp/jacocoDir/pulsar_lib ]; then |
| mkdir /tmp/jacocoDir/pulsar_lib |
| docker run --rm -u "$UID:${GID:-"$(id -g)"}" -v /tmp/jacocoDir/pulsar_lib:/pulsar_lib:rw ${PULSAR_TEST_IMAGE_NAME:-apachepulsar/java-test-image:latest} bash -c "cp -p /pulsar/lib/org.apache.pulsar-* /pulsar_lib; [ -d /pulsar/connectors ] && cp -R /pulsar/connectors /pulsar_lib || true" |
| # remove jar file that causes duplicate classes issue |
| rm /tmp/jacocoDir/pulsar_lib/org.apache.pulsar-bouncy-castle* || true |
| # remove any bundled dependencies as part of .jar/.nar files |
| find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' \; |grep -E -v "Nothing to do|^$" || true |
| fi |
| # projects that aren't considered as production code and their own src/main/java source code shouldn't be analysed |
| local excludeProjectsPattern="testmocks|testclient|buildtools" |
| # produce jacoco XML coverage report from the exec files and using the extracted jar files |
| java -jar /tmp/jacocoDir/jacococli.jar report /tmp/jacocoDir/*.exec \ |
| --classfiles /tmp/jacocoDir/pulsar_lib --encoding UTF-8 --name "Pulsar Integration Tests - coverage in containers" \ |
| $(find -path "*/src/main/java" -printf "--sourcefiles %P " | grep -v -E "$excludeProjectsPattern") \ |
| --xml target/jacoco_inttest_coverage_report/jacoco.xml \ |
| --html target/jacoco_inttest_coverage_report/html \ |
| --csv target/jacoco_inttest_coverage_report/jacoco.csv |
| set +x |
| else |
| echo "No /tmp/jacocoDir/*.exec files to process" |
| fi |
| echo "::endgroup::" |
| } |
| |
| if [ -z "$1" ]; then |
| echo "usage: $0 [ci_tool_function_name]" |
| echo "Available ci tool functions:" |
| ci_list_functions |
| exit 1 |
| fi |
| ci_function_name="ci_$1" |
| shift |
| |
| if [[ "$(LC_ALL=C type -t "${ci_function_name}")" == "function" ]]; then |
| eval "$ci_function_name" "$@" |
| else |
| echo "Invalid ci tool function" |
| echo "Available ci tool functions:" |
| ci_list_functions |
| exit 1 |
| fi |
