| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| |
| name: BookKeeper CI |
| |
| on: |
| pull_request: |
| branches: |
| - master |
| - branch-* |
| push: |
| branches: |
| - master |
| - branch-* |
| workflow_dispatch: |
| |
| env: |
| MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 |
| |
| concurrency: |
| group: ${{ github.workflow }}-${{ github.ref }} |
| cancel-in-progress: true |
| |
| jobs: |
| build-and-license-check: |
| name: PR Validation |
| runs-on: ubuntu-latest |
| timeout-minutes: 60 |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| |
| - name: Detect changed files |
| id: changes |
| uses: apache/pulsar-test-infra/paths-filter@master |
| with: |
| filters: .github/changes-filter.yaml |
| list-files: csv |
| |
| - name: Check changed files |
| id: check_changes |
| run: | |
| echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT |
| echo "need_owasp=${{ fromJSON(steps.changes.outputs.need_owasp) }}" >> $GITHUB_OUTPUT |
| |
| - name: Cache local Maven repository |
| if: steps.check_changes.outputs.docs_only != 'true' |
| id: cache |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository/*/*/* |
| !~/.m2/repository/org/apache/bookkeeper |
| !~/.m2/repository/org/apache/distributedlog |
| key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} |
| |
| - name: Set up JDK 11 |
| if: steps.check_changes.outputs.docs_only != 'true' |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: 11 |
| |
| - name: Validate pull request |
| if: steps.check_changes.outputs.docs_only != 'true' |
| run: | |
| mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests |
| mvn -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests |
| |
| - name: Check license files |
| if: steps.check_changes.outputs.docs_only != 'true' |
| run: dev/check-all-licenses |
| |
| - name: Generate Javadoc |
| if: steps.check_changes.outputs.docs_only != 'true' |
| run: mvn -B -nsu -am -pl bookkeeper-common,bookkeeper-server,:bookkeeper-stats-api,:bookkeeper-stats-providers,:codahale-metrics-provider,:prometheus-metrics-provider javadoc:aggregate -DskipTests -Pdelombok -Dchesktyle.skip -Dspotbugs.skip |
| outputs: |
| docs_only: ${{ steps.check_changes.outputs.docs_only }} |
| need_owasp: ${{ steps.check_changes.outputs.need_owasp }} |
| |
| unit-tests: |
| name: ${{ matrix.step_name }} |
| runs-on: ubuntu-latest |
| timeout-minutes: ${{ matrix.timeout || 60 }} |
| needs: [ 'build-and-license-check' ] |
| if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} |
| strategy: |
| fail-fast: false |
| matrix: |
| include: |
| - step_name: Bookie Tests |
| module: bookkeeper-server |
| flag: bookie |
| test_args: "-Dtest='org.apache.bookkeeper.bookie.**'" |
| - step_name: Client Tests |
| module: bookkeeper-server |
| flag: client |
| test_args: "-Dtest='org.apache.bookkeeper.client.**'" |
| timeout: 75 |
| - step_name: Replication Tests |
| module: bookkeeper-server |
| flag: replication |
| test_args: "-Dtest='org.apache.bookkeeper.replication.**'" |
| - step_name: Remaining Tests |
| module: bookkeeper-server |
| flag: remaining |
| test_args: "-Dtest='!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**'" |
| - step_name: TLS Tests |
| module: bookkeeper-server |
| flag: tls |
| test_args: "-Dtest='org.apache.bookkeeper.tls.**'" |
| - step_name: StreamStorage Tests |
| test_args: "-f stream/pom.xml -DstreamTests" |
| flag: stream |
| - step_name: Shell tests |
| module: tests/scripts |
| flag: shell |
| |
| steps: |
| - name: checkout |
| uses: actions/checkout@v4 |
| |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| |
| - name: Cache local Maven repository |
| id: cache |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository/*/*/* |
| !~/.m2/repository/org/apache/bookkeeper |
| !~/.m2/repository/org/apache/distributedlog |
| key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} |
| |
| - name: Set up JDK 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: 11 |
| |
| - name: Build |
| run: | |
| projects_list= |
| if [[ ! -z "${{ matrix.module }}" ]]; then |
| projects_list="-pl ${{ matrix.module }}" |
| fi |
| mvn -q -T 1C -B -nsu $projects_list install -am -DskipTests -Dcheckstyle.skip -Dspotbugs.skip -Drat.skip -Dmaven.javadoc.skip |
| |
| - name: Test - ${{ matrix.step_name }} |
| run: | |
| projects_list= |
| if [[ ! -z "${{ matrix.module }}" ]]; then |
| projects_list="-pl ${{ matrix.module }}" |
| fi |
| mvn -B -nsu $projects_list verify ${{ matrix.test_args }} |
| |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories |
| if: ${{ always() }} |
| uses: ./.github/actions/copy-test-reports |
| |
| - name: Publish Test Report |
| uses: apache/pulsar-test-infra/action-junit-report@master |
| if: ${{ always() }} |
| with: |
| report_paths: 'surefire-reports/TEST-*.xml' |
| annotate_only: 'true' |
| |
| - name: Upload Surefire reports |
| uses: actions/upload-artifact@v4 |
| if: failure() |
| continue-on-error: true |
| with: |
| name: unit-${{ matrix.step_name }}-reports |
| path: surefire-reports |
| retention-days: 7 |
| |
| - name: print JVM thread dumps when cancelled |
| if: cancelled() |
| run: ./dev/ci-tool print_thread_dumps |
| |
| integration-tests: |
| name: Integration Tests |
| runs-on: ubuntu-latest |
| timeout-minutes: 75 |
| needs: [ 'build-and-license-check' ] |
| if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| |
| - name: Clean Disk |
| uses: ./.github/actions/clean-disk |
| with: |
| mode: full |
| |
| - name: Cache local Maven repository |
| id: cache |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository/*/*/* |
| !~/.m2/repository/org/apache/bookkeeper |
| !~/.m2/repository/org/apache/distributedlog |
| key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} |
| |
| - name: Set up JDK 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: 11 |
| |
| - name: Pick ubuntu mirror for the docker image build |
| run: | |
| # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable |
| $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror |
| |
| - name: Build with Maven |
| run: mvn -B -nsu clean install -Pdocker -DskipTests |
| |
| - name: Run metadata driver tests |
| run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests |
| |
| - name: Run all integration tests (except backward compatibility tests) |
| run: | |
| mvn -B -nsu -f tests/pom.xml test -DintegrationTests -DredirectTestOutputToFile=false -DtestRetryCount=0 |
| |
| - name: print JVM thread dumps when cancelled |
| if: cancelled() |
| run: ./dev/ci-tool print_thread_dumps |
| |
| - name: Upload container logs on failure |
| uses: actions/upload-artifact@v4 |
| if: ${{ !success() }} |
| continue-on-error: true |
| with: |
| retention-days: 7 |
| name: integration-tests-container-logs |
| if-no-files-found: ignore |
| path: | |
| **/docker.log |
| |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories |
| if: ${{ always() }} |
| uses: ./.github/actions/copy-test-reports |
| |
| - name: Publish Test Report |
| uses: apache/pulsar-test-infra/action-junit-report@master |
| if: ${{ always() }} |
| with: |
| report_paths: 'surefire-reports/TEST-*.xml' |
| annotate_only: 'true' |
| |
| - name: Upload Surefire reports |
| uses: actions/upload-artifact@v4 |
| if: failure() |
| continue-on-error: true |
| with: |
| name: integration-tests-reports |
| path: surefire-reports |
| if-no-files-found: ignore |
| retention-days: 7 |
| |
| backward-compatibility-tests: |
| name: Backward compatibility tests |
| runs-on: ubuntu-latest |
| timeout-minutes: 75 |
| needs: [ 'build-and-license-check' ] |
| if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| |
| - name: Clean Disk |
| uses: ./.github/actions/clean-disk |
| with: |
| mode: full |
| |
| - name: Cache local Maven repository |
| id: cache |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository/*/*/* |
| !~/.m2/repository/org/apache/bookkeeper |
| !~/.m2/repository/org/apache/distributedlog |
| key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} |
| |
| - name: Set up JDK 8 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: 8 |
| |
| - name: Pick ubuntu mirror for the docker image build |
| run: | |
| # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable |
| $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror |
| |
| - name: Build with Maven |
| run: mvn -B -nsu clean install -Pdocker -DskipTests |
| |
| - name: Test current server with old clients |
| run: mvn -B -nsu -DbackwardCompatTests -pl :backward-compat-current-server-old-clients test |
| |
| - name: Test progressive upgrade |
| run: mvn -B -nsu -DbackwardCompatTests -pl :upgrade test |
| |
| - name: Other tests |
| run: | |
| mvn -B -nsu -DbackwardCompatTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct test |
| |
| - name: Upload container logs on failure |
| uses: actions/upload-artifact@v4 |
| if: ${{ !success() }} |
| continue-on-error: true |
| with: |
| retention-days: 7 |
| name: backward-compatibility-tests-container-logs |
| if-no-files-found: ignore |
| path: | |
| **/docker.log |
| |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories |
| if: ${{ always() }} |
| uses: ./.github/actions/copy-test-reports |
| |
| - name: Publish Test Report |
| uses: apache/pulsar-test-infra/action-junit-report@master |
| if: ${{ always() }} |
| with: |
| report_paths: 'surefire-reports/TEST-*.xml' |
| annotate_only: 'true' |
| |
| - name: Upload Surefire reports |
| uses: actions/upload-artifact@v4 |
| if: failure() |
| continue-on-error: true |
| with: |
| name: backward-compatibility-tests-reports |
| path: surefire-reports |
| if-no-files-found: ignore |
| retention-days: 7 |
| |
| windows-build: |
| name: Build with windows on JDK 11 |
| runs-on: windows-latest |
| timeout-minutes: 30 |
| needs: [ 'build-and-license-check' ] |
| if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| |
| - name: Install mingw |
| run: choco install mingw |
| |
| - name: Cache local Maven repository |
| id: cache |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository/*/*/* |
| !~/.m2/repository/org/apache/bookkeeper |
| !~/.m2/repository/org/apache/distributedlog |
| key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} |
| |
| - name: Set up JDK 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: 11 |
| |
| - name: mvn package |
| run: mvn -B -nsu clean package -DskipTests |
| |
| macos-build: |
| name: Build with macos on JDK 11 |
| runs-on: macos-latest |
| timeout-minutes: 30 |
| needs: [ 'build-and-license-check' ] |
| if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| |
| - name: Cache local Maven repository |
| id: cache |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository/*/*/* |
| !~/.m2/repository/org/apache/bookkeeper |
| !~/.m2/repository/org/apache/distributedlog |
| key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} |
| |
| - name: Set up JDK 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: 11 |
| |
| - name: mvn package |
| run: mvn -B -nsu clean package -DskipTests |
| |
| jdk-compatibility-checks: |
| name: ${{ matrix.step_name }} |
| runs-on: ubuntu-latest |
| timeout-minutes: ${{ matrix.timeout || 60 }} |
| needs: [ 'build-and-license-check' ] |
| if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} |
| strategy: |
| fail-fast: false |
| matrix: |
| include: |
| - step_name: Compatibility Check Java8 |
| jdk_version: 8 |
| - step_name: Compatibility Check Java11 |
| jdk_version: 11 |
| - step_name: Compatibility Check Java17 |
| jdk_version: 17 |
| - step_name: Compatibility Check Java21 |
| jdk_version: 21 |
| |
| steps: |
| - name: checkout |
| uses: actions/checkout@v4 |
| |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| |
| - name: Cache local Maven repository |
| id: cache |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository/*/*/* |
| !~/.m2/repository/org/apache/bookkeeper |
| !~/.m2/repository/org/apache/distributedlog |
| key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} |
| |
| - name: Set up JDK ${{ matrix.jdk_version }} |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ matrix.jdk_version }} |
| |
| - name: Build with Maven |
| run: mvn clean package -B -nsu -DskipBookKeeperServerTests |
| |
| - name: print JVM thread dumps when cancelled |
| if: cancelled() |
| run: ./dev/ci-tool print_thread_dumps |
| |
| typo-check: |
| name: Typo Check |
| runs-on: ubuntu-latest |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Check typos |
| uses: crate-ci/typos@master |
| |
| owasp-dependency-check: |
| name: OWASP Dependency Check |
| runs-on: ubuntu-latest |
| timeout-minutes: 60 |
| needs: [ 'build-and-license-check' ] |
| if: ${{ needs.build-and-license-check.outputs.need_owasp == 'true' }} |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| |
| - name: Cache local Maven repository |
| id: cache |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository/*/*/* |
| !~/.m2/repository/org/apache/bookkeeper |
| !~/.m2/repository/org/apache/distributedlog |
| key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} |
| |
| - name: Set up JDK 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: 21 |
| |
| - name: run "clean install verify" to trigger dependency check |
| # excluding dlfs because it includes hadoop lib with |
| # CVEs that we cannot patch up anyway |
| run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs' |
| |
| - name: Upload report |
| uses: actions/upload-artifact@v4 |
| if: ${{ cancelled() || failure() }} |
| continue-on-error: true |
| with: |
| name: dependency report |
| path: target/dependency-check-report.html |
| retention-days: 7 |