| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| name: Package Linux |
| |
| on: |
| push: |
| branches: |
| - '**' |
| - '!dependabot/**' |
| - '!release-*' |
| paths: |
| - '.github/workflows/check_labels.yml' |
| - '.github/workflows/package_linux.yml' |
| - '.github/workflows/report_ci.yml' |
| - 'cpp/**' |
| - 'c_glib/**' |
| - 'dev/archery/archery/**' |
| - 'dev/release/binary-task.rb' |
| - 'dev/release/verify-apt.sh' |
| - 'dev/release/verify-yum.sh' |
| - 'dev/tasks/linux-packages/**' |
| - 'format/Flight.proto' |
| tags: |
| - "apache-arrow-*-rc*" |
| pull_request: |
| # This trigger ensures that the `check_labels` workflow runs on manual label changes |
| types: |
| - labeled |
| - opened |
| - reopened |
| - synchronize |
| schedule: |
| - cron: "0 2 * * *" |
| |
| concurrency: |
| group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} |
| cancel-in-progress: true |
| |
| permissions: |
| actions: read |
| contents: read |
| pull-requests: read |
| |
| jobs: |
| check-labels: |
| uses: ./.github/workflows/check_labels.yml |
| with: |
| parent-workflow: package_linux |
| |
| check-enabled: |
| # Check whether the CI builds in this workflow need to be run |
| needs: check-labels |
| runs-on: ubuntu-latest |
| outputs: |
| is_enabled: ${{ steps.set_enabled.outputs.is_enabled }} |
| steps: |
| - id: set_enabled |
| if: >- |
| github.ref_type == 'tag' || |
| (github.event_name == 'schedule' && github.repository == 'apache/arrow') || |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || |
| contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: Package: Linux') |
| run: echo "is_enabled=true" >> "$GITHUB_OUTPUT" |
| |
| package: |
| needs: check-enabled |
| if: needs.check-enabled.outputs.is_enabled == 'true' |
| permissions: |
| # Upload artifacts to GitHub Release |
| contents: write |
| # Upload cached Docker images to GitHub Packages |
| packages: write |
| name: ${{ matrix.id }} |
| runs-on: ${{ contains(matrix.id, 'amd64') && 'ubuntu-latest' || 'ubuntu-24.04-arm' }} |
| timeout-minutes: 160 |
| strategy: |
| fail-fast: false |
| matrix: |
| id: |
| - almalinux-8-amd64 |
| - almalinux-8-arm64 |
| - almalinux-9-amd64 |
| - almalinux-9-arm64 |
| - almalinux-10-amd64 |
| - almalinux-10-arm64 |
| - amazon-linux-2023-amd64 |
| - amazon-linux-2023-arm64 |
| - centos-9-stream-amd64 |
| - centos-9-stream-arm64 |
| - debian-trixie-amd64 |
| - debian-trixie-arm64 |
| - debian-forky-amd64 |
| - debian-forky-arm64 |
| - ubuntu-jammy-amd64 |
| - ubuntu-jammy-arm64 |
| - ubuntu-noble-amd64 |
| - ubuntu-noble-arm64 |
| - ubuntu-resolute-amd64 |
| - ubuntu-resolute-arm64 |
| env: |
| BUILD_DIR: "${{ github.workspace }}/packages.build" |
| steps: |
| - name: Checkout Arrow |
| uses: actions/checkout@v7 |
| with: |
| persist-credentials: false |
| fetch-depth: 0 |
| submodules: recursive |
| - name: Free up disk space |
| run: | |
| ci/scripts/util_free_space.sh |
| - name: Prepare environment variables |
| env: |
| ID: ${{ matrix.id }} |
| run: | |
| set -x |
| |
| case "${ID}" in |
| centos-*) |
| # Example: centos-9-stream-amd64 -> centos |
| distribution="${ID%%-*}" |
| ;; |
| *) |
| # Example: almalinux-8-amd64 -> almalinux |
| # Example: amazon-linux-2023-amd64 -> amazon-linux |
| distribution="${ID%-*-*}" |
| ;; |
| esac |
| echo "DISTRIBUTION=${distribution}" >> "${GITHUB_ENV}" |
| |
| # Example: almalinux-8-amd64 -> amd64 |
| architecture="${ID##*-}" |
| echo "ARCHITECTURE=${architecture}" >> "${GITHUB_ENV}" |
| |
| # Example: almalinux-8-amd64 -> almalinux-8 |
| target="${ID%-*}" |
| case "${target}" in |
| almalinux-*|amazon-linux-*|centos-*) |
| echo "TASK_NAMESPACE=yum" >> "${GITHUB_ENV}" |
| # Example: centos-9-stream-amd64 -> centos-9-stream |
| # Example: amazon-linux-2023-amd64 -> amazon-linux-2023 |
| version="${ID%-*}" |
| # Example: centos-9-stream -> 9-stream |
| # Example: amazon-linux-2023 -> 2023 |
| version="${version##${distribution}-}" |
| echo "DISTRIBUTION_VERSION=${version}" >> "${GITHUB_ENV}" |
| if [ "${architecture}" = "arm64" ]; then |
| # Example: almalinux-8 -> almalinux-8-aarch64 |
| target="${target}-aarch64" |
| fi |
| echo "YUM_TARGETS=${target}" >> "${GITHUB_ENV}" |
| ;; |
| *) |
| echo "TASK_NAMESPACE=apt" >> "${GITHUB_ENV}" |
| # Example: debian-bookworm-amd64 -> debian-bookworm |
| code_name="${ID%-*}" |
| # Example: debian-bookworm -> bookworm |
| code_name="${code_name#*-}" |
| echo "DISTRIBUTION_CODE_NAME=${code_name}" >> "${GITHUB_ENV}" |
| if [ "${architecture}" = "arm64" ]; then |
| # Example: ubuntu-noble -> ubuntu-noble-arm64 |
| target="${target}-arm64" |
| fi |
| echo "APT_TARGETS=${target}" >> "${GITHUB_ENV}" |
| ;; |
| esac |
| echo "TARGET=${target}" >> "${GITHUB_ENV}" |
| |
| if [ "${GITHUB_REF_TYPE}" = "tag" ]; then |
| # Example: apache-arrow-21.0.0-rc0 -> 21.0.0-rc0 |
| version="${GITHUB_REF_NAME#apache-arrow-}" |
| echo "ARROW_VERSION=${version}" >> "${GITHUB_ENV}" |
| fi |
| - name: Restore ccache |
| uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 |
| with: |
| path: ${{ env.BUILD_DIR }}/${{ env.TARGET }} |
| key: package-linux-${{ matrix.id }} |
| - name: Install dependencies |
| run: | |
| sudo apt update |
| sudo apt install -y \ |
| rake \ |
| reprotest \ |
| ruby \ |
| ruby-dev |
| - name: Allow unprivileged user namespaces |
| if: env.TASK_NAMESPACE == 'apt' |
| # Reprotest's domain_host variation uses `unshare -r --uts`, |
| # which writes to /proc/self/uid_map. Ubuntu 24.04 restricts |
| # unprivileged user namespaces via AppArmor by default, so |
| # the call fails. Try to lift the restriction so reprotest |
| # can run the full set of variations. |
| run: | |
| sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 |
| - name: Prepare apache-arrow-apt-source for arm64 |
| if: env.ARCHITECTURE == 'arm64' |
| run: | |
| pushd dev/tasks/linux-packages/apache-arrow-apt-source/apt |
| for target in *-*; do |
| cp -a ${target} ${target}-arm64 |
| done |
| popd |
| - name: Prepare apache-arrow-release for arm64 |
| if: env.ARCHITECTURE == 'arm64' |
| run: | |
| pushd dev/tasks/linux-packages/apache-arrow-release/yum |
| for target in *-*; do |
| cp -a ${target} ${target}-aarch64 |
| done |
| popd |
| - name: Update version |
| if: github.ref_type != 'tag' |
| run: | |
| pushd dev/tasks/linux-packages |
| rake version:update |
| popd |
| - name: Login to GitHub Container registry |
| uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 |
| with: |
| registry: ghcr.io |
| username: ${{ github.actor }} |
| password: ${{ secrets.GITHUB_TOKEN }} |
| - name: Wait for creating GitHub Release |
| if: github.ref_type == 'tag' |
| env: |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| run: | |
| dev/release/utils-watch-gh-workflow.sh \ |
| ${GITHUB_REF_NAME} \ |
| release_candidate.yml |
| - name: Build |
| env: |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| run: | |
| rake -C dev/tasks/linux-packages/apache-arrow docker:pull:${TARGET} || : |
| rake -C dev/tasks/linux-packages ${TASK_NAMESPACE}:build |
| - name: Save ccache |
| if: >- |
| !cancelled() |
| continue-on-error: true |
| uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 |
| with: |
| path: ${{ env.BUILD_DIR }}/${{ env.TARGET }} |
| key: package-linux-${{ matrix.id }} |
| include-hidden-files: true |
| - name: Docker Push |
| continue-on-error: true |
| if: >- |
| success() && |
| github.event_name == 'push' && |
| github.ref_name == 'main' |
| run: | |
| rake -C dev/tasks/linux-packages/apache-arrow docker:push:${TARGET} |
| - name: Build artifact tarball |
| run: | |
| mkdir -p "${DISTRIBUTION}" |
| cp -a \ |
| dev/tasks/linux-packages/*/${TASK_NAMESPACE}/repositories/${DISTRIBUTION}/* \ |
| "${DISTRIBUTION}/" |
| set -x |
| # We use latest .deb/.rpm of |
| # apache-arrow-apt-source/apache-arrow-release built for |
| # amd64 because they are architecture independent. |
| if [ "${ARCHITECTURE}" = "amd64" ]; then |
| if [ "${TASK_NAMESPACE}" = "apt" ]; then |
| # Create |
| # https://packages.apache.org/artifactory/arrow/${DISTRIBUTION}/apache-arrow-apt-source-latest-${DISTRIBUTION_CODE_NAME}.deb |
| # for easy to install. |
| cp -a \ |
| ${DISTRIBUTION}/pool/${DISTRIBUTION_CODE_NAME}/*/a/apache-arrow-apt-source/*.deb \ |
| ${DISTRIBUTION}/apache-arrow-apt-source-latest-${DISTRIBUTION_CODE_NAME}.deb |
| else |
| # Create |
| # https://packages.apache.org/artifactory/arrow/${DISTRIBUTION}/${DISTRIBUTION_VERSION}/apache-arrow-release-latest.rpm |
| # for easy to install. |
| cp -a \ |
| ${DISTRIBUTION}/${DISTRIBUTION_VERSION}/x86_64/Packages/apache-arrow-release-*.rpm \ |
| ${DISTRIBUTION}/${DISTRIBUTION_VERSION}/apache-arrow-release-latest.rpm |
| fi |
| fi |
| tar cvzf ${{ matrix.id }}.tar.gz ${DISTRIBUTION} |
| dev/release/utils-generate-checksum.sh ${{ matrix.id }}.tar.gz |
| - name: Upload the artifacts to the job |
| uses: actions/upload-artifact@v7 |
| with: |
| name: ${{ matrix.id }} |
| path: ${{ matrix.id }}.tar.gz* |
| - name: Upload the artifacts to GitHub Release |
| if: github.ref_type == 'tag' |
| env: |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| run: | |
| gh release upload ${GITHUB_REF_NAME} \ |
| --clobber \ |
| ${{ matrix.id }}.tar.gz* |
| - name: Set up test |
| run: | |
| sudo apt install -y \ |
| apt-utils \ |
| cpio \ |
| createrepo-c \ |
| devscripts \ |
| gpg \ |
| rpm \ |
| rsync |
| gem install --user-install apt-dists-merge |
| { |
| echo "Key-Type: RSA" |
| echo "Key-Length: 4096" |
| echo "Name-Real: Test" |
| echo "Name-Email: test@example.com" |
| echo "%no-protection" |
| } | gpg --full-generate-key --batch |
| GPG_KEY_ID=$(gpg --list-keys --with-colon test@example.com | grep fpr | cut -d: -f10) |
| echo "GPG_KEY_ID=${GPG_KEY_ID}" >> ${GITHUB_ENV} |
| if [ "${TASK_NAMESPACE}" = "yum" ]; then |
| repositories_dir=dev/tasks/linux-packages/apache-arrow-release/yum/repositories |
| rpm2cpio ${repositories_dir}/*/*/*/Packages/apache-arrow-release-*.noarch.rpm | |
| cpio -id |
| mv etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow \ |
| dev/tasks/linux-packages/KEYS |
| fi |
| gpg --export --armor test@example.com >> dev/tasks/linux-packages/KEYS |
| - name: Test |
| run: | |
| pushd dev/tasks/linux-packages |
| rake --trace ${TASK_NAMESPACE}:test |
| popd |
| - name: Verify Reproducibility |
| if: env.TASK_NAMESPACE == 'apt' |
| env: |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| run: | |
| # Validate reproducibility. Reprotest runs the build twice |
| # inside its own tempdir and doesn't copy artifacts back, |
| # so this is purely a verification step. |
| reprotest \ |
| --vary=-fileordering \ |
| --build-command \ |
| "rake -C dev/tasks/linux-packages ${TASK_NAMESPACE}:build" \ |
| "${PWD}" \ |
| "dev/tasks/linux-packages/*/apt/repositories/${DISTRIBUTION}/pool/${DISTRIBUTION_CODE_NAME}/*/*/*/*.deb" |
| |
| report-package-linux: |
| if: github.event_name == 'schedule' && always() |
| needs: |
| - package |
| uses: ./.github/workflows/report_ci.yml |
| secrets: |
| ARROW_SMTP_PASSWORD: ${{ secrets.ARROW_SMTP_PASSWORD }} |
| ARROW_ZULIP_WEBHOOK: ${{ secrets.ARROW_ZULIP_WEBHOOK }} |
