| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| name: Upload Java Nightly builds |
| |
| on: |
| workflow_dispatch: |
| inputs: |
| prefix: |
| description: Job prefix to use. |
| required: false |
| default: '' |
| keep: |
| description: Number of versions to keep. |
| required: false |
| default: 14 |
| schedule: |
| - cron: '0 14 * * *' |
| |
| permissions: |
| contents: read |
| |
| jobs: |
| upload: |
| if: github.repository == 'apache/arrow' |
| env: |
| PREFIX: ${{ github.event.inputs.prefix || ''}} |
| CROSSBOW_GITHUB_TOKEN: ${{ github.token }} |
| runs-on: ubuntu-latest |
| steps: |
| - name: Checkout Arrow |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 1 |
| path: arrow |
| repository: apache/arrow |
| ref: main |
| submodules: recursive |
| - name: Checkout Crossbow |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 0 |
| path: crossbow |
| repository: ursacomputing/crossbow |
| ref: main |
| - name: Set up Python |
| uses: actions/setup-python@v4 |
| with: |
| cache: 'pip' |
| python-version: 3.8 |
| - name: Install Archery |
| shell: bash |
| run: pip install -e arrow/dev/archery[all] |
| - run: mkdir -p binaries |
| - name: Download Artifacts |
| run: | |
| if [ -z $PREFIX ]; then |
| PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0 |
| fi |
| echo $PREFIX |
| archery crossbow download-artifacts -f java-jars -t binaries $PREFIX |
| - name: Cache Repo |
| uses: actions/cache@v3 |
| with: |
| path: repo |
| key: java-nightly-${{ github.run_id }} |
| restore-keys: java-nightly |
| - name: Sync from Remote |
| uses: ./arrow/.github/actions/sync-nightlies |
| with: |
| switches: -avzh --update --delete --progress |
| local_path: repo |
| remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java |
| remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} |
| remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} |
| remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} |
| remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} |
| remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} |
| - shell: bash |
| name: Show local repo sync from remote |
| run: | |
| for i in `ls -t repo/org/apache/arrow`; do |
| echo "- $i: $(find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d \ |
| | wc -l \ |
| | xargs) versions available" |
| done |
| - shell: bash |
| name: Build Repository |
| run: | |
| DATE=$(date +%Y-%m-%d) |
| if [ -z $PREFIX ]; then |
| PREFIX=nightly-packaging-${DATE}-0 |
| fi |
| PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+-SNAPSHOT)' |
| mkdir -p repo/org/apache/arrow/ |
| for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E '.jar|.json|.pom|.xml' | grep SNAPSHOT); do |
| [[ $LIBRARY =~ $PATTERN_TO_GET_LIB_AND_VERSION ]] |
| mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} |
| mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} |
| # Copy twice to maintain a latest snapshot and some earlier versions |
| cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} |
| touch repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} |
| cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} |
| echo "Artifacts $LIBRARY configured" |
| done |
| - name: Prune Repository |
| shell: bash |
| env: |
| KEEP: ${{ github.event.inputs.keep || 14 }} |
| run: | |
| for i in `ls -t repo/org/apache/arrow`; do |
| find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d -print0 \ |
| | xargs -0 ls -t -d \ |
| | tail -n +$((KEEP + 1)) \ |
| | xargs rm -rf |
| done |
| - name: Show repo contents |
| run: tree repo |
| - name: Sync to Remote |
| if: ${{ github.repository == 'apache/arrow' }} |
| uses: ./arrow/.github/actions/sync-nightlies |
| with: |
| upload: true |
| switches: -avzh --update --delete --progress |
| local_path: repo |
| remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java |
| remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} |
| remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} |
| remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} |
| remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} |
| remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} |