.github/workflows/ci.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.

name: CI

on:
  push:
  pull_request:
    branches: [ "master" ]

env:
  MAVEN_OPTS: -Xmx5g -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  build-17:
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v6
      - name: Cache for maven dependencies
        uses: actions/cache@v5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/ranger
          key: maven-repo-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-repo-
      - name: Set up JDK 17
        uses: actions/setup-java@v5
        with:
          java-version: '17'
          distribution: 'temurin'
      - name: Clear npm cache
        run: npm cache clean --force
      - name: build (17)
        run: |
          set -o pipefail
          mvn -T 8 clean verify --no-transfer-progress -B -V 2>&1 | tee "$RUNNER_TEMP/build.log"

      # fail on stale or non-portable patterns in assembly descriptors (distro/src/main/assembly)
      - name: check assembly warnings
        run: |
          patterns='patterns were never triggered|filesystem-root relative reference'
          # -a: build.log has binary bytes from test output, without it grep prints only "binary file matches"
          if grep -aEq "$patterns" "$RUNNER_TEMP/build.log"; then
            echo "::error::Assembly descriptors produced warnings; fix the patterns listed below"
            # each archive's warnings are logged just before its "Building tar/zip" line; drop repeats per archive
            grep -aE "$patterns|^o +'|Building (tar|zip):" "$RUNNER_TEMP/build.log" | awk '/Building/ { print; delete seen; next } !seen[$0]++'
            exit 1
          fi

      - name: run code coverage
        run: |
          cd dev-support/checks
          chmod +x coverage.sh && ./coverage.sh

      - name: upload coverage results
        uses: actions/upload-artifact@v7
        with:
          name: jacoco
          path: target/coverage/all/*

      - name: Upload artifacts
        uses: actions/upload-artifact@v7
        with:
          name: target-17
          path: target/*
  
  services-docker-build:
    needs:
      - build-17
    runs-on: ubuntu-24.04 # bumping os version for more disk space
    timeout-minutes: 60 # bumping timeout to allow containers to come up
    steps:
      - uses: actions/checkout@v6
      - name: Download build-17 artifacts
        uses: actions/download-artifact@v8
        with:
          name: target-17
      
      - name: Copy artifacts for docker build
        run: |
          mv ranger-*-admin.tar.gz dev-support/ranger-docker/dist
          mv ranger-*-pdp.tar.gz dev-support/ranger-docker/dist
          mv ranger-*-usersync.tar.gz dev-support/ranger-docker/dist
          mv ranger-*-tagsync.tar.gz dev-support/ranger-docker/dist
          mv ranger-*-kms.tar.gz dev-support/ranger-docker/dist
          # Audit index stores are only brought up via the audit service, which
          # also needs Kafka + the audit ingestor/dispatcher images.
          mv ranger-*-kafka-plugin.tar.gz dev-support/ranger-docker/dist
          mv ranger-*-audit-ingestor.tar.gz dev-support/ranger-docker/dist
          mv ranger-*-audit-dispatcher.tar.gz dev-support/ranger-docker/dist
          mv version dev-support/ranger-docker/dist
          rm -f ranger-*.tar.gz # clean up workspace

      - name: Free up disk space
        run: docker system prune --all --force --volumes

      - name: Download DB and Log4J connectors
        run: |
          cd dev-support/ranger-docker
          # Kafka archive is needed to build the ranger-kafka image (audit pipeline).
          ./download-archives.sh kafka
      
      - name: Build all ranger-service images
        run: |
          cd dev-support/ranger-docker
          export RANGER_DB_TYPE=postgres
          export AUDIT_INDEX_STORE=opensearch
          export AUDIT_DESTINATIONS=audit-store-opensearch
          docker compose --profile ${AUDIT_DESTINATIONS} \
          -f docker-compose.ranger.yml \
          -f docker-compose.ranger-audit-service.yml \
          -f docker-compose.ranger-pdp.yml \
          -f docker-compose.ranger-usersync.yml \
          -f docker-compose.ranger-tagsync.yml \
          -f docker-compose.ranger-kms.yml build

      - name: Bring up ranger-service containers
        run: |
          cd dev-support/ranger-docker
          export RANGER_DB_TYPE=postgres
          export AUDIT_INDEX_STORE=opensearch
          export AUDIT_DESTINATIONS=audit-store-opensearch
          docker compose --profile ${AUDIT_DESTINATIONS} \
          -f docker-compose.ranger.yml \
          -f docker-compose.ranger-audit-service.yml \
          -f docker-compose.ranger-pdp.yml \
          -f docker-compose.ranger-usersync.yml \
          -f docker-compose.ranger-tagsync.yml \
          -f docker-compose.ranger-kms.yml up -d

      - name: Check status of containers and remove them
        run: |
          sleep 60
          containers=(ranger ranger-zk ranger-opensearch ranger-postgres ranger-pdp ranger-usersync ranger-tagsync ranger-kms ranger-kafka ranger-audit-ingestor ranger-audit-dispatcher-opensearch);
          flag=true;
          for container in "${containers[@]}"; do
              if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then
                  echo "Container $container is running!";
              else
                  flag=false;
                  echo "Container $container is NOT running! Status: $(docker inspect -f '{{.State.Status}}' $container 2>/dev/null)";
                  echo "docker logs $container:";
                  docker logs $container
              fi
          done
          
          if [[ $flag == true ]]; then
              echo "All required containers are up and running";
              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
          else
              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
              exit 1;
          fi

  # Download large plugin distro archives in parallel with build-17 so a slow
  # archive.apache.org mirror does not consume the docker-build job budget.
  plugins-download-archives:
    runs-on: ubuntu-24.04
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v6

      - name: Cache downloaded archives
        uses: actions/cache@v5
        with:
          path: dev-support/ranger-docker/downloads
          key: ${{ runner.os }}-ranger-downloads-${{ hashFiles('dev-support/ranger-docker/.env') }}
          restore-keys: |
            ${{ runner.os }}-ranger-downloads-

      # Populates downloads/ for plugins-docker-build; Ozone tarball + extract dir are
      # cached together (extractOzoneIfNeeded in download-archives.sh).
      - name: Run download-archives.sh
        run: |
          cd dev-support/ranger-docker
          ./download-archives.sh hadoop hive hbase kafka knox ozone opensearch

  plugins-docker-build:
    needs:
      - build-17
      - plugins-download-archives
    runs-on: ubuntu-24.04 # bumping os version for more disk space
    timeout-minutes: 60 # bumping timeout to allow containers to come up
    steps:
      - uses: actions/checkout@v6
      - name: Download build-17 artifacts
        uses: actions/download-artifact@v8
        with:
          name: target-17

      - name: Copy artifacts for docker build
        run: |
          mv ranger-*-admin.tar.gz dev-support/ranger-docker/dist
          mv ranger-*-plugin.tar.gz dev-support/ranger-docker/dist
          # Audit index stores are only brought up via the audit service, which
          # also needs the audit ingestor/dispatcher images.
          mv ranger-*-audit-ingestor.tar.gz dev-support/ranger-docker/dist
          mv ranger-*-audit-dispatcher.tar.gz dev-support/ranger-docker/dist
          mv version dev-support/ranger-docker/dist
          rm -f ranger-*.tar.gz # clean up workspace

      - name: Free up disk space
        run: docker system prune --all --force --volumes

      - name: Restore plugin archives from cache
        uses: actions/cache@v5
        with:
          path: dev-support/ranger-docker/downloads
          key: ${{ runner.os }}-ranger-downloads-${{ hashFiles('dev-support/ranger-docker/.env') }}
          restore-keys: |
            ${{ runner.os }}-ranger-downloads-

      # Idempotent: skips downloads already in the Actions cache; Ozone re-extract runs
      # only when the tarball or extract dir is missing/stale (see extractOzoneIfNeeded).
      - name: Verify plugin archives (download any cache misses)
        run: |
          cd dev-support/ranger-docker
          ./download-archives.sh hadoop hive hbase kafka knox ozone

      - name: Build all ranger-plugin images
        run: |
          cd dev-support/ranger-docker
          export RANGER_DB_TYPE=postgres
          export AUDIT_INDEX_STORE=opensearch
          export AUDIT_DESTINATIONS=audit-store-opensearch
          docker compose --profile ${AUDIT_DESTINATIONS} \
          -f docker-compose.ranger.yml \
          -f docker-compose.ranger-audit-service.yml \
          -f docker-compose.ranger-hadoop.yml \
          -f docker-compose.ranger-hbase.yml \
          -f docker-compose.ranger-hive.yml \
          -f docker-compose.ranger-knox.yml \
          -f docker-compose.ranger-ozone.yml build

      - name: Bring up ranger-plugin containers
        run: |
          cd dev-support/ranger-docker
          ./scripts/ozone/ozone-plugin-docker-setup.sh
          export RANGER_DB_TYPE=postgres
          export AUDIT_INDEX_STORE=opensearch
          export AUDIT_DESTINATIONS=audit-store-opensearch
          docker compose --profile ${AUDIT_DESTINATIONS} \
          -f docker-compose.ranger.yml \
          -f docker-compose.ranger-audit-service.yml \
          -f docker-compose.ranger-hadoop.yml \
          -f docker-compose.ranger-hbase.yml \
          -f docker-compose.ranger-hive.yml \
          -f docker-compose.ranger-knox.yml \
          -f docker-compose.ranger-ozone.yml up -d

      - name: Wait for KDC Kerberos readiness
        run: |
          cd dev-support/ranger-docker
          for i in $(seq 1 90); do
            status=$(docker inspect -f '{{.State.Health.Status}}' ranger-kdc 2>/dev/null || echo missing)
            if [ "$status" = "healthy" ]; then
              echo "ranger-kdc is healthy after ${i} attempt(s)"
              exit 0
            fi
            echo "Waiting for ranger-kdc (status=${status})..."
            sleep 2
          done
          echo "ranger-kdc did not become healthy in time"
          docker logs ranger-kdc || true
          exit 1

      - name: Check status of containers and remove them
        run: |
          sleep 60
          containers=(ranger ranger-zk ranger-opensearch ranger-postgres ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode ranger-audit-ingestor ranger-audit-dispatcher-opensearch);
          flag=true;
          for container in "${containers[@]}"; do
              if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then
                  echo "Container $container is running!";
              else
                  flag=false;
                  echo "Container $container is NOT running! Status: $(docker inspect -f '{{.State.Status}}' $container 2>/dev/null)";
                  echo "docker logs $container:";
                  docker logs $container
              fi
          done

          if [[ $flag == true ]]; then
              echo "All required containers are up and running";
              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
          else
              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
              exit 1;
          fi