Remove pull_request_target workflow
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
deleted file mode 100644
index 1877d97..0000000
--- a/.github/workflows/build-images.yml
+++ /dev/null
@@ -1,343 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: "Build Images"
-on: # yamllint disable-line rule:truthy
- schedule:
- - cron: '28 0 * * *'
- pull_request_target:
- branches:
- - main
- push:
- branches: ['main', 'v[0-9]+-[0-9]+-test']
-permissions:
- # all other permissions are set to none
- contents: read
-env:
- MOUNT_SELECTED_LOCAL_SOURCES: "false"
- FORCE_ANSWER_TO_QUESTIONS: "yes"
- FORCE_PULL_IMAGES: "false"
- CHECK_IMAGE_FOR_REBUILD: "true"
- SKIP_CHECK_REMOTE_IMAGE: "true"
- DB_RESET: "true"
- VERBOSE: "true"
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_USERNAME: ${{ github.actor }}
- # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
- # Airflow one is going to be used
- CONSTRAINTS_GITHUB_REPOSITORY: >-
- ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
- secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
- # This token is WRITE one - pull_request_target type of events always have the WRITE token
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
- INSTALL_PROVIDERS_FROM_SOURCES: "true"
- AIRFLOW_LOGIN_TO_GITHUB_REGISTRY: "true"
- GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.pull_request.head.sha || github.sha }}
-
-concurrency:
- group: build-${{ github.event.pull_request.number || github.ref }}
- cancel-in-progress: true
-
-jobs:
-
- build-info:
- timeout-minutes: 10
- name: "Build Info"
- runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }}
- env:
- targetBranch: ${{ github.event.pull_request.base.ref }}
- pullRequestLabels: "${{ toJSON(github.event.pull_request.labels.*.name) }}"
- outputs:
- runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }}
- pythonVersions: "${{ steps.selective-checks.python-versions }}"
- upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
- defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
- run-tests: ${{ steps.selective-checks.outputs.run-tests }}
- run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
- image-build: ${{ steps.dynamic-outputs.outputs.image-build }}
- cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
- targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }}
- defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
- targetCommitSha: "${{steps.discover-pr-merge-commit.outputs.targetCommitSha ||
- github.event.pull_request.head.sha ||
- github.sha
- }}"
- steps:
- - name: Discover PR merge commit
- id: discover-pr-merge-commit
- run: |
- TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
- echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> $GITHUB_ENV
- echo "::set-output name=targetCommitSha::${TARGET_COMMIT_SHA}"
- if: github.event_name == 'pull_request_target'
- # Retrieve it to be able to determine which files has changed in the incoming commit of the PR
- # we checkout the target commit and it's parent to be able to compare them
- - uses: actions/checkout@v2
- with:
- ref: ${{ env.TARGET_COMMIT_SHA }}
- persist-credentials: false
- fetch-depth: 2
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: env
- run: printenv
- env:
- dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }}
- GITHUB_CONTEXT: ${{ toJson(github) }}
- - name: Selective checks
- id: selective-checks
- env:
- PR_LABELS: ${{ env.pullRequestLabels }}
- run: |
- if [[ ${GITHUB_EVENT_NAME} == "pull_request_target" ]]; then
- # Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
- else
- # Run all checks
- ./scripts/ci/selective_ci_checks.sh
- fi
- - name: Compute dynamic outputs
- id: dynamic-outputs
- run: |
- set -x
- if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then
- echo "::set-output name=targetBranch::${targetBranch}"
- else
- # Direct push to branch, or scheduled build
- echo "::set-output name=targetBranch::${GITHUB_REF#refs/heads/}"
- fi
-
- if [[ "${{ github.event_name }}" == 'schedule' ]]; then
- echo "::set-output name=cacheDirective::disabled"
- else
- echo "::set-output name=cacheDirective::pulled"
- fi
-
- if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" && "$BUILD_IMAGES_OVERRIDE" != "false" ]]; then
- echo "::set-output name=image-build::true"
- else
- echo "::set-output name=image-build::false"
- fi
- env:
- SELECTIVE_CHECKS_IMAGE_BUILD: ${{ steps.selective-checks.outputs.image-build }}
- BUILD_IMAGES_OVERRIDE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE }}
-
- build-ci-images:
- permissions:
- packages: write
- timeout-minutes: 80
- name: "Build CI images ${{matrix.python-version}}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- strategy:
- matrix:
- # We need to attempt to build all possible versions here because pull_request_target
- # event is run for both main and v1-10-tests
- python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }}
- fail-fast: true
- if: needs.build-info.outputs.image-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
- BACKEND: postgres
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
- CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: >
- ${{ github.event_name == 'pull_request_target' && 'false' || 'true' }}
- outputs: ${{toJSON(needs.build-info.outputs) }}
- steps:
- - uses: actions/checkout@v2
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- - name: >
- Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
- to use ci/scripts from there.
- uses: actions/checkout@v2
- with:
- path: "main-airflow"
- ref: "${{ needs.build-info.outputs.targetBranch }}"
- persist-credentials: false
- submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- - name: >
- Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
- so that the PR does not override it
- # We should not override those scripts which become part of the image as they will not be
- # changed in the image built - we should only override those that are executed to build
- # the image.
- run: |
- rm -rf "scripts/ci"
- mv "main-airflow/scripts/ci" "scripts"
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Build CI images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Push CI images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_push_ci_images.sh
- # Remove me on 15th of August 2021 after all users had chance to rebase
- - name: "Push Legacy CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}"
- run: ./scripts/ci/images/ci_push_legacy_ci_images.sh
- if: github.event_name == 'pull_request_target'
-
- build-prod-images:
- permissions:
- packages: write
- timeout-minutes: 80
- name: "Build PROD images ${{matrix.python-version}}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, build-ci-images]
- strategy:
- matrix:
- # We need to attempt to build all possible versions here because pull_request_target
- # event is run for both main and v1-10-tests
- python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }}
- fail-fast: true
- if: needs.build-info.outputs.image-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
- BACKEND: postgres
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
- CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: >
- ${{ github.event_name == 'pull_request_target' && 'false' || 'true' }}
- VERSION_SUFFIX_FOR_PYPI: ".dev0"
- INSTALL_PROVIDERS_FROM_SOURCES: >
- ${{ needs.build-info.outputs.defaultBranch == 'main' && 'true' || 'false' }}
- steps:
- - uses: actions/checkout@v2
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- - name: >
- Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
- to use ci/scripts from there.
- uses: actions/checkout@v2
- with:
- path: "main-airflow"
- ref: "${{ needs.build-info.outputs.targetBranch }}"
- persist-credentials: false
- submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- - name: >
- Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
- so that the PR does not override it
- # We should not override those scripts which become part of the image as they will not be
- # changed in the image built - we should only override those that are executed to build
- # the image.
- run: |
- rm -rf "scripts/ci"
- mv "main-airflow/scripts/ci" "scripts"
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Build CI images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- # Pull images built in the previous step
- env:
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true"
- # Here we are using PULL_IMAGE_TAG set in the environment variables above
- - name: "Build PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- env:
- # GITHUB_REGISTRY_PULL_IMAGE_TAG is overriden to latest in order to build PROD image using "latest"
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
- - name: "Push PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_push_production_images.sh
- env:
- # GITHUB_REGISTRY_PULL_IMAGE_TAG is overriden to latest in order to build PROD image using "latest"
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
- # Remove me on 15th of August 2021 after all users had chance to rebase
- - name: "Push Legacy PROD images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}"
- run: ./scripts/ci/images/ci_push_legacy_prod_images.sh
- if: github.event_name == 'pull_request_target'
-
- cancel-on-ci-build:
- permissions:
- actions: write
- name: "Cancel 'CI Build' jobs on workflow failed/cancelled"
- runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }}
- if: failure() || cancelled()
- needs: [build-ci-images, build-prod-images]
- env:
- branch: ${{ github.event.pull_request.head.ref }}
- thisRun: ${{ github.run_id }}
- steps:
- - name: Find running CI Build jobs for ${{ env.TARGET_COMMIT_SHA }}
- run: |
- if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then
- event_filter="event=pull_request&"
- else
- branch="${GITHUB_REF#refs/heads/}"
- event_filter=""
- fi
-
- for cancel_url in $(
- gh api "/repos/$GITHUB_REPOSITORY/actions/runs?${event_filter}branch=${branch}" \
- jq -r '
- .workflow_runs[] |
- select(.head_sha == $ENV.GITHUB_REGISTRY_PUSH_IMAGE_TAG and .status != "completed") |
- .cancel_url
- ' \
- ); do
- # One of these URls will be _this_ workflow, so lets exclude that!
- [[ $cancel_url == */$thisRun/* ]] && continue
-
- echo "Cancelling $cancel_url"
- gh api -X POST --silent "$cancel_url"
- done
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
deleted file mode 100644
index f8bc251..0000000
--- a/.github/workflows/ci.yml
+++ /dev/null
@@ -1,1180 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: CI Build
-on: # yamllint disable-line rule:truthy
- schedule:
- - cron: '28 0 * * *'
- push:
- branches: ['main', 'v[0-9]+-[0-9]+-test']
- pull_request:
- branches: ['main', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable']
-permissions:
- # All other permissions are set to none
- contents: read
-env:
- MOUNT_SELECTED_LOCAL_SOURCES: "false"
- FORCE_ANSWER_TO_QUESTIONS: "yes"
- FORCE_PULL_IMAGES: "false"
- CHECK_IMAGE_FOR_REBUILD: "true"
- SKIP_CHECK_REMOTE_IMAGE: "true"
- DB_RESET: "true"
- VERBOSE: "true"
- DOCKER_CACHE: "pulled"
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_USERNAME: ${{ github.actor }}
- # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
- # Airflow one is going to be used
- CONSTRAINTS_GITHUB_REPOSITORY: >-
- ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
- secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
- # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- # In builds from forks, this token is empty, and this is good because such builds do not even try
- # to push images to the registry.
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
- GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
- INSTALL_PROVIDERS_FROM_SOURCES: "true"
- AIRFLOW_LOGIN_TO_GITHUB_REGISTRY: "true"
-
- # You can switch between building the image in "Build Images" workflow or building them in CI workflow
- # Separately for each job.
- #
- # a) Using images build in the separate "Build Image" workflow:
- #
- # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true"
- #
- # b) Building images in CI workflow - separately for each job:
- #
- # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
- #
- # You can also switch back to building images locally and disabling the "Build Images" workflow
- # by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false"
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}
-
-concurrency:
- group: ci-${{ github.event.pull_request.number || github.ref }}
- cancel-in-progress: true
-
-jobs:
-
- build-info:
- name: "Build info"
- # The runs-on cannot refer to env. or secrets. context, so we have no
- # option but to specify a hard-coded list here. This is "safe", as the list
- # is checked again by the runner using it's own list, so a PR author cannot
- # change this and get access to our self-hosted runners
- #
- # When changing this list, ensure that it is kept in sync with the
- # /runners/apache/airflow/configOverlay
- # parameter in AWS SSM ParameterStore (which is what the runner uses)
- runs-on: >-
- ${{ (
- (
- github.event_name == 'push' ||
- github.event_name == 'schedule' ||
- contains(fromJSON('[
- "BasPH",
- "Fokko",
- "KevinYang21",
- "XD-DENG",
- "aijamalnk",
- "alexvanboxel",
- "aoen",
- "artwr",
- "ashb",
- "bolkedebruin",
- "criccomini",
- "dimberman",
- "feng-tao",
- "houqp",
- "jghoman",
- "jmcarp",
- "kaxil",
- "leahecole",
- "mik-laj",
- "milton0825",
- "mistercrunch",
- "msumit",
- "potiuk",
- "r39132",
- "ryanahamilton",
- "ryw",
- "saguziel",
- "sekikn",
- "turbaszek",
- "zhongjiajie",
- "ephraimbuddy",
- "jhtimmins",
- "dstandish",
- "xinbinhuang",
- "yuqian",
- "eladkal"
- ]'), github.event.pull_request.user.login)
- ) && github.repository == 'apache/airflow'
- ) && 'self-hosted' || 'ubuntu-20.04' }}
- env:
- GITHUB_CONTEXT: ${{ toJson(github) }}
- outputs:
- waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
- upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
- pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
- defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
- kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }}
- kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
- defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
- kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }}
- defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }}
- postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }}
- defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }}
- mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }}
- defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }}
- helmVersions: ${{ steps.selective-checks.outputs.helm-versions }}
- defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }}
- kindVersions: ${{ steps.selective-checks.outputs.kind-versions }}
- defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }}
- testTypes: ${{ steps.selective-checks.outputs.test-types }}
- postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
- mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
- sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
- run-tests: ${{ steps.selective-checks.outputs.run-tests }}
- run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
- run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
- basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
- image-build: ${{ steps.selective-checks.outputs.image-build }}
- docs-build: ${{ steps.selective-checks.outputs.docs-build }}
- needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
- needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
- needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
- default-branch: ${{ steps.selective-checks.outputs.default-branch }}
- pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
- runsOn: ${{ steps.set-runs-on.outputs.runsOn }}
- steps:
- # Avoid having to specify the runs-on logic every time. We use the custom
- # env var AIRFLOW_SELF_HOSTED_RUNNER set only on our runners, but never
- # on the public runners
- - name: Set runs-on
- id: set-runs-on
- run: |
- if [[ ${AIRFLOW_SELF_HOSTED_RUNNER} != "" ]]; then
- echo "::set-output name=runsOn::\"self-hosted\""
- else
- echo "::set-output name=runsOn::\"ubuntu-20.04\""
- fi
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Get information about the PR"
- uses: ./.github/actions/get-workflow-origin
- id: source-run-info
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- - name: Set wait for image
- id: wait-for-image
- run: |
- if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == 'true' ]]; then
- echo "::set-output name=wait-for-image::true"
- else
- echo "::set-output name=wait-for-image::false"
- fi
- - name: Fetch incoming commit ${{ github.sha }} with its parent
- uses: actions/checkout@v2
- with:
- ref: ${{ github.sha }}
- fetch-depth: 2
- persist-credentials: false
- if: github.event_name == 'pull_request'
- - name: Selective checks
- id: selective-checks
- env:
- PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
- run: |
- if [[ ${GITHUB_EVENT_NAME} == "pull_request" ]]; then
- # Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${GITHUB_SHA}"
- else
- # Run all checks
- ./scripts/ci/selective_ci_checks.sh
- fi
-
- tests-ui:
- timeout-minutes: 10
- name: React UI tests
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- if: needs.build-info.outputs.run-ui-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup node"
- uses: actions/setup-node@v2
- with:
- node-version: 14
- - name: "Cache eslint"
- uses: actions/cache@v2
- with:
- path: 'airflow/ui/node_modules'
- key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }}
- - run: yarn --cwd airflow/ui/ install --frozen-lockfile --non-interactive
- - run: yarn --cwd airflow/ui/ run test
-
-
- test-openapi-client-generation:
- timeout-minutes: 10
- name: "Test OpenAPI client generation"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- if: needs.build-info.outputs.needs-api-codegen == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- fetch-depth: 2
- persist-credentials: false
- - name: "Generate client codegen diff"
- run: ./scripts/ci/openapi/client_codegen_diff.sh
-
- test-examples-of-prod-image-building:
- timeout-minutes: 60
- name: "Test examples of production image building"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- if: needs.build-info.outputs.image-build == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- fetch-depth: 2
- persist-credentials: false
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- if: |
- needs.build-info.outputs.waitForImage == 'true'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- - name: "Test examples of PROD image building"
- run: ./scripts/ci/images/ci_test_examples_of_prod_image_building.sh
-
- ci-images:
- timeout-minutes: 120
- name: "Wait for CI images"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- if: needs.build-info.outputs.image-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: sqlite
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- WAIT_FOR_IMAGE: ${{ needs.build-info.outputs.waitForImage }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- if: needs.build-info.outputs.waitForImage == 'true'
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: needs.build-info.outputs.waitForImage == 'true'
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- if: |
- needs.build-info.outputs.waitForImage == 'true'
- - name: >
- Wait for CI images
- ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
- id: wait-for-images
- # We wait for the images to be available either from the build-ci-image step or from
- # "build-images-workflow-run.yml' run as pull_request_target.
- # We are utilising single job to wait for all images because this job merely waits
- # for the images to be available.
- # The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'!
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
- env:
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
- VERIFY_IMAGE: "true"
-
- static-checks:
- timeout-minutes: 30
- name: "Static checks"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- SKIP: "identity"
- MOUNT_SELECTED_LOCAL_SOURCES: "true"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: needs.build-info.outputs.basic-checks-only == 'false'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Get Python version"
- run: "echo \"::set-output name=host-python-version::$(python -c
- 'import platform; print(platform.python_version())')\""
- id: host-python-version
- - name: "Cache pre-commit local-installation"
- uses: actions/cache@v2
- with:
- path: ~/.local
- key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('setup.py', 'setup.cfg') }}"
- restore-keys: "\
-pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-"
- - name: "Cache pre-commit envs"
- uses: actions/cache@v2
- with:
- path: ~/.cache/pre-commit
- key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
-
- - name: "Cache eslint"
- uses: actions/cache@v2
- with:
- path: 'airflow/ui/node_modules'
- key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }}
- - name: "Static checks"
- run: ./scripts/ci/static_checks/run_static_checks.sh
- env:
- VERBOSE: false
-
- # Those checks are run if no image needs to be built for checks. This is for simple changes that
- # Do not touch any of the python code or any of the important files that might require building
- # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
- static-checks-basic-checks-only:
- timeout-minutes: 30
- name: "Static checks: basic checks only"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- SKIP: "build,mypy,flake8,bats-in-container-tests,identity"
- MOUNT_SELECTED_LOCAL_SOURCES: "true"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: needs.build-info.outputs.basic-checks-only == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- - name: >
- Fetch incoming commit ${{ github.sha }} with its parent
- uses: actions/checkout@v2
- with:
- ref: ${{ github.sha }}
- fetch-depth: 2
- persist-credentials: false
- - name: "Get Python version"
- run: "echo \"::set-output name=host-python-version::$(python -c
- 'import platform; print(platform.python_version())')\""
- id: host-python-version
- - name: "Cache pre-commit local-installation"
- uses: actions/cache@v2
- with:
- path: ~/.local
- key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('setup.py', 'setup.cfg') }}"
- restore-keys: "\
-pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-"
- - name: "Cache pre-commit envs"
- uses: actions/cache@v2
- with:
- path: ~/.cache/pre-commit
- key: "pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys: pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}
- - name: "Static checks: basic checks only"
- run: ./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}"
- env:
- VERBOSE: false
- docs:
- timeout-minutes: 45
- name: "Build docs"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- if: needs.build-info.outputs.docs-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Fetch inventory versions"
- run: ./scripts/ci/docs/ci_docs_prepare.sh
- - uses: actions/cache@v2
- id: cache-doc-inventories
- with:
- path: ./docs/_inventory_cache/
- key: docs-inventory-v1-${{ hashFiles('constraints.txt') }}
- restore-keys: |
- docs-inventory-${{ hashFiles('constraints.txt') }}
- docs-inventory-
- - name: "Build docs"
- run: ./scripts/ci/docs/ci_docs.sh
- - name: Configure AWS credentials
- uses: ./.github/actions/configure-aws-credentials
- if: >
- github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
- github.event_name == 'push'
- with:
- aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
- aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
- aws-region: eu-central-1
- - name: "Upload documentation to AWS S3"
- if: >
- github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
- github.event_name == 'push'
- run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
-
- prepare-test-provider-packages-wheel:
- timeout-minutes: 40
- name: "Build and test provider packages wheel"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- AIRFLOW_EXTRAS: "all"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- VERSION_SUFFIX_FOR_PYPI: ".dev0"
- NON_INTERACTIVE: "true"
- GENERATE_PROVIDERS_ISSUE: "true"
- if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- if: needs.build-info.outputs.default-branch == 'main'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Prepare provider documentation"
- run: ./scripts/ci/provider_packages/ci_prepare_provider_documentation.sh
- - name: "Prepare provider packages: wheel"
- run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
- env:
- PACKAGE_FORMAT: "wheel"
- - name: "Prepare airflow package: wheel"
- run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh
- env:
- PACKAGE_FORMAT: "wheel"
- - name: "Install and test provider packages and airflow via wheel files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- USE_AIRFLOW_VERSION: "wheel"
- PACKAGE_FORMAT: "wheel"
- - name: "Install and test provider packages and airflow on Airflow 2.1 files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- USE_AIRFLOW_VERSION: "2.1.0"
- PACKAGE_FORMAT: "wheel"
-
- prepare-test-provider-packages-sdist:
- timeout-minutes: 40
- name: "Build and test provider packages sdist"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- AIRFLOW_EXTRAS: "all"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- VERSION_SUFFIX_FOR_PYPI: ".dev0"
- NON_INTERACTIVE: "true"
- GENERATE_PROVIDERS_ISSUE: "true"
- if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- if: needs.build-info.outputs.default-branch == 'main'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Prepare provider packages: sdist"
- run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
- env:
- PACKAGE_FORMAT: "sdist"
- - name: "Prepare airflow package: sdist"
- run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh
- env:
- PACKAGE_FORMAT: "sdist"
- - name: "Install and test provider packages and airflow via sdist files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- USE_AIRFLOW_VERSION: "sdist"
- PACKAGE_FORMAT: "sdist"
-
- tests-helm:
- timeout-minutes: 40
- name: "Python unit tests for helm chart"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- MOUNT_SELECTED_LOCAL_SOURCES: "true"
- TEST_TYPES: "Helm"
- BACKEND: ""
- DB_RESET: "false"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: >
- needs.build-info.outputs.needs-helm-tests == 'true' &&
- (github.repository == 'apache/airflow' || github.event_name != 'schedule') &&
- needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: Helm"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-helm
- path: "./files/airflow_logs*"
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-helm
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- with:
- name: >
- coverage-helm
- path: "./files/coverage*.xml"
- retention-days: 7
-
- tests-postgres:
- timeout-minutes: 130
- name: >
- Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
- ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }}
- fail-fast: false
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: postgres
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- POSTGRES_VERSION: ${{ matrix.postgres-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}}
- path: "./files/airflow_logs*"
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- with:
- name: >
- coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
- path: "./files/coverage*.xml"
- retention-days: 7
-
- tests-mysql:
- timeout-minutes: 130
- name: >
- MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }}
- fail-fast: false
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: mysql
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- MYSQL_VERSION: ${{ matrix.mysql-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}}
- path: "./files/airflow_logs*"
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- with:
- name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
- path: "./files/coverage*.xml"
- retention-days: 7
-
- tests-sqlite:
- timeout-minutes: 130
- name: >
- Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }}
- fail-fast: false
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: sqlite
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-${{matrix.python-version}}
- path: './files/airflow_logs*'
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-sqlite-${{matrix.python-version}}
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- with:
- name: coverage-sqlite-${{matrix.python-version}}
- path: ./files/coverage*.xml
- retention-days: 7
-
- tests-quarantined:
- timeout-minutes: 60
- name: "Quarantined tests"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- continue-on-error: true
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
- MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}}
- POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}}
- TEST_TYPES: "Quarantined"
- NUM_RUNS: 10
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Set issue id for main"
- if: github.ref == 'refs/heads/main'
- run: |
- echo "ISSUE_ID=10118" >> $GITHUB_ENV
- - name: "Set issue id for v1-10-stable"
- if: github.ref == 'refs/heads/v1-10-stable'
- run: |
- echo "ISSUE_ID=10127" >> $GITHUB_ENV
- - name: "Set issue id for v1-10-test"
- if: github.ref == 'refs/heads/v1-10-test'
- run: |
- echo "ISSUE_ID=10128" >> $GITHUB_ENV
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: Quarantined"
- run: ./scripts/ci/testing/ci_run_quarantined_tests.sh
- - name: "Upload Quarantine test results"
- uses: actions/upload-artifact@v2
- if: always()
- with:
- name: quarantined_tests
- path: "files/test_result-*.xml"
- retention-days: 7
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-quarantined-${{ matrix.backend }}
- path: "./files/airflow_logs*"
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-quarantined-${{ matrix.backend }}
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- with:
- name: coverage-quarantined-${{ matrix.backend }}
- path: "./files/coverage*.xml"
- retention-days: 7
-
- upload-coverage:
- timeout-minutes: 15
- name: "Upload coverage"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- continue-on-error: true
- needs:
- - build-info
- - tests-postgres
- - tests-sqlite
- - tests-mysql
- - tests-quarantined
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- # Only upload coverage on merges to main
- if: >
- github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
- github.event_name == 'push' &&
- needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Download all artifacts from the current build"
- uses: actions/download-artifact@v2
- with:
- path: ./coverage-files
- - name: "Removes unnecessary artifacts"
- run: ls ./coverage-files | grep -v coverage | xargs rm -rf
- - name: "Upload all coverage reports to codecov"
- uses: ./.github/actions/codecov-action
- with:
- directory: "./coverage-files"
-
- prod-images:
- timeout-minutes: 120
- name: "Wait for PROD images"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- if: needs.build-info.outputs.image-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: sqlite
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- if: needs.build-info.outputs.waitForImage == 'true'
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- if: needs.build-info.outputs.waitForImage == 'true'
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- if: |
- needs.build-info.outputs.waitForImage == 'true'
- - name: >
- Wait for PROD images
- ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
- # We wait for the images to be available either from the build-ci-image step or from
- # "build-images-workflow-run.yml' run as pull_request_target.
- # We are utilising single job to wait for all images because this job merely waits
- # For the images to be available. The test jobs wait for it to complete!
- #
- id: wait-for-images
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
- env:
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
- VERIFY_IMAGE: "true"
-
- tests-kubernetes:
- timeout-minutes: 50
- name: Helm Chart; ${{matrix.executor}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, prod-images]
- strategy:
- matrix:
- executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
- fail-fast: false
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: postgres
- RUN_TESTS: "true"
- RUNTIME: "kubernetes"
- KUBERNETES_MODE: "image"
- EXECUTOR: ${{matrix.executor}}
- KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
- HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
- CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- if: >
- ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
- needs.build-info.outputs.needs-helm-tests == 'true' ) &&
- needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Get all PROD images"
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
- env:
- VERIFY_IMAGE: "false"
- - name: "Cache bin folder with tools for kubernetes testing"
- uses: actions/cache@v2
- with:
- path: ".build/kubernetes-bin"
- key: "kubernetes-binaries
--${{ needs.build-info.outputs.defaultKindVersion }}\
--${{ needs.build-info.outputs.defaultHelmVersion }}"
- restore-keys: "kubernetes-binaries"
- - name: "Kubernetes Tests"
- run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
- - name: "Upload KinD logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: >
- kind-logs-${{matrix.executor}}
- path: /tmp/kind_logs_*
- retention-days: 7
-
- tests-helm-executor-upgrade:
- timeout-minutes: 50
- name: Helm Chart Executor Upgrade
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, prod-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: postgres
- RUN_TESTS: "true"
- RUNTIME: "kubernetes"
- KUBERNETES_MODE: "image"
- EXECUTOR: "KubernetesExecutor"
- KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
- HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
- CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- if: >
- needs.build-info.outputs.run-kubernetes-tests == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Get all PROD images"
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
- env:
- VERIFY_IMAGE: "false"
- - name: "Cache virtualenv for kubernetes testing"
- uses: actions/cache@v2
- with:
- path: ".build/.kubernetes_venv"
- key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\
- -${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- -${{needs.build-info.outputs.pythonVersionsListAsString}}
- -${{ hashFiles('setup.py','setup.cfg') }}"
- restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\
- -${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- -${{needs.build-info.outputs.pythonVersionsListAsString}}"
- - name: "Cache bin folder with tools for kubernetes testing"
- uses: actions/cache@v2
- with:
- path: ".build/kubernetes-bin"
- key: "kubernetes-binaries
- -${{ needs.build-info.outputs.defaultKindVersion }}\
- -${{ needs.build-info.outputs.defaultHelmVersion }}"
- restore-keys: "kubernetes-binaries"
- - name: "Kubernetes Helm Chart Executor Upgrade Tests"
- run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh
- - name: "Upload KinD logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: >
- kind-logs-KubernetesExecutor
- path: /tmp/kind_logs_*
- retention-days: 7
-
- constraints:
- permissions:
- contents: write
- timeout-minutes: 10
- name: "Constraints"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs:
- - build-info
- - ci-images
- - prod-images
- - static-checks
- - tests-sqlite
- - tests-mysql
- - tests-postgres
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: ${{needs.build-info.outputs.pythonVersionsListAsString}}
- # Only run it for direct pushes and scheduled builds
- if: github.event_name == 'push' || github.event_name == 'schedule'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: >
- Wait for CI images
- ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
- env:
- VERIFY_IMAGE: "false"
- - name: "Generate constraints with PyPI providers"
- run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
- env:
- GENERATE_CONSTRAINTS_MODE: "pypi-providers"
- - name: "Generate constraints with source providers"
- run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
- env:
- GENERATE_CONSTRAINTS_MODE: "source-providers"
- - name: "Generate constraints without providers"
- run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
- env:
- GENERATE_CONSTRAINTS_MODE: "no-providers"
- - name: "Set constraints branch name"
- id: constraints-branch
- run: ./scripts/ci/constraints/ci_branch_constraints.sh
- # only actually push it when we are in apache/airflow repository
- - name: Checkout ${{ steps.constraints-branch.outputs.branch }}
- uses: actions/checkout@v2
- if: github.repository == 'apache/airflow'
- with:
- path: "repo"
- ref: ${{ steps.constraints-branch.outputs.branch }}
- persist-credentials: false
- - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}"
- run: ./scripts/ci/constraints/ci_commit_constraints.sh
- if: github.repository == 'apache/airflow'
- - name: "Push changes"
- uses: ./.github/actions/github-push-action
- if: github.repository == 'apache/airflow'
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
- branch: ${{ steps.constraints-branch.outputs.branch }}
- directory: "repo"
-
- # Push images to GitHub Registry in Apache repository, if all tests are successful and build
- # is executed as result of direct push to "main" or one of the "test" branches
- # It actually rebuilds all images using just-pushed constraints if they changed
- # It will also check if a new python image was released and will pull the latest one if needed
- # Same as build-images.yaml
- push-images-to-github-registry:
- permissions:
- packages: write
- timeout-minutes: 10
- name: "Push images as cache to GitHub Registry"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs:
- - build-info
- - constraints
- - docs
- # Only run it for direct pushes and scheduled builds
- if: github.event_name == 'push' || github.event_name == 'schedule'
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- # Rebuild images before push using the latest constraints (just pushed) without
- # eager upgrade. Do not wait for images, but rebuild them, but always check if
- # there is a new Python base image to pull and rebuild. This way, when latest python
- # is not pushed, we will re-use the last cache to start from and when there is
- # a new python image, we will rebuild it from scratch (same as during the "build-images.ci")
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
- GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
- PUSH_PYTHON_BASE_IMAGE: "true"
- FORCE_PULL_IMAGES: "true"
- CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: "true"
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
- UPGRADE_TO_NEWER_DEPENDENCIES: "false"
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:latest"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Prepare PROD image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:latest"
- run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- env:
- VERSION_SUFFIX_FOR_PYPI: ".dev0"
- - name: "Push CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:latest"
- run: ./scripts/ci/images/ci_push_ci_images.sh
- - name: "Push PROD images ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:latest"
- run: ./scripts/ci/images/ci_push_production_images.sh
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
deleted file mode 100644
index 6d6f4d0..0000000
--- a/.github/workflows/codeql-analysis.yml
+++ /dev/null
@@ -1,109 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: "CodeQL"
-
-on: # yamllint disable-line rule:truthy
- push:
- branches: [main]
- schedule:
- - cron: '0 2 * * *'
-
-permissions:
- contents: read
-concurrency:
- group: codeql-${{ github.event.pull_request.number || github.ref }}
- cancel-in-progress: true
-
-jobs:
- selective-checks:
- name: Selective checks
- runs-on: ubuntu-20.04
- outputs:
- needs-python-scans: ${{ steps.selective-checks.outputs.needs-python-scans }}
- needs-javascript-scans: ${{ steps.selective-checks.outputs.needs-javascript-scans }}
- steps:
- - name: Checkout repository
- uses: actions/checkout@v2
- with:
- fetch-depth: 2
- persist-credentials: false
- - name: Selective checks
- id: selective-checks
- env:
- EVENT_NAME: ${{ github.event_name }}
- TARGET_COMMIT_SHA: ${{ github.sha }}
- run: |
- if [[ ${EVENT_NAME} == "pull_request" ]]; then
- # Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
- else
- # Run all checks
- ./scripts/ci/selective_ci_checks.sh
- fi
-
- analyze:
- name: Analyze
- runs-on: ubuntu-20.04
- needs: [selective-checks]
- strategy:
- fail-fast: false
- matrix:
- # Override automatic language detection by changing the below list
- # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
- language: ['python', 'javascript']
- permissions:
- actions: read
- contents: read
- pull-requests: read
- security-events: write
- steps:
- - name: Checkout repository
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- if: |
- matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' ||
- matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true'
-
- # Initializes the CodeQL tools for scanning.
- - name: Initialize CodeQL
- uses: github/codeql-action/init@v1
- with:
- languages: ${{ matrix.language }}
- # If you wish to specify custom queries, you can do so here or in a config file.
- # By default, queries listed here will override any specified in a config file.
- # Prefix the list here with "+" to use these queries and those in the config file.
- # queries: ./path/to/local/query, your-org/your-repo/queries@main
- if: |
- matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' ||
- matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true'
-
- # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
- # If this step fails, then you should remove it and run the build manually (see below)
- - name: Autobuild
- uses: github/codeql-action/autobuild@v1
- if: |
- matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' ||
- matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true'
-
- - name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@v1
- if: |
- matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' ||
- matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true'
diff --git a/.github/workflows/label_when_reviewed.yml b/.github/workflows/label_when_reviewed.yml
deleted file mode 100644
index 189a2d7..0000000
--- a/.github/workflows/label_when_reviewed.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: Label when reviewed
-on: pull_request_review # yamllint disable-line rule:truthy
-jobs:
-
- label-when-reviewed:
- name: "Label PRs when reviewed"
- runs-on: ubuntu-20.04
- steps:
- - name: "Do nothing. Only trigger corresponding workflow_run event"
- run: echo
diff --git a/.github/workflows/label_when_reviewed_workflow_run.yml b/.github/workflows/label_when_reviewed_workflow_run.yml
deleted file mode 100644
index 661cf04..0000000
--- a/.github/workflows/label_when_reviewed_workflow_run.yml
+++ /dev/null
@@ -1,176 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: Label when reviewed workflow run
-on: # yamllint disable-line rule:truthy
- workflow_run:
- workflows: ["Label when reviewed"]
- types: ['requested']
-permissions:
- # All other permissions are set to none
- contents: read
- pull-requests: write
-jobs:
-
- label-when-reviewed:
- name: "Label PRs when reviewed workflow run"
- runs-on: ubuntu-20.04
- outputs:
- labelSet: ${{ steps.label-when-reviewed.outputs.labelSet }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Get information about the original trigger of the run"
- uses: ./.github/actions/get-workflow-origin
- id: source-run-info
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- sourceRunId: ${{ github.event.workflow_run.id }}
- - name: Initiate Selective Build check
- uses: ./.github/actions/checks-action
- id: selective-build-check
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- name: "Selective build check"
- status: "in_progress"
- sha: ${{ steps.source-run-info.outputs.sourceHeadSha }}
- details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
- output: >
- {"summary":
- "Checking selective status of the build in
- [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
- "}
- - name: >
- Event: ${{ steps.source-run-info.outputs.sourceEvent }}
- Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
- Branch: ${{ steps.source-run-info.outputs.sourceHeadBranch }}
- Run id: ${{ github.run_id }}
- Source Run id: ${{ github.event.workflow_run.id }}
- Sha: ${{ github.sha }}
- Source Sha: ${{ steps.source-run-info.outputs.sourceHeadSha }}
- Merge commit Sha: ${{ steps.source-run-info.outputs.mergeCommitSha }}
- Target commit Sha: ${{ steps.source-run-info.outputs.targetCommitSha }}
- run: printenv
- - name: >
- Fetch incoming commit ${{ steps.source-run-info.outputs.targetCommitSha }} with its parent
- uses: actions/checkout@v2
- with:
- ref: ${{ steps.source-run-info.outputs.targetCommitSha }}
- fetch-depth: 2
- persist-credentials: false
- # checkout the main branch again, to use the right script in main workflow
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: Selective checks
- id: selective-checks
- env:
- EVENT_NAME: ${{ steps.source-run-info.outputs.sourceEvent }}
- TARGET_COMMIT_SHA: ${{ steps.source-run-info.outputs.targetCommitSha }}
- PR_LABELS: ${{ steps.source-run-info.outputs.pullRequestLabels }}
- run: |
- if [[ ${EVENT_NAME} == "pull_request_review" ]]; then
- # Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
- else
- # Run all checks
- ./scripts/ci/selective_ci_checks.sh
- fi
- - name: "Label when approved by committers for PRs that require full tests"
- uses: ./.github/actions/label-when-approved-action
- id: label-full-test-prs-when-approved-by-commiters
- if: >
- steps.selective-checks.outputs.run-tests == 'true' &&
- contains(steps.selective-checks.outputs.test-types, 'Core')
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- label: 'full tests needed'
- require_committers_approval: 'true'
- remove_label_when_approval_missing: 'false'
- pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- comment: >
- The PR most likely needs to run full matrix of tests because it modifies parts of the core
- of Airflow. However, committers might decide to merge it quickly and take the risk.
- If they don't merge it quickly - please rebase it to the latest main at your convenience,
- or amend the last commit of the PR, and push it with --force-with-lease.
- - name: "Initiate GitHub Check forcing rerun of SH ${{ github.event.pull_request.head.sha }}"
- uses: ./.github/actions/checks-action
- id: full-test-check
- if: steps.label-full-test-prs-when-approved-by-commiters.outputs.labelSet == 'true'
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- name: "Please rebase or amend, and force push the PR to run full tests"
- status: "in_progress"
- sha: ${{ steps.source-run-info.outputs.sourceHeadSha }}
- details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
- output: >
- {"summary":
- "The PR likely needs to run all tests! This was determined via selective check in
- [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
- "}
- - name: "Label when approved by committers for PRs that do not require full tests"
- uses: ./.github/actions/label-when-approved-action
- id: label-simple-test-prs-when-approved-by-commiters
- if: >
- steps.selective-checks.outputs.run-tests == 'true' &&
- ! contains(steps.selective-checks.outputs.test-types, 'Core')
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- label: 'okay to merge'
- require_committers_approval: 'true'
- pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- comment: >
- The PR is likely OK to be merged with just subset of tests for default Python and Database
- versions without running the full matrix of tests, because it does not modify the core of
- Airflow. If the committers decide that the full tests matrix is needed, they will add the label
- 'full tests needed'. Then you should rebase to the latest main or amend the last commit
- of the PR, and push it with --force-with-lease.
- - name: "Label when approved by committers for PRs that do not require tests at all"
- uses: ./.github/actions/label-when-approved-action
- id: label-no-test-prs-when-approved-by-commiters
- if: steps.selective-checks.outputs.run-tests != 'true'
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- label: 'okay to merge'
- pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- require_committers_approval: 'true'
- comment: >
- The PR is likely ready to be merged. No tests are needed as no important environment files,
- nor python files were modified by it. However, committers might decide that full test matrix is
- needed and add the 'full tests needed' label. Then you should rebase it to the latest main
- or amend the last commit of the PR, and push it with --force-with-lease.
- - name: Update Selective Build check
- uses: ./.github/actions/checks-action
- if: always()
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- check_id: ${{ steps.selective-build-check.outputs.check_id }}
- status: "completed"
- sha: ${{ steps.source-run-info.outputs.sourceHeadSha }}
- conclusion: ${{ job.status }}
- details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
- output: >
- {"summary":
- "Checking selective status of the build completed in
- [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
- "}
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
deleted file mode 100644
index a1bdd2b..0000000
--- a/.github/workflows/stale.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-# https://github.com/actions/stale
-name: 'Close stale PRs & Issues'
-on: # yamllint disable-line rule:truthy
- schedule:
- - cron: '0 0 * * *'
-permissions:
- # All other permissions are set to none
- pull-requests: write
- issues: write
-jobs:
- stale:
- runs-on: ubuntu-20.04
- steps:
- - uses: actions/stale@v4
- with:
- stale-pr-message: >
- This pull request has been automatically marked as stale because it has not had
- recent activity. It will be closed in 5 days if no further activity occurs. Thank you
- for your contributions.
- days-before-pr-stale: 45
- days-before-pr-close: 5
- exempt-pr-labels: 'pinned,security'
- only-issue-labels: 'pending-response'
- days-before-issue-stale: 30
- days-before-issue-close: 7
- stale-issue-message: >
- This issue has been automatically marked as stale because it has been open for 30 days
- with no response from the author. It will be closed in next 7 days if no further
- activity occurs from the issue author.
- close-issue-message: >
- This issue has been closed because it has not received response from the issue author.
