| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| --- |
| default_stages: [commit, push] |
| default_language_version: |
| python: python3 |
| node: 21.6.1 |
| minimum_pre_commit_version: '3.2.0' |
| repos: |
| - repo: meta |
| hooks: |
| - id: identity |
| name: Print input to the static check hooks for troubleshooting |
| - id: check-hooks-apply |
| name: Check if all hooks apply to the repository |
| - repo: https://github.com/thlorenz/doctoc.git |
| rev: v2.2.0 |
| hooks: |
| - id: doctoc |
| name: Add TOC for Markdown and RST files |
| files: |
| ^README\.md$|^UPDATING.*\.md$|^chart/UPDATING.*\.md$|^dev/.*\.md$|^dev/.*\.rst$|^.github/.*\.md|^tests/system/README.md$ |
| exclude: ^.*/.*_vendor/ |
| args: |
| - "--maxlevel" |
| - "2" |
| - repo: https://github.com/Lucas-C/pre-commit-hooks |
| rev: v1.5.5 |
| hooks: |
| - id: insert-license |
| name: Add license for all SQL files |
| files: \.sql$ |
| exclude: | |
| (?x) |
| ^\.github/| |
| ^.*/.*_vendor/ |
| args: |
| - --comment-style |
| - "/*||*/" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all RST files |
| exclude: ^\.github/.*$|^.*/.*_vendor/|newsfragments/.*\.rst$ |
| args: |
| - --comment-style |
| - "||" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.rst |
| - --fuzzy-match-generates-todo |
| files: \.rst$ |
| - id: insert-license |
| name: Add license for all CSS/JS/JSX/PUML/TS/TSX files |
| files: \.(css|jsx?|puml|tsx?)$ |
| exclude: ^\.github/.*$|^.*/.*_vendor/ |
| args: |
| - --comment-style |
| - "/*!| *| */" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all JINJA template files |
| files: ^airflow/www/templates/.*\.html$ |
| exclude: ^\.github/.*$|^.*/.*_vendor/ |
| args: |
| - --comment-style |
| - "{#||#}" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all Shell files |
| exclude: ^\.github/.*$|^.*/.*_vendor/|^dev/breeze/autocomplete/.*$ |
| files: \.bash$|\.sh$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all toml files |
| exclude: ^\.github/.*$|^.*/.*_vendor/|^dev/breeze/autocomplete/.*$ |
| files: \.toml$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all Python files |
| exclude: ^\.github/.*$|^.*/.*_vendor/ |
| files: \.py$|\.pyi$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all XML files |
| exclude: ^\.github/.*$|^.*/.*_vendor/ |
| files: \.xml$ |
| args: |
| - --comment-style |
| - "<!--||-->" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all Helm template files |
| files: ^chart/templates/.* |
| args: |
| - --comment-style |
| - "{{/*||*/}}" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all YAML files except Helm templates |
| exclude: ^\.github/.*$|^.*/.*_vendor/|^chart/templates/.*|.*/reproducible_build.yaml$ |
| types: [yaml] |
| files: \.ya?ml$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all Markdown files |
| files: \.md$ |
| exclude: PROVIDER_CHANGES.*\.md$|^.*/.*_vendor/ |
| args: |
| - --comment-style |
| - "<!--|| -->" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all other files |
| exclude: ^\.github/.*$|^.*/.*_vendor/ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - scripts/ci/license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| files: > |
| \.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$ |
| - repo: local |
| hooks: |
| - id: update-common-sql-api-stubs |
| name: Check and update common.sql API stubs |
| entry: ./scripts/ci/pre_commit/update_common_sql_api_stubs.py |
| language: python |
| files: ^scripts/ci/pre_commit/update_common_sql_api\.py|^airflow/providers/common/sql/.*\.pyi?$ |
| additional_dependencies: ['rich>=12.4.4', 'mypy==1.9.0', 'black==23.10.0', 'jinja2'] |
| pass_filenames: false |
| require_serial: true |
| - id: update-black-version |
| name: Update black versions everywhere (manual) |
| entry: ./scripts/ci/pre_commit/update_black_version.py |
| stages: ['manual'] |
| language: python |
| files: ^.pre-commit-config.yaml$ |
| additional_dependencies: ['pyyaml'] |
| pass_filenames: false |
| require_serial: true |
| - id: update-build-dependencies |
| name: Update build-dependencies to latest (manual) |
| entry: ./scripts/ci/pre_commit/update_build_dependencies.py |
| stages: ['manual'] |
| language: python |
| files: ^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_build_dependencies.py$ |
| pass_filenames: false |
| require_serial: true |
| - id: update-installers |
| name: Update installers to latest (manual) |
| entry: ./scripts/ci/pre_commit/update_installers.py |
| stages: ['manual'] |
| language: python |
| files: ^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_installers.py$ |
| pass_filenames: false |
| require_serial: true |
| additional_dependencies: ['pyyaml', 'rich>=12.4.4', 'requests'] |
| - id: update-chart-dependencies |
| name: Update chart dependencies to latest (manual) |
| entry: ./scripts/ci/pre_commit/update_chart_dependencies.py |
| stages: ['manual'] |
| language: python |
| additional_dependencies: ['pyyaml', 'rich>=12.4.4', 'requests'] |
| files: ^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_build_dependencies.py$ |
| pass_filenames: false |
| require_serial: true |
| - repo: https://github.com/asottile/blacken-docs |
| rev: 1.16.0 |
| hooks: |
| - id: blacken-docs |
| name: Run black on Python code blocks in documentation files |
| args: |
| - --line-length=110 |
| - --target-version=py37 |
| - --target-version=py38 |
| - --target-version=py39 |
| - --target-version=py310 |
| alias: blacken-docs |
| additional_dependencies: [black==23.10.0] |
| - repo: https://github.com/pre-commit/pre-commit-hooks |
| rev: v4.5.0 |
| hooks: |
| - id: check-merge-conflict |
| name: Check that merge conflicts are not being committed |
| - id: debug-statements |
| name: Detect accidentally committed debug statements |
| - id: check-builtin-literals |
| name: Require literal syntax when initializing builtin types |
| exclude: ^.*/.*_vendor/ |
| - id: detect-private-key |
| name: Detect if private key is added to the repository |
| exclude: ^docs/apache-airflow-providers-ssh/connections/ssh.rst$ |
| - id: end-of-file-fixer |
| name: Make sure that there is an empty line at the end |
| exclude: ^.*/.*_vendor/|^docs/apache-airflow/img/.*\.dot|^docs/apache-airflow/img/.*\.sha256 |
| - id: mixed-line-ending |
| name: Detect if mixed line ending is used (\r vs. \r\n) |
| exclude: ^.*/.*_vendor/ |
| - id: check-executables-have-shebangs |
| name: Check that executables have shebang |
| exclude: ^.*/.*_vendor/ |
| - id: check-xml |
| name: Check XML files with xmllint |
| exclude: ^.*/.*_vendor/ |
| - id: trailing-whitespace |
| name: Remove trailing whitespace at end of line |
| exclude: ^.*/.*_vendor/|^docs/apache-airflow/img/.*\.dot|^dev/breeze/doc/images/output.*$ |
| - id: fix-encoding-pragma |
| name: Remove encoding header from Python files |
| exclude: ^.*/.*_vendor/ |
| args: |
| - --remove |
| - id: pretty-format-json |
| name: Format JSON files |
| args: |
| - --autofix |
| - --no-sort-keys |
| - --indent |
| - "4" |
| files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$ |
| pass_filenames: true |
| - repo: https://github.com/pre-commit/pygrep-hooks |
| rev: v1.10.0 |
| hooks: |
| - id: rst-backticks |
| name: Check if RST files use double backticks for code |
| exclude: ^.*/.*_vendor/ |
| - id: python-no-log-warn |
| name: Check if there are no deprecate log warn |
| exclude: ^.*/.*_vendor/ |
| - repo: https://github.com/adrienverge/yamllint |
| rev: v1.35.1 |
| hooks: |
| - id: yamllint |
| name: Check YAML files with yamllint |
| entry: yamllint -c yamllint-config.yml --strict |
| types: [yaml] |
| exclude: ^.*airflow\.template\.yaml$|^.*init_git_sync\.template\.yaml$|^.*/.*_vendor/|^chart/(?:templates|files)/.*\.yaml$|openapi/.*\.yaml$|^\.pre-commit-config\.yaml$|^.*/reproducible_build.yaml$ |
| - repo: https://github.com/ikamensh/flynt |
| rev: '1.0.1' |
| hooks: |
| - id: flynt |
| name: Run flynt string format converter for Python |
| exclude: | |
| (?x) |
| ^.*/.*_vendor/ |
| args: |
| # If flynt detects too long text it ignores it. So we set a very large limit to make it easy |
| # to split the text by hand. Too long lines are detected by flake8 (below), |
| # so the user is informed to take action. |
| - --line-length |
| - '99999' |
| - repo: https://github.com/codespell-project/codespell |
| rev: v2.2.6 |
| hooks: |
| - id: codespell |
| name: Run codespell to check for common misspellings in files |
| entry: bash -c 'echo "If you think that this failure is an error, consider adding the word(s) |
| to the codespell dictionary at docs/spelling_wordlist.txt. |
| The word(s) should be in lowercase." && exec codespell "$@"' -- |
| language: python |
| types: [text] |
| exclude: ^.*/.*_vendor/|^airflow/www/static/css/material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$ |
| args: |
| - --ignore-words=docs/spelling_wordlist.txt |
| - --skip=airflow/providers/*/*.rst,airflow/www/*.log,docs/*/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md |
| - --exclude-file=.codespellignorelines |
| - repo: local |
| # Note that this is the 2nd "local" repo group in the .pre-commit-config.yaml file. This is because |
| # we try to minimise the number of passes that must happen in order to apply some of the changes |
| # done by pre-commits. Some of the pre-commits not only check for errors but also fix them. This means |
| # that output from an earlier pre-commit becomes input to another pre-commit. Splitting the local |
| # scripts of our and adding some other non-local pre-commit in-between allows us to handle such |
| # changes quickly - especially when we want the early modifications from the first local group |
| # to be applied before the non-local pre-commits are run |
| hooks: |
| - id: validate-operators-init |
| name: Prevent templated field logic checks in operators' __init__ |
| language: python |
| entry: ./scripts/ci/pre_commit/validate_operators_init.py |
| pass_filenames: true |
| files: ^airflow/providers/.*/(operators|transfers|sensors)/.*\.py$ |
| additional_dependencies: [ 'rich>=12.4.4' ] |
| - id: ruff |
| name: Run 'ruff' for extremely fast Python linting |
| description: "Run 'ruff' for extremely fast Python linting" |
| entry: ruff check --force-exclude |
| language: python |
| types_or: [python, pyi] |
| args: [--fix] |
| require_serial: true |
| additional_dependencies: ["ruff==0.4.3"] |
| exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py |
| - id: ruff-format |
| name: Run 'ruff format' for extremely fast Python formatting |
| description: "Run 'ruff format' for extremely fast Python formatting" |
| entry: ./scripts/ci/pre_commit/ruff_format.py |
| language: python |
| types_or: [python, pyi] |
| args: [] |
| require_serial: true |
| additional_dependencies: ["ruff==0.4.3"] |
| exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py|^airflow/contrib/ |
| - id: replace-bad-characters |
| name: Replace bad characters |
| entry: ./scripts/ci/pre_commit/replace_bad_characters.py |
| language: python |
| types: [file, text] |
| exclude: ^.*/.*_vendor/|^clients/gen/go\.sh$|^\.gitmodules$ |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: lint-openapi |
| name: Lint OpenAPI using spectral |
| language: docker_image |
| entry: stoplight/spectral lint -r ./scripts/ci/spectral_rules/connexion.yml |
| files: ^airflow/api_connexion/openapi/ |
| - id: lint-openapi |
| name: Lint OpenAPI using openapi-spec-validator |
| entry: openapi-spec-validator --schema 3.0.0 |
| language: python |
| additional_dependencies: ['openapi-spec-validator>=0.7.1', 'openapi-schema-validator>=0.6.2'] |
| files: ^airflow/api_connexion/openapi/ |
| - id: lint-dockerfile |
| name: Lint Dockerfile |
| language: python |
| entry: ./scripts/ci/pre_commit/lint_dockerfile.py |
| files: Dockerfile.*$ |
| pass_filenames: true |
| require_serial: true |
| - id: check-airflow-k8s-not-used |
| name: Check airflow.kubernetes imports are not used |
| language: python |
| files: ^airflow/.*\.py$ |
| require_serial: true |
| exclude: ^airflow/kubernetes/ |
| entry: ./scripts/ci/pre_commit/check_airflow_k8s_not_used.py |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: check-airflow-providers-bug-report-template |
| name: Check airflow-bug-report provider list is sorted/unique |
| language: python |
| files: ^.github/ISSUE_TEMPLATE/airflow_providers_bug_report\.yml$ |
| require_serial: true |
| entry: ./scripts/ci/pre_commit/check_airflow_bug_report_template.py |
| additional_dependencies: ['rich>=12.4.4', 'pyyaml'] |
| - id: check-cncf-k8s-only-for-executors |
| name: Check cncf.kubernetes imports used for executors only |
| language: python |
| files: ^airflow/.*\.py$ |
| require_serial: true |
| exclude: ^airflow/kubernetes/|^airflow/providers/ |
| entry: ./scripts/ci/pre_commit/check_cncf_k8s_used_for_k8s_executor_only.py |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: check-airflow-provider-compatibility |
| name: Check compatibility of Providers with Airflow |
| entry: ./scripts/ci/pre_commit/check_provider_airflow_compatibility.py |
| language: python |
| pass_filenames: true |
| files: ^airflow/providers/.*\.py$ |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: check-google-re2-as-dependency |
| name: Check google-re2 is declared as dependency when needed |
| entry: ./scripts/ci/pre_commit/check_google_re2_imports.py |
| language: python |
| pass_filenames: true |
| require_serial: true |
| files: ^airflow/providers/.*\.py$ |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: update-local-yml-file |
| name: Update mounts in the local yml file |
| entry: ./scripts/ci/pre_commit/local_yml_mounts.py |
| language: python |
| files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$ |
| pass_filenames: false |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: check-sql-dependency-common-data-structure |
| name: Check dependency of SQL Providers with common data structure |
| entry: ./scripts/ci/pre_commit/check_common_sql_dependency.py |
| language: python |
| files: ^airflow/providers/.*/hooks/.*\.py$ |
| additional_dependencies: ['rich>=12.4.4', 'pyyaml', 'packaging'] |
| - id: update-providers-dependencies |
| name: Update dependencies for provider packages |
| entry: ./scripts/ci/pre_commit/update_providers_dependencies.py |
| language: python |
| files: ^airflow/providers/.*\.py$|^airflow/providers/.*/provider\.yaml$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$|^scripts/ci/pre_commit/update_providers_dependencies\.py$ |
| pass_filenames: false |
| additional_dependencies: ['setuptools', 'rich>=12.4.4', 'pyyaml', 'tomli'] |
| - id: check-extra-packages-references |
| name: Checks setup extra packages |
| description: Checks if all the extras defined in hatch_build.py are listed in extra-packages-ref.rst file |
| language: python |
| files: ^docs/apache-airflow/extra-packages-ref\.rst$|^hatch_build.py |
| pass_filenames: false |
| entry: ./scripts/ci/pre_commit/check_extra_packages_ref.py |
| additional_dependencies: ['rich>=12.4.4', 'hatchling==1.24.2', 'tabulate'] |
| - id: check-hatch-build-order |
| name: Check order of dependencies in hatch_build.py |
| language: python |
| files: ^hatch_build.py$ |
| pass_filenames: false |
| entry: ./scripts/ci/pre_commit/check_order_hatch_build.py |
| additional_dependencies: ['rich>=12.4.4', 'hatchling==1.24.2'] |
| - id: update-extras |
| name: Update extras in documentation |
| entry: ./scripts/ci/pre_commit/insert_extras.py |
| language: python |
| files: ^contributing-docs/12_airflow_dependencies_and_extras.rst$|^INSTALL$|^airflow/providers/.*/provider\.yaml$|^Dockerfile.* |
| pass_filenames: false |
| additional_dependencies: ['rich>=12.4.4', 'hatchling==1.24.2'] |
| - id: check-extras-order |
| name: Check order of extras in Dockerfile |
| entry: ./scripts/ci/pre_commit/check_order_dockerfile_extras.py |
| language: python |
| files: ^Dockerfile$ |
| pass_filenames: false |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: generate-airflow-diagrams |
| name: Generate airflow diagrams |
| entry: ./scripts/ci/pre_commit/generate_airflow_diagrams.py |
| language: python |
| files: ^docs/.*/diagram_[^/]*\.py$ |
| pass_filenames: true |
| additional_dependencies: ['rich>=12.4.4', "diagrams>=0.23.4"] |
| - id: update-supported-versions |
| name: Updates supported versions in documentation |
| entry: ./scripts/ci/pre_commit/supported_versions.py |
| language: python |
| files: ^docs/apache-airflow/installation/supported-versions\.rst$|^scripts/ci/pre_commit/supported_versions\.py$|^README\.md$ |
| pass_filenames: false |
| additional_dependencies: ['tabulate'] |
| - id: check-revision-heads-map |
| name: Check that the REVISION_HEADS_MAP is up-to-date |
| language: python |
| entry: ./scripts/ci/pre_commit/version_heads_map.py |
| pass_filenames: false |
| files: > |
| (?x) |
| ^scripts/ci/pre_commit/version_heads_map\.py$| |
| ^airflow/migrations/versions/.*$|^airflow/migrations/versions| |
| ^airflow/utils/db.py$ |
| additional_dependencies: ['packaging','google-re2'] |
| - id: update-version |
| name: Update version to the latest version in the documentation |
| entry: ./scripts/ci/pre_commit/update_versions.py |
| language: python |
| files: ^docs|^airflow/__init__.py$ |
| pass_filenames: false |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: check-pydevd-left-in-code |
| language: pygrep |
| name: Check for pydevd debug statements accidentally left |
| entry: "pydevd.*settrace\\(" |
| pass_filenames: true |
| files: \.py$ |
| - id: check-links-to-example-dags-do-not-use-hardcoded-versions |
| name: Verify example dags do not use hard-coded version numbers |
| description: The links to example dags should use |version| as version specification |
| language: pygrep |
| entry: > |
| (?i) |
| .*https://github.*[0-9]/tests/system/providers| |
| .*https://github.*/main/tests/system/providers| |
| .*https://github.*/master/tests/system/providers| |
| .*https://github.*/main/airflow/providers/.*/example_dags/| |
| .*https://github.*/master/airflow/providers/.*/example_dags/ |
| pass_filenames: true |
| files: ^docs/apache-airflow-providers-.*\.rst |
| - id: check-safe-filter-usage-in-html |
| language: pygrep |
| name: Don't use safe in templates |
| description: the Safe filter is error-prone, use Markup() in code instead |
| entry: "\\|\\s*safe" |
| files: \.html$ |
| pass_filenames: true |
| - id: check-no-providers-in-core-examples |
| language: pygrep |
| name: No providers imports in core example DAGs |
| description: The core example DAGs have no dependencies other than core Airflow |
| entry: "^\\s*from airflow\\.providers.*" |
| pass_filenames: true |
| files: ^airflow/example_dags/.*\.py$ |
| - id: check-no-airflow-deprecation-in-providers |
| language: pygrep |
| name: Do not use DeprecationWarning in providers |
| description: Use AirflowProviderDeprecationWarning in providers |
| entry: "^\\s*DeprecationWarning*" |
| pass_filenames: true |
| files: ^airflow/providers/.*\.py$ |
| - id: check-urlparse-usage-in-code |
| language: pygrep |
| name: Don't use urlparse in code |
| description: urlparse is not recommended, use urlsplit() in code instead |
| entry: "^\\s*from urllib\\.parse import ((\\|, )(urlparse\\|urlunparse))+$" |
| pass_filenames: true |
| files: \.py$ |
| - id: check-only-new-session-with-provide-session |
| name: Check NEW_SESSION is only used with @provide_session |
| language: python |
| entry: ./scripts/ci/pre_commit/new_session_in_provide_session.py |
| pass_filenames: true |
| files: ^airflow/.+\.py$ |
| - id: check-for-inclusive-language |
| language: pygrep |
| name: Check for language that we do not accept as community |
| description: Please use more appropriate words for community documentation. |
| entry: > |
| (?ix) |
| (black|white)[_-]?list| |
| \bshe\b| |
| \bhe\b| |
| \bher\b| |
| \bhis\b| |
| \bmaster\b| |
| \bslave\b| |
| \bsanity\b| |
| \bdummy\b |
| pass_filenames: true |
| exclude: > |
| (?x) |
| ^airflow/api_connexion/openapi/v1.yaml$| |
| ^airflow/cli/commands/webserver_command.py$| |
| ^airflow/cli/commands/internal_api_command.py$| |
| ^airflow/config_templates/| |
| ^airflow/models/baseoperator.py$| |
| ^airflow/operators/__init__.py$| |
| ^airflow/providers/amazon/aws/hooks/emr.py$| |
| ^airflow/providers/amazon/aws/operators/emr.py$| |
| ^airflow/providers/apache/cassandra/hooks/cassandra.py$| |
| ^airflow/providers/apache/hive/operators/hive_stats.py$| |
| ^airflow/providers/apache/hive/transfers/vertica_to_hive.py$| |
| ^airflow/providers/apache/spark/decorators/| |
| ^airflow/providers/apache/spark/hooks/| |
| ^airflow/providers/apache/spark/operators/| |
| ^airflow/providers/exasol/hooks/exasol.py$| |
| ^airflow/providers/fab/auth_manager/security_manager/| |
| ^airflow/providers/google/cloud/hooks/bigquery.py$| |
| ^airflow/providers/google/cloud/operators/cloud_build.py$| |
| ^airflow/providers/google/cloud/operators/dataproc.py$| |
| ^airflow/providers/google/cloud/operators/mlengine.py$| |
| ^airflow/providers/microsoft/azure/hooks/cosmos.py$| |
| ^airflow/providers/microsoft/winrm/hooks/winrm.py$| |
| ^airflow/www/fab_security/manager.py$| |
| ^docs/.*commits.rst$| |
| ^docs/apache-airflow-providers-apache-cassandra/connections/cassandra.rst$| |
| ^airflow/providers/microsoft/winrm/operators/winrm.py$| |
| ^airflow/providers/opsgenie/hooks/opsgenie.py$| |
| ^airflow/providers/redis/provider.yaml$| |
| ^airflow/serialization/serialized_objects.py$| |
| ^airflow/utils/db.py$| |
| ^airflow/utils/trigger_rule.py$| |
| ^airflow/www/static/css/bootstrap-theme.css$| |
| ^airflow/www/static/js/types/api-generated.ts$| |
| ^airflow/www/templates/appbuilder/flash.html$| |
| ^chart/values.schema.json$| |
| ^.*/.*_vendor/| |
| ^dev/| |
| ^docs/README.rst$| |
| ^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store.rst$| |
| ^docs/apache-airflow-providers-apache-hdfs/connections.rst$| |
| ^docs/apache-airflow-providers-apache-kafka/connections/kafka.rst$| |
| ^docs/apache-airflow-providers-apache-spark/decorators/pyspark.rst$| |
| ^docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst$| |
| ^docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst$| |
| ^docs/apache-airflow-providers-microsoft-azure/connections/azure_cosmos.rst$| |
| ^docs/apache-airflow-providers-cncf-kubernetes/operators.rst$| |
| ^docs/conf.py$| |
| ^docs/exts/removemarktransform.py$| |
| ^scripts/ci/pre_commit/vendor_k8s_json_schema.py$| |
| ^tests/| |
| ^.pre-commit-config\.yaml$| |
| ^.*CHANGELOG\.(rst|txt)$| |
| ^.*RELEASE_NOTES\.rst$| |
| ^contributing-docs/03_contributors_quick_start.rst$| |
| ^.*\.(png|gif|jp[e]?g|tgz|lock)$| |
| git |
| - id: check-base-operator-partial-arguments |
| name: Check BaseOperator and partial() arguments |
| language: python |
| entry: ./scripts/ci/pre_commit/base_operator_partial_arguments.py |
| pass_filenames: false |
| files: ^airflow/models/(?:base|mapped)operator\.py$ |
| - id: check-init-decorator-arguments |
| name: Check model __init__ and decorator arguments are in sync |
| language: python |
| entry: ./scripts/ci/pre_commit/sync_init_decorator.py |
| pass_filenames: false |
| files: ^airflow/models/dag\.py$|^airflow/(?:decorators|utils)/task_group\.py$ |
| - id: check-template-context-variable-in-sync |
| name: Check all template context variable references are in sync |
| language: python |
| entry: ./scripts/ci/pre_commit/template_context_key_sync.py |
| files: ^airflow/models/taskinstance\.py$|^airflow/utils/context\.pyi?$|^docs/apache-airflow/templates-ref\.rst$ |
| - id: check-base-operator-usage |
| language: pygrep |
| name: Check BaseOperator core imports |
| description: Make sure BaseOperator is imported from airflow.models.baseoperator in core |
| entry: "from airflow\\.models import.* BaseOperator\\b" |
| files: \.py$ |
| pass_filenames: true |
| exclude: > |
| (?x) |
| ^airflow/decorators/.*$| |
| ^airflow/hooks/.*$| |
| ^airflow/operators/.*$| |
| ^airflow/providers/.*$| |
| ^airflow/sensors/.*$| |
| ^dev/provider_packages/.*$ |
| - id: check-base-operator-usage |
| language: pygrep |
| name: Check BaseOperatorLink core imports |
| description: Make sure BaseOperatorLink is imported from airflow.models.baseoperatorlink in core |
| entry: "from airflow\\.models import.* BaseOperatorLink" |
| files: \.py$ |
| pass_filenames: true |
| exclude: > |
| (?x) |
| ^airflow/decorators/.*$| |
| ^airflow/hooks/.*$| |
| ^airflow/operators/.*$| |
| ^airflow/providers/.*$| |
| ^airflow/sensors/.*$| |
| ^dev/provider_packages/.*$ |
| - id: check-base-operator-usage |
| language: pygrep |
| name: Check BaseOperator[Link] other imports |
| description: Make sure BaseOperator[Link] is imported from airflow.models outside of core |
| entry: "from airflow\\.models\\.baseoperator(link)? import.* BaseOperator" |
| pass_filenames: true |
| files: > |
| (?x) |
| ^airflow/providers/.*\.py$ |
| exclude: ^.*/.*_vendor/ |
| - id: check-decorated-operator-implements-custom-name |
| name: Check @task decorator implements custom_operator_name |
| language: python |
| entry: ./scripts/ci/pre_commit/decorator_operator_implements_custom_name.py |
| pass_filenames: true |
| files: ^airflow/.*\.py$ |
| - id: check-core-deprecation-classes |
| language: pygrep |
| name: Verify usage of Airflow deprecation classes in core |
| entry: category=DeprecationWarning|category=PendingDeprecationWarning |
| files: \.py$ |
| exclude: ^airflow/configuration\.py$|^airflow/providers|^scripts/in_container/verify_providers\.py$|^tests/.*$ |
| pass_filenames: true |
| - id: check-provide-create-sessions-imports |
| language: pygrep |
| name: Check provide_session and create_session imports |
| description: provide_session and create_session should be imported from airflow.utils.session |
| to avoid import cycles. |
| entry: "from airflow\\.utils\\.db import.* (provide_session|create_session)" |
| files: \.py$ |
| exclude: ^.*/.*_vendor/ |
| pass_filenames: true |
| - id: check-incorrect-use-of-LoggingMixin |
| language: pygrep |
| name: Make sure LoggingMixin is not used alone |
| entry: "LoggingMixin\\(\\)" |
| files: \.py$ |
| exclude: ^.*/.*_vendor/ |
| pass_filenames: true |
| - id: check-daysago-import-from-utils |
| language: pygrep |
| name: Make sure days_ago is imported from airflow.utils.dates |
| entry: "(airflow\\.){0,1}utils\\.dates\\.days_ago" |
| files: \.py$ |
| exclude: ^.*/.*_vendor/ |
| pass_filenames: true |
| - id: check-start-date-not-used-in-defaults |
| language: pygrep |
| name: start_date not to be defined in default_args in example_dags |
| entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):" |
| files: \.*example_dags.*\.py$ |
| exclude: ^.*/.*_vendor/ |
| pass_filenames: true |
| - id: check-apache-license-rat |
| name: Check if licenses are OK for Apache |
| entry: ./scripts/ci/pre_commit/check_license.py |
| language: python |
| files: ^.*LICENSE.*$|^.*LICENCE.*$ |
| pass_filenames: false |
| - id: check-aiobotocore-optional |
| name: Check if aiobotocore is an optional dependency only |
| entry: ./scripts/ci/pre_commit/check_aiobotocore_optional.py |
| language: python |
| files: ^airflow/providers/.*/provider\.yaml$ |
| pass_filenames: true |
| additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml'] |
| require_serial: true |
| - id: check-boring-cyborg-configuration |
| name: Checks for Boring Cyborg configuration consistency |
| language: python |
| entry: ./scripts/ci/pre_commit/boring_cyborg.py |
| pass_filenames: false |
| require_serial: true |
| additional_dependencies: ['pyyaml', 'termcolor==1.1.0', 'wcmatch==8.2'] |
| - id: update-in-the-wild-to-be-sorted |
| name: Sort INTHEWILD.md alphabetically |
| entry: ./scripts/ci/pre_commit/sort_in_the_wild.py |
| language: python |
| files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$ |
| pass_filenames: false |
| require_serial: true |
| - id: update-installed-providers-to-be-sorted |
| name: Sort alphabetically and uniquify installed_providers.txt |
| entry: ./scripts/ci/pre_commit/sort_installed_providers.py |
| language: python |
| files: ^\.pre-commit-config\.yaml$|^.*_installed_providers\.txt$ |
| pass_filenames: false |
| require_serial: true |
| - id: update-spelling-wordlist-to-be-sorted |
| name: Sort alphabetically and uniquify spelling_wordlist.txt |
| entry: ./scripts/ci/pre_commit/sort_spelling_wordlist.py |
| language: python |
| files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$ |
| require_serial: true |
| pass_filenames: false |
| - id: update-openapi-spec-tags-to-be-sorted |
| name: Sort alphabetically openapi spec tags |
| entry: ./scripts/ci/pre_commit/sort_tags_in_openapi_spec.py |
| types: [yaml] |
| language: python |
| files: ^\.pre-commit-config\.yaml$|^airflow/api_connexion/openapi/v1\.yaml$ |
| require_serial: true |
| pass_filenames: false |
| - id: lint-helm-chart |
| name: Lint Helm Chart |
| entry: ./scripts/ci/pre_commit/helm_lint.py |
| language: python |
| pass_filenames: false |
| files: ^chart |
| require_serial: true |
| additional_dependencies: ['rich>=12.4.4','requests'] |
| - id: kubeconform |
| name: Kubeconform check on our helm chart |
| entry: ./scripts/ci/pre_commit/kubeconform.py |
| language: python |
| pass_filenames: false |
| files: ^chart |
| require_serial: true |
| additional_dependencies: ['rich>=12.4.4','requests'] |
| - id: shellcheck |
| name: Check Shell scripts syntax correctness |
| language: docker_image |
| entry: koalaman/shellcheck:v0.8.0 -x -a |
| files: \.(bash|sh)$|^hooks/build$|^hooks/push$ |
| exclude: ^dev/breeze/autocomplete/.*$ |
| - id: lint-css |
| name: stylelint |
| entry: "stylelint" |
| language: node |
| files: ^airflow/www/.*\.(css|sass|scss)$ |
| # Keep dependency versions in sync w/ airflow/www/package.json |
| additional_dependencies: ['stylelint@13.3.1', 'stylelint-config-standard@20.0.0', 'stylelint-config-prettier@9.0.5'] |
| - id: compile-www-assets |
| name: Compile www assets (manual) |
| language: node |
| stages: ['manual'] |
| 'types_or': [javascript, ts, tsx] |
| files: ^airflow/www/ |
| entry: ./scripts/ci/pre_commit/compile_www_assets.py |
| pass_filenames: false |
| additional_dependencies: ['yarn@1.22.21'] |
| - id: compile-www-assets-dev |
| name: Compile www assets in dev mode (manual) |
| language: node |
| stages: ['manual'] |
| 'types_or': [javascript, ts, tsx] |
| files: ^airflow/www/ |
| entry: ./scripts/ci/pre_commit/compile_www_assets_dev.py |
| pass_filenames: false |
| additional_dependencies: ['yarn@1.22.21'] |
| - id: check-providers-init-file-missing |
| name: Provider init file is missing |
| pass_filenames: false |
| always_run: true |
| entry: ./scripts/ci/pre_commit/check_providers_init.py |
| language: python |
| - id: check-providers-subpackages-init-file-exist |
| name: Provider subpackage init files are there |
| pass_filenames: false |
| always_run: true |
| entry: ./scripts/ci/pre_commit/check_providers_subpackages_all_have_init.py |
| language: python |
| require_serial: true |
| - id: check-pre-commit-information-consistent |
| name: Validate hook IDs & names and sync with docs |
| entry: ./scripts/ci/pre_commit/check_pre_commit_hooks.py |
| args: |
| - --max-length=60 |
| language: python |
| files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/check_pre_commit_hooks\.py$ |
| additional_dependencies: ['pyyaml', 'jinja2', 'black==23.10.0', 'tabulate', 'rich>=12.4.4'] |
| require_serial: true |
| pass_filenames: false |
| - id: check-integrations-list-consistent |
| name: Sync integrations list with docs |
| entry: ./scripts/ci/pre_commit/check_integrations_list.py |
| language: python |
| files: ^scripts/ci/docker-compose/integration-.*\.yml$|^contributing-docs/testing/integration_tests.rst$ |
| additional_dependencies: ['black==23.10.0', 'tabulate', 'rich>=12.4.4', 'pyyaml'] |
| require_serial: true |
| pass_filenames: false |
| - id: update-breeze-readme-config-hash |
| name: Update Breeze README.md with config files hash |
| language: python |
| entry: ./scripts/ci/pre_commit/update_breeze_config_hash.py |
| files: ^dev/breeze/pyproject\.toml$|^dev/breeze/README\.md$ |
| pass_filenames: false |
| require_serial: true |
| - id: update-reproducible-source-date-epoch |
| name: Update Source Date Epoch for reproducible builds |
| language: python |
| entry: ./scripts/ci/pre_commit/update_source_date_epoch.py |
| files: ^RELEASE_NOTES.rst$|^chart/RELEASE_NOTES.rst$ |
| additional_dependencies: ['rich>=12.4.4', 'pyyaml'] |
| pass_filenames: false |
| require_serial: true |
| - id: check-breeze-top-dependencies-limited |
| name: Breeze should have small number of top-level dependencies |
| language: python |
| entry: ./scripts/tools/check_if_limited_dependencies.py |
| files: ^dev/breeze/.*$ |
| pass_filenames: false |
| require_serial: true |
| additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml'] |
| - id: check-tests-in-the-right-folders |
| name: Check if tests are in the right folders |
| entry: ./scripts/ci/pre_commit/check_tests_in_right_folders.py |
| language: python |
| files: ^tests/.*\.py |
| pass_filenames: true |
| require_serial: true |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: check-system-tests-present |
| name: Check if system tests have required segments of code |
| entry: ./scripts/ci/pre_commit/check_system_tests.py |
| language: python |
| files: ^tests/system/.*/example_[^/]*\.py$ |
| exclude: ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$ |
| pass_filenames: true |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: generate-pypi-readme |
| name: Generate PyPI README |
| entry: ./scripts/ci/pre_commit/generate_pypi_readme.py |
| language: python |
| files: ^README\.md$ |
| pass_filenames: false |
| - id: lint-markdown |
| name: Run markdownlint |
| description: Checks the style of Markdown files. |
| entry: markdownlint |
| language: node |
| types: [markdown] |
| files: \.(md|mdown|markdown)$ |
| additional_dependencies: ['markdownlint-cli@0.38.0'] |
| - id: lint-json-schema |
| name: Lint JSON Schema files with JSON Schema |
| entry: ./scripts/ci/pre_commit/json_schema.py |
| args: |
| - --spec-url |
| - https://json-schema.org/draft-07/schema |
| language: python |
| pass_filenames: true |
| files: .*\.schema\.json$ |
| exclude: ^.*/.*_vendor/ |
| require_serial: true |
| additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0'] |
| - id: lint-json-schema |
| name: Lint NodePort Service with JSON Schema |
| entry: ./scripts/ci/pre_commit/json_schema.py |
| args: |
| - --spec-url |
| - https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.20.2-standalone/service-v1.json |
| language: python |
| pass_filenames: true |
| files: ^scripts/ci/kubernetes/nodeport\.yaml$ |
| require_serial: true |
| additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0'] |
| - id: lint-json-schema |
| name: Lint Docker compose files with JSON Schema |
| entry: ./scripts/ci/pre_commit/json_schema.py |
| args: |
| - --spec-url |
| - https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json |
| language: python |
| pass_filenames: true |
| files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose\.ya?ml$ |
| exclude: > |
| (?x) |
| ^scripts/ci/docker-compose/grafana/.| |
| ^scripts/ci/docker-compose/.+-config\.ya?ml |
| require_serial: true |
| additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0'] |
| - id: lint-json-schema |
| name: Lint chart/values.schema.json file with JSON Schema |
| entry: ./scripts/ci/pre_commit/json_schema.py |
| args: |
| - --spec-file |
| - chart/values_schema.schema.json |
| - chart/values.schema.json |
| language: python |
| pass_filenames: false |
| files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$ |
| require_serial: true |
| additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0'] |
| - id: update-vendored-in-k8s-json-schema |
| name: Vendor k8s definitions into values.schema.json |
| entry: ./scripts/ci/pre_commit/vendor_k8s_json_schema.py |
| language: python |
| files: ^chart/values\.schema\.json$ |
| additional_dependencies: ['requests==2.25.0'] |
| - id: lint-json-schema |
| name: Lint chart/values.yaml file with JSON Schema |
| entry: ./scripts/ci/pre_commit/json_schema.py |
| args: |
| - --enforce-defaults |
| - --spec-file |
| - chart/values.schema.json |
| - chart/values.yaml |
| language: python |
| pass_filenames: false |
| files: ^chart/values\.yaml$|^chart/values\.schema\.json$ |
| require_serial: true |
| additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0'] |
| - id: lint-json-schema |
| name: Lint config_templates/config.yml file with JSON Schema |
| entry: ./scripts/ci/pre_commit/json_schema.py |
| args: |
| - --spec-file |
| - airflow/config_templates/config.yml.schema.json |
| language: python |
| pass_filenames: true |
| files: ^airflow/config_templates/config\.yml$ |
| require_serial: true |
| additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0'] |
| - id: check-persist-credentials-disabled-in-github-workflows |
| name: Check that workflow files have persist-credentials disabled |
| entry: ./scripts/ci/pre_commit/checkout_no_credentials.py |
| language: python |
| pass_filenames: true |
| files: ^\.github/workflows/.*\.yml$ |
| additional_dependencies: ['PyYAML', 'rich>=12.4.4'] |
| - id: check-docstring-param-types |
| name: Check that docstrings do not specify param types |
| entry: ./scripts/ci/pre_commit/docstring_param_type.py |
| language: python |
| pass_filenames: true |
| files: \.py$ |
| exclude: ^.*/.*_vendor/ |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: check-compat-cache-on-methods |
| name: Check that compat cache do not use on class methods |
| entry: ./scripts/ci/pre_commit/compat_cache_on_methods.py |
| language: python |
| pass_filenames: true |
| files: ^airflow/.*\.py$ |
| exclude: ^.*/.*_vendor/ |
| - id: check-code-deprecations |
| name: Check deprecations categories in decorators |
| entry: ./scripts/ci/pre_commit/check_deprecations.py |
| language: python |
| pass_filenames: true |
| files: ^airflow/.*\.py$ |
| exclude: ^.*/.*_vendor/ |
| - id: lint-chart-schema |
| name: Lint chart/values.schema.json file |
| entry: ./scripts/ci/pre_commit/chart_schema.py |
| language: python |
| pass_filenames: false |
| files: ^chart/values\.schema\.json$ |
| require_serial: true |
| - id: update-inlined-dockerfile-scripts |
| name: Inline Dockerfile and Dockerfile.ci scripts |
| entry: ./scripts/ci/pre_commit/inline_scripts_in_docker.py |
| language: python |
| pass_filenames: false |
| files: ^Dockerfile$|^Dockerfile\.ci$|^scripts/docker/.*$ |
| require_serial: true |
| - id: check-changelog-has-no-duplicates |
| name: Check changelogs for duplicate entries |
| language: python |
| files: CHANGELOG\.(rst|txt)$ |
| entry: ./scripts/ci/pre_commit/changelog_duplicates.py |
| pass_filenames: true |
| - id: check-changelog-format |
| name: Check changelog format |
| language: python |
| files: CHANGELOG\.(rst|txt)$ |
| entry: ./scripts/ci/pre_commit/check_changelog_format.py |
| pass_filenames: true |
| - id: check-newsfragments-are-valid |
| name: Check newsfragments are valid |
| language: python |
| files: newsfragments/.*\.rst |
| entry: ./scripts/ci/pre_commit/newsfragments.py |
| pass_filenames: true |
| # We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes |
| # This is fast, so not too much downside |
| always_run: true |
| - id: update-breeze-cmd-output |
| name: Update output of breeze commands in Breeze documentation |
| entry: ./scripts/ci/pre_commit/breeze_cmd_line.py |
| language: python |
| files: > |
| (?x) |
| ^dev/breeze/.*$| |
| ^\.pre-commit-config\.yaml$| |
| ^scripts/ci/pre_commit/breeze_cmd_line.py$| |
| ^generated/provider_dependencies.json$ |
| require_serial: true |
| pass_filenames: false |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: check-example-dags-urls |
| name: Check that example dags url include provider versions |
| entry: ./scripts/ci/pre_commit/update_example_dags_paths.py |
| language: python |
| pass_filenames: true |
| files: ^docs/.*example-dags\.rst$|^docs/.*index\.rst$ |
| additional_dependencies: ['rich>=12.4.4', 'pyyaml'] |
| always_run: true |
| - id: check-system-tests-tocs |
| name: Check that system tests is properly added |
| entry: ./scripts/ci/pre_commit/check_system_tests_hidden_in_index.py |
| language: python |
| pass_filenames: true |
| files: ^docs/apache-airflow-providers-[^/]*/index\.rst$ |
| additional_dependencies: ['rich>=12.4.4', 'pyyaml'] |
| - id: check-lazy-logging |
| name: Check that all logging methods are lazy |
| entry: ./scripts/ci/pre_commit/check_lazy_logging.py |
| language: python |
| pass_filenames: true |
| files: \.py$ |
| exclude: ^.*/.*_vendor/ |
| additional_dependencies: ['rich>=12.4.4', 'astor'] |
| - id: create-missing-init-py-files-tests |
| name: Create missing init.py files in tests |
| entry: ./scripts/ci/pre_commit/check_init_in_tests.py |
| language: python |
| additional_dependencies: ['rich>=12.4.4'] |
| pass_filenames: false |
| files: ^tests/.*\.py$ |
| - id: ts-compile-format-lint-www |
| name: TS types generation / ESLint / Prettier against UI files |
| language: node |
| 'types_or': [javascript, ts, tsx, yaml, css, json] |
| files: ^airflow/www/static/js/|^airflow/api_connexion/openapi/v1\.yaml$ |
| entry: ./scripts/ci/pre_commit/www_lint.py |
| additional_dependencies: ['yarn@1.22.21', "openapi-typescript@>=6.7.4"] |
| pass_filenames: false |
| - id: check-tests-unittest-testcase |
| name: Check that unit tests do not inherit from unittest.TestCase |
| entry: ./scripts/ci/pre_commit/unittest_testcase.py |
| language: python |
| pass_filenames: true |
| files: ^tests/.*\.py$ |
| - id: check-usage-of-re2-over-re |
| language: pygrep |
| name: Use re2 module instead of re |
| description: Use re2 module instead of re |
| entry: "^\\s*from re\\s|^\\s*import re\\s" |
| pass_filenames: true |
| files: \.py$ |
| exclude: ^airflow/providers|^dev/.*\.py$|^scripts/.*\.py$|^tests/|^\w+_tests/|^docs/.*\.py$|^airflow/utils/helpers.py$|^hatch_build.py$ |
| - id: check-deferrable-default-value |
| name: Check default value of deferrable attribute |
| language: python |
| entry: ./scripts/ci/pre_commit/check_deferrable_default.py |
| pass_filenames: false |
| files: ^airflow/.*/sensors/.*\.py$|^airflow/.*/operators/.*\.py$ |
| - id: check-provider-docs-valid |
| name: Validate provider doc files |
| entry: ./scripts/ci/pre_commit/check_provider_docs.py |
| language: python |
| files: ^airflow/providers/.*/provider\.yaml|^docs/.* |
| additional_dependencies: ['rich>=12.4.4', 'pyyaml', 'jinja2'] |
| require_serial: true |
| - id: bandit |
| name: bandit |
| description: "Bandit is a tool for finding common security issues in Python code" |
| entry: bandit |
| language: python |
| language_version: python3 |
| types: [ python ] |
| additional_dependencies: ['bandit==1.7.6'] |
| require_serial: true |
| files: ^airflow/.* |
| exclude: |
| airflow/example_dags/.* |
| args: |
| - "--skip" |
| - "B101,B301,B324,B403,B404,B603" |
| - "--severity-level" |
| - "high" # TODO: remove this line when we fix all the issues |
| - id: pylint |
| name: pylint |
| description: "Pylint is a static code analyser for Python 2 or 3." |
| entry: pylint |
| language: python |
| language_version: python3 |
| types: [ python ] |
| additional_dependencies: ['pylint==3.1.0'] |
| require_serial: true |
| files: ^airflow/.* |
| exclude: |
| airflow/example_dags/.* |
| args: |
| # Use pylint only for the specific check, which are not available into the ruff |
| - "--disable=all" |
| # W0133: "Exception statement has no effect" |
| # see: https://github.com/astral-sh/ruff/issues/10145 |
| - "--enable=W0133" |
| - id: check-fab-migrations |
| language: pygrep |
| name: Check no migration is done on FAB related table |
| description: > |
| FAB tables are no longer used in core Airflow but in FAB provider. |
| As such, it is forbidden to create migrations related to FAB tables in core Airflow. |
| Such migrations should be in FAB provider. To achieve this, a new capability must be implemented: |
| support migrations for providers. In other words, providers need to be able to specify migrations |
| so that, any FAB related migration (besides the legacy ones) is defined in FAB provider. |
| See https://github.com/apache/airflow/issues/32210 |
| entry: > |
| (?ix) |
| \bab_permission\b| |
| \bab_view_menu\b| |
| \bab_role\b| |
| \bab_permission_view\b| |
| \bab_permission_view_role\b| |
| \bab_user\b| |
| \bab_user_role\b| |
| \bab_register_user\b |
| pass_filenames: true |
| files: ^airflow/migrations/versions/.*\.py$ |
| # These migrations contain FAB related changes but existed before moving FAB auth manager |
| # to its own provider |
| exclude: > |
| (?ix) |
| ^airflow/migrations/versions/00.*\.py$| |
| ^airflow/migrations/versions/0106.*\.py$| |
| ^airflow/migrations/versions/0118.*\.py$| |
| ^airflow/migrations/versions/0119.*\.py$| |
| ^airflow/migrations/versions/0121.*\.py$| |
| ^airflow/migrations/versions/0124.*\.py$ |
| ## ADD MOST PRE-COMMITS ABOVE THAT LINE |
| # The below pre-commits are those requiring CI image to be built |
| - id: mypy-dev |
| name: Run mypy for dev |
| language: python |
| entry: ./scripts/ci/pre_commit/mypy.py |
| files: ^dev/.*\.py$|^scripts/.*\.py$ |
| require_serial: true |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: mypy-dev |
| stages: [ 'manual' ] |
| name: Run mypy for dev (manual) |
| language: python |
| entry: ./scripts/ci/pre_commit/mypy_folder.py dev |
| pass_filenames: false |
| files: ^.*\.py$ |
| require_serial: true |
| additional_dependencies: [ 'rich>=12.4.4' ] |
| - id: mypy-airflow |
| name: Run mypy for airflow |
| language: python |
| entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages |
| files: \.py$ |
| exclude: ^.*/.*_vendor/|^airflow/migrations|^airflow/providers|^dev|^scripts|^docs|^provider_packages|^tests/providers|^tests/system/providers|^tests/dags/test_imports.py|^clients/python/test_.*\.py |
| require_serial: true |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: mypy-airflow |
| stages: [ 'manual' ] |
| name: Run mypy for airflow (manual) |
| language: python |
| entry: ./scripts/ci/pre_commit/mypy_folder.py airflow |
| pass_filenames: false |
| files: ^.*\.py$ |
| require_serial: true |
| additional_dependencies: [ 'rich>=12.4.4' ] |
| - id: mypy-providers |
| name: Run mypy for providers |
| language: python |
| entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages |
| files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$ |
| exclude: ^.*/.*_vendor/ |
| require_serial: true |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: mypy-providers |
| stages: ['manual'] |
| name: Run mypy for providers (manual) |
| language: python |
| entry: ./scripts/ci/pre_commit/mypy_folder.py airflow/providers |
| pass_filenames: false |
| files: ^.*\.py$ |
| require_serial: true |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: mypy-docs |
| name: Run mypy for /docs/ folder |
| language: python |
| entry: ./scripts/ci/pre_commit/mypy.py |
| files: ^docs/.*\.py$ |
| exclude: ^docs/rtd-deprecation |
| require_serial: true |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: mypy-docs |
| stages: ['manual'] |
| name: Run mypy for /docs/ folder (manual) |
| language: python |
| entry: ./scripts/ci/pre_commit/mypy_folder.py docs |
| pass_filenames: false |
| files: ^.*\.py$ |
| require_serial: true |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: check-provider-yaml-valid |
| name: Validate provider.yaml files |
| entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py |
| language: python |
| files: ^airflow/providers/.*/provider\.yaml$ |
| additional_dependencies: ['rich>=12.4.4'] |
| require_serial: true |
| - id: update-migration-references |
| name: Update migration ref doc |
| language: python |
| entry: ./scripts/ci/pre_commit/migration_reference.py |
| pass_filenames: false |
| files: ^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$ |
| additional_dependencies: ['rich>=12.4.4'] |
| - id: update-er-diagram |
| name: Update ER diagram |
| language: python |
| entry: ./scripts/ci/pre_commit/update_er_diagram.py |
| pass_filenames: false |
| files: ^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$ |
| additional_dependencies: ['rich>=12.4.4'] |
| ## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE |