| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| --- |
| default_stages: [commit, push] |
| default_language_version: |
| # force all unspecified python hooks to run python3 |
| python: python3 |
| minimum_pre_commit_version: "1.20.0" |
| repos: |
| - repo: https://github.com/Lucas-C/pre-commit-hooks |
| rev: v1.1.9 |
| hooks: |
| - id: forbid-tabs |
| exclude: ^docs/Makefile$|^clients/gen/go.sh |
| - id: insert-license |
| name: Add license for all SQL files |
| files: \.sql$ |
| exclude: ^\.github/.*$ |
| args: |
| - --comment-style |
| - "/*||*/" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all other files |
| exclude: ^\.github/.*$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| files: > |
| \.properties$|\.cfg$|\.conf$|\.ini$|\.ldif$|\.readthedocs$|\.service$|\.tf$|^Dockerfile.*$ |
| - id: insert-license |
| name: Add license for all rst files |
| exclude: ^\.github/.*$ |
| args: |
| - --comment-style |
| - "||" |
| - --license-filepath |
| - license-templates/LICENSE.rst |
| - --fuzzy-match-generates-todo |
| files: \.rst$ |
| - id: insert-license |
| name: Add license for all JS/CSS/PUML files |
| files: \.(js|css|puml)$ |
| exclude: ^\.github/.*$ |
| args: |
| - --comment-style |
| - "/**| *| */" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all JINJA template files |
| files: "^airflow/www/templates/.*\\.html$|^docs/templates/.*\\.html$.*\\.jinja2" |
| exclude: ^\.github/.*$ |
| args: |
| - --comment-style |
| - "{#||#}" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all shell files |
| exclude: ^\.github/.*$ |
| files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|\.bats$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all python files |
| exclude: ^\.github/.*$ |
| types: [python] |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all XML files |
| exclude: ^\.github/.*$ |
| files: \.xml$ |
| args: |
| - --comment-style |
| - "<!--||-->" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all yaml files |
| exclude: ^\.github/.*$ |
| types: [yaml] |
| files: \.yml$|\.yaml$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all md files |
| files: \.md$ |
| exclude: ^\.github/.*$|PROVIDERS_CHANGES.*\.md |
| args: |
| - --comment-style |
| - "<!--|| -->" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all mermaid files |
| args: |
| - --comment-style |
| - "|%%|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| files: \.mermaid$ |
| - repo: https://github.com/thlorenz/doctoc.git |
| rev: v1.4.0 |
| hooks: |
| - id: doctoc |
| name: Add TOC for md files |
| files: ^README\.md$|^CONTRIBUTING\.md$|^UPDATING.md$|^dev/README\.md$|^dev/BACKPORT_PACKAGES.md$ |
| args: |
| - "--maxlevel" |
| - "2" |
| - repo: meta |
| hooks: |
| - id: check-hooks-apply |
| - repo: https://github.com/psf/black |
| rev: 20.8b1 |
| hooks: |
| - id: black |
| files: api_connexion/.*\.py|.*providers.*\.py |
| exclude: .*kubernetes_pod\.py|.*google/common/hooks/base_google\.py$ |
| args: [--config=./pyproject.toml] |
| - repo: https://github.com/pre-commit/pre-commit-hooks |
| rev: v3.2.0 |
| hooks: |
| - id: check-merge-conflict |
| - id: debug-statements |
| - id: check-builtin-literals |
| - id: detect-private-key |
| - id: end-of-file-fixer |
| - id: mixed-line-ending |
| - id: check-executables-have-shebangs |
| - id: check-xml |
| - id: trailing-whitespace |
| - id: fix-encoding-pragma |
| args: |
| - --remove |
| - repo: https://github.com/pre-commit/pygrep-hooks |
| rev: v1.6.0 |
| hooks: |
| - id: rst-backticks |
| - id: python-no-log-warn |
| - repo: https://github.com/adrienverge/yamllint |
| rev: v1.24.2 |
| hooks: |
| - id: yamllint |
| name: Check yaml files with yamllint |
| entry: yamllint -c yamllint-config.yml |
| types: [yaml] |
| exclude: |
| ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/(?:templates|files)/.*\.yaml |
| - repo: https://github.com/timothycrosley/isort |
| rev: 5.5.3 |
| hooks: |
| - id: isort |
| name: Run isort to sort imports |
| types: [python] |
| # To keep consistent with the global isort skip config defined in setup.cfg |
| exclude: ^build/.*$|^.tox/.*$|^venv/.*$|.*api_connexion/.*\.py|.*providers.*\.py |
| - repo: https://github.com/pycqa/pydocstyle |
| rev: 5.1.1 |
| hooks: |
| - id: pydocstyle |
| name: Run pydocstyle |
| args: |
| - --convention=pep257 |
| - --add-ignore=D100,D102,D104,D105,D107,D200,D205,D400,D401 |
| exclude: ^tests/.*\.py$|^scripts/.*\.py$|^dev|^backport_packages|^kubernetes_tests |
| - repo: local |
| hooks: |
| - id: shellcheck |
| name: Check Shell scripts syntax correctness |
| language: docker_image |
| entry: koalaman/shellcheck:stable -x -a |
| files: ^breeze$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$|\.bats$ |
| - id: lint-openapi |
| name: Lint OpenAPI using spectral |
| language: docker_image |
| entry: stoplight/spectral lint -r ./scripts/ci/spectral_rules/connexion.yml |
| files: ^airflow/api_connexion/openapi/ |
| - id: lint-openapi |
| name: Lint OpenAPI using openapi-spec-validator |
| entry: openapi-spec-validator --schema 3.0.0 |
| language: python |
| additional_dependencies: ['openapi-spec-validator'] |
| files: ^airflow/api_connexion/openapi/ |
| - id: lint-dockerfile |
| name: Lint dockerfile |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh" |
| files: Dockerfile.*$ |
| pass_filenames: true |
| - id: setup-order |
| name: Checks for an order of dependencies in setup.py |
| language: python |
| files: ^setup.py$ |
| pass_filenames: false |
| entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py |
| - id: update-breeze-file |
| name: Update output of breeze command in BREEZE.rst |
| entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh" |
| language: system |
| files: ^BREEZE.rst$|^breeze$|^breeze-complete$ |
| pass_filenames: false |
| - id: update-local-yml-file |
| name: Update mounts in the local yml file |
| entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh" |
| language: system |
| files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml" |
| pass_filenames: false |
| - id: update-setup-cfg-file |
| name: Update setup.cfg file with all licenses |
| entry: "./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh" |
| language: system |
| files: ^setup.cfg$ |
| pass_filenames: false |
| - id: build-providers-dependencies |
| name: Build cross-dependencies for providers packages |
| entry: "./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh" |
| language: system |
| files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$ |
| pass_filenames: false |
| - id: update-extras |
| name: Update extras in documentation |
| entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py |
| language: python |
| files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$ |
| pass_filenames: false |
| - id: pydevd |
| language: pygrep |
| name: Check for pydevd debug statements accidentally left |
| entry: "pydevd.*settrace\\(" |
| pass_filenames: true |
| files: \.py$ |
| - id: dont-use-safe-filter |
| language: pygrep |
| name: Don't use safe in templates |
| description: the Safe filter is error-prone, use Markup() in code instead |
| entry: "\\|\\s*safe" |
| files: \.html$ |
| pass_filenames: true |
| - id: no-relative-imports |
| language: pygrep |
| name: No relative imports |
| description: Airflow style is to use absolute imports only |
| entry: "^\\s*from\\s+\\." |
| pass_filenames: true |
| files: \.py$ |
| exclude: ^tests/ |
| - id: language-matters |
| language: pygrep |
| name: Check for language that we do not accept as community |
| description: Please use "deny_list" or "allow_list" instead. |
| entry: "(?i)(black|white)[_-]?list" |
| pass_filenames: true |
| exclude: > |
| (?x) |
| ^airflow/providers/apache/cassandra/hooks/cassandra.py$| |
| ^airflow/providers/apache/hive/operators/hive_stats.py$| |
| ^airflow/providers/apache/hive/PROVIDERS_CHANGES_*| |
| ^airflow/providers/apache/hive/README.md$| |
| ^tests/providers/apache/cassandra/hooks/test_cassandra.py$| |
| ^docs/howto/connection/cassandra.rst$| |
| ^CHANGELOG.txt$ |
| - id: consistent-pylint |
| language: pygrep |
| name: Check for inconsitent pylint disable/enable without space |
| entry: "pylint:disable|pylint:enable" |
| pass_filenames: true |
| files: \.py$ |
| - id: base-operator |
| language: pygrep |
| name: Make sure BaseOperator[Link] is imported from airflow.models.baseoperator in core |
| entry: "from airflow\\.models import.* BaseOperator" |
| files: \.py$ |
| pass_filenames: true |
| exclude: > |
| (?x) |
| ^airflow/gcp/.*$| |
| ^airflow/hooks/.*$| |
| ^airflow/operators/.*$| |
| ^airflow/sensors/.*$| |
| ^airflow/providers/.*$| |
| ^backport_packages/.*$ |
| - id: base-operator |
| language: pygrep |
| name: Make sure BaseOperator[Link] is imported from airflow.models outside of core |
| entry: "from airflow\\.models\\.baseoperator import.* BaseOperator" |
| pass_filenames: true |
| files: > |
| (?x) |
| ^airflow/gcp/.*$| |
| ^airflow/hooks/.*$| |
| ^airflow/operators/.*$| |
| ^airflow/sensors/.*$| |
| ^airflow/providers/.*\.py$ |
| - id: provide-create-sessions |
| language: pygrep |
| name: To avoid import cycles make sure provide_session and create_session are imported from |
| airflow.utils.session |
| entry: "from airflow\\.utils\\.db import.* (provide_session|create_session)" |
| files: \.py$ |
| pass_filenames: true |
| - id: incorrect-use-of-LoggingMixin |
| language: pygrep |
| name: Make sure LoggingMixin is not used alone |
| entry: "LoggingMixin\\(\\)" |
| files: \.py$ |
| pass_filenames: true |
| - id: daysago-import-check |
| language: pygrep |
| name: Make sure days_ago is imported from airflow.utils.dates |
| entry: "(airflow\\.){0,1}utils\\.dates\\.days_ago" |
| files: \.py$ |
| pass_filenames: true |
| - id: restrict-start_date |
| language: pygrep |
| name: "'start_date' should not be defined in default_args in example_dags" |
| entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):" |
| files: \.*example_dags.*.py$ |
| pass_filenames: true |
| - id: check-integrations |
| name: Check if integration list is aligned |
| entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh |
| language: system |
| pass_filenames: false |
| files: ^common/_common_values.sh$|^breeze-complete$ |
| - id: build |
| name: Check if image build is needed |
| entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false |
| language: system |
| always_run: true |
| pass_filenames: false |
| - id: check-apache-license |
| name: Check if licenses are OK for Apache |
| entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh" |
| language: system |
| files: ^.*LICENSE.*$|^.*LICENCE.*$ |
| pass_filenames: false |
| - id: airflow-config-yaml |
| name: Checks for consistency between config.yml and default_config.cfg |
| language: python |
| entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py |
| files: "config.yml$|default_airflow.cfg$|default.cfg$" |
| pass_filenames: false |
| require_serial: true |
| additional_dependencies: ['pyyaml'] |
| - id: mypy |
| name: Run mypy |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh" |
| files: \.py$ |
| exclude: ^dev|^backport_packages |
| require_serial: true |
| - id: pylint |
| name: Run pylint |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_pylint.sh" |
| files: \.py$ |
| exclude: ^scripts/.*\.py$|^dev|^backport_packages |
| pass_filenames: true |
| require_serial: true |
| - id: flake8 |
| name: Run flake8 |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh" |
| files: \.py$ |
| pass_filenames: true |
| - id: bats-tests |
| name: Run BATS bash tests for changed bash files |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh" |
| files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|.bats$ |
| pass_filenames: false |
| - id: stylelint |
| name: stylelint |
| entry: "stylelint" |
| language: node |
| files: ^airflow/www/.*\.(css|scss|sass)$ |
| # Keep dependency versions in sync w/ airflow/www/package.json |
| additional_dependencies: ['stylelint@13.3.1', 'stylelint-config-standard@20.0.0'] |
| - id: mermaid |
| name: Generate mermaid images |
| entry: ./scripts/ci/pre_commit/pre_commit_mermaid.sh |
| language: system |
| files: \.mermaid$ |
| require_serial: true |
| - id: pre-commit-descriptions |
| name: Check if pre-commits are described |
| entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh |
| language: system |
| files: ^.pre-commit-config.yaml$|^STATIC_CODE_CHECKS.rst|^breeze-complete$ |
| require_serial: true |
| - id: sort-in-the-wild |
| name: Sort INTHEWILD.md alphabetically |
| entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.sh |
| language: system |
| files: ^.pre-commit-config.yaml$|^INTHEWILD.md$ |
| require_serial: true |
| - id: helm-lint |
| name: Lint Helm Chart |
| entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.sh |
| language: system |
| pass_filenames: false |
| files: ^chart |
| require_serial: true |
