| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| --- |
| default_stages: [commit, push] |
| default_language_version: |
| # force all unspecified python hooks to run python3 |
| python: python3 |
| minimum_pre_commit_version: "1.20.0" |
| repos: |
| - repo: meta |
| hooks: |
| - id: identity |
| - id: check-hooks-apply |
| - repo: https://github.com/Lucas-C/pre-commit-hooks |
| rev: v1.1.9 |
| hooks: |
| - id: forbid-tabs |
| exclude: ^docs/Makefile$|^clients/gen/go.sh|\.gitmodules$ |
| - id: insert-license |
| name: Add license for all SQL files |
| files: \.sql$ |
| exclude: ^\.github/.*$ |
| args: |
| - --comment-style |
| - "/*||*/" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all other files |
| exclude: ^\.github/.*$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| files: > |
| \.properties$|\.cfg$|\.conf$|\.ini$|\.ldif$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$ |
| - id: insert-license |
| name: Add license for all rst files |
| exclude: ^\.github/.*$ |
| args: |
| - --comment-style |
| - "||" |
| - --license-filepath |
| - license-templates/LICENSE.rst |
| - --fuzzy-match-generates-todo |
| files: \.rst$ |
| - id: insert-license |
| name: Add license for all JS/CSS/PUML files |
| files: \.(js|css|puml)$ |
| exclude: ^\.github/.*$|^airflow/www/static/.*|^airflow/www_rbac/static/.*$ |
| args: |
| - --comment-style |
| - "/*!| *| */" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all JINJA template files |
| files: "^airflow/www/templates/.*\\.html$|^docs/templates/.*\\.html$.*\\.jinja2" |
| exclude: ^\.github/.*$ |
| args: |
| - --comment-style |
| - "{#||#}" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all shell files |
| exclude: ^\.github/.*$ |
| files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|\.bats$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all python files |
| exclude: ^\.github/.*$ |
| types: [python] |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all XML files |
| exclude: ^\.github/.*$ |
| files: \.xml$ |
| args: |
| - --comment-style |
| - "<!--||-->" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all yaml files |
| exclude: ^\.github/.*$ |
| types: [yaml] |
| files: \.yml$|\.yaml$ |
| args: |
| - --comment-style |
| - "|#|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all md files |
| files: \.md$ |
| exclude: ^\.github/.*$|PROVIDER_CHANGES.*\.md |
| args: |
| - --comment-style |
| - "<!--|| -->" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| - id: insert-license |
| name: Add license for all mermaid files |
| args: |
| - --comment-style |
| - "|%%|" |
| - --license-filepath |
| - license-templates/LICENSE.txt |
| - --fuzzy-match-generates-todo |
| files: \.mermaid$ |
| - repo: https://github.com/thlorenz/doctoc.git |
| rev: v1.4.0 |
| hooks: |
| - id: doctoc |
| name: Add TOC for md files |
| files: ^README\.md$|^CONTRIBUTING\.md$|^UPDATING.*.md$|^dev/README\.md$|^dev/PROVIDER_PACKAGES.md$ |
| args: |
| - "--maxlevel" |
| - "2" |
| - repo: https://github.com/pre-commit/pre-commit-hooks |
| rev: v3.3.0 |
| hooks: |
| - id: check-merge-conflict |
| - id: debug-statements |
| - id: detect-private-key |
| - id: end-of-file-fixer |
| - id: mixed-line-ending |
| - id: check-executables-have-shebangs |
| - id: check-xml |
| - id: trailing-whitespace |
| - repo: https://github.com/pre-commit/pygrep-hooks |
| rev: v1.6.0 |
| hooks: |
| - id: rst-backticks |
| - id: python-no-log-warn |
| - repo: https://github.com/adrienverge/yamllint |
| rev: v1.25.0 |
| hooks: |
| - id: yamllint |
| name: Check yaml files with yamllint |
| entry: yamllint -c yamllint-config.yml --strict |
| types: [yaml] |
| exclude: |
| ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/(?:templates|files)/.*\.yaml |
| ## |
| ## Dear committer. |
| ## |
| ## If you ever come here to add the missing isort step here - hear a little warning. |
| ## |
| ## Initially isort will cause surprising duplicates of urlparse and other urllib related methods. |
| ## The urllib imports seem broken for python 2 but they are actually fine due to future |
| ## backport aliases installed elsewhere in the code (implicitly) - in 6 places. |
| ## |
| ## When you decide how to fix it (likely talking to other people in community) and you push |
| ## build to CI you will find terrible truth that in Airflow 1.10 modules are so much |
| ## cross-dependent, that imports in a number of places have to be done in specific order and |
| ## if this is not followed properly, circular imports kick-in and you are doomed. |
| ## |
| ## Running isort breaks the import House of Cards and there is no easy way to fix it short of |
| ## splitting a number of files and probably breaking compatibility. |
| ## |
| ## Luckily this has been fixed in Airflow 2.0 by proper untangling of the cross-dependencies and |
| ## 1.10.* branch is really in maintenance mode, so do not really waste your time here. |
| ## |
| ## Unless you really want of course. But then either delete this comment or increase the counter |
| ## below after you give up. |
| ## |
| ## Total hours wasted here = 3 |
| ## |
| - repo: local |
| hooks: |
| - id: lint-dockerfile |
| name: Lint dockerfile |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh" |
| files: Dockerfile.*$ |
| pass_filenames: true |
| - id: setup-order |
| name: Checks for an order of dependencies in setup.py |
| language: python |
| files: ^setup.py$ |
| pass_filenames: false |
| entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py |
| - id: setup-installation |
| name: Checks if all the libraries in setup.py are listed in installation.rst file |
| language: python |
| files: ^setup.py$|^docs/installation.rst$ |
| pass_filenames: false |
| entry: ./scripts/ci/pre_commit/pre_commit_check_setup_installation.py |
| additional_dependencies: ['rich==9.2.0'] |
| - id: update-breeze-file |
| name: Update output of breeze command in BREEZE.rst |
| entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh" |
| language: system |
| files: ^BREEZE.rst$|^breeze$|^breeze-complete$ |
| pass_filenames: false |
| - id: update-local-yml-file |
| name: Update mounts in the local yml file |
| entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh" |
| language: system |
| files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml" |
| pass_filenames: false |
| - id: update-extras |
| name: Update extras in documentation |
| entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py |
| language: python |
| files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$ |
| pass_filenames: false |
| - id: pydevd |
| language: pygrep |
| name: Check for pydevd debug statements accidentally left |
| entry: "pydevd.*settrace\\(" |
| pass_filenames: true |
| files: \.py$ |
| - id: dont-use-safe-filter |
| language: pygrep |
| name: Don't use safe in templates |
| description: the Safe filter is error-prone, use Markup() in code instead |
| entry: "\\|\\s*safe" |
| files: \.html$ |
| pass_filenames: true |
| - id: language-matters |
| language: pygrep |
| name: Check for language that we do not accept as community |
| description: Please use "deny_list" or "allow_list" instead. |
| entry: "(?i)(black|white)[_-]?list" |
| pass_filenames: true |
| exclude: > |
| (?x) |
| ^airflow/contrib/hooks/cassandra_hook.py$| |
| ^airflow/operators/hive_stats_operator.py$| |
| ^tests/contrib/hooks/test_cassandra_hook.py| |
| ^CHANGELOG.txt |
| - id: python2-fastcheck |
| name: Find common Python 3 vs. 2.7 compatibility problems |
| language: pygrep |
| entry: > |
| (?!.*\#\ noqa)(?!.*//\ noqa)( # Exclude lines with '# noqa' or '// noqa' comment |
| .super\(\).*| # Matches super() call from Python 3 |
| ^\s*def\s*\S*\([^:#)]*:.*| # Matches function param with Python3 type |
| ^\sdef\s*\S*\(.*\):\s*\-\>\s*\S*.* # Matches -> return value syntax from Python3 |
| )$ |
| exclude: ^dev|^scripts|^docs|^chart |
| pass_filenames: true |
| - id: python2-compile |
| name: Compile code using python2 |
| language: system |
| entry: python2.7 -m py_compile |
| files: \.py$ |
| exclude: ^dev|^scripts|^docs|^chart |
| pass_filenames: true |
| require_serial: true |
| - id: incorrect-use-of-LoggingMixin |
| language: pygrep |
| name: Make sure LoggingMixin is not used alone |
| entry: "LoggingMixin\\(\\)" |
| files: \.py$ |
| pass_filenames: true |
| - id: check-integrations |
| name: Check if integration list is aligned |
| entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh |
| language: system |
| pass_filenames: false |
| files: ^common/_common_values.sh$|^breeze-complete$ |
| - id: check-apache-license |
| name: Check if licenses are OK for Apache |
| entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh" |
| language: system |
| files: ^.*LICENSE.*$|^.*LICENCE.*$ |
| pass_filenames: false |
| - id: airflow-config-yaml |
| name: Checks for consistency between config.yml and default_config.cfg |
| language: python |
| entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py |
| files: "config.yml$|default_airflow.cfg$|default.cfg$" |
| pass_filenames: false |
| require_serial: true |
| additional_dependencies: ['pyyaml'] |
| - id: pre-commit-descriptions |
| name: Check if pre-commits are described |
| entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh |
| language: system |
| files: ^.pre-commit-config.yaml$|^STATIC_CODE_CHECKS.rst|^breeze-complete$ |
| require_serial: true |
| - id: helm-lint |
| name: Lint Helm Chart |
| entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.sh |
| language: system |
| pass_filenames: false |
| files: ^chart |
| require_serial: true |
| - id: shellcheck |
| name: Check Shell scripts syntax correctness |
| language: docker_image |
| entry: koalaman/shellcheck:stable -x -a |
| files: ^breeze$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$|\.bats$ |
| - id: bats-tests |
| name: Run BATS bash tests for changed bash files |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh" |
| files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|\.bats$ |
| exclude: ^tests/bats/in_container/.*bats$|^scripts/in_container/.*sh$ |
| pass_filenames: false |
| - id: pre-commit-descriptions |
| name: Check if pre-commits are described |
| entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh |
| language: system |
| files: ^.pre-commit-config.yaml$|^STATIC_CODE_CHECKS.rst|^breeze-complete$ |
| require_serial: true |
| - id: sort-in-the-wild |
| name: Sort INTHEWILD.md alphabetically |
| entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.sh |
| language: system |
| files: ^.pre-commit-config.yaml$|^INTHEWILD.md$ |
| require_serial: true |
| - id: markdownlint |
| name: Run markdownlint |
| description: "Checks the style of Markdown files." |
| entry: markdownlint |
| language: node |
| types: [markdown] |
| files: \.(md|mdown|markdown)$ |
| additional_dependencies: ['markdownlint-cli'] |
| - id: build |
| name: Check if image build is needed |
| entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false |
| language: system |
| always_run: true |
| pass_filenames: false |
| - id: mypy |
| name: Run mypy |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh" |
| files: \.py$ |
| exclude: ^dev|^provider_packages|^chart |
| - id: mypy |
| name: Run mypy for helm chart tests |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh" |
| files: ^chart/.*\.py$ |
| exclude: ^dev |
| require_serial: true |
| - id: flake8 |
| name: Run flake8 |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh" |
| files: \.py$ |
| pass_filenames: true |
| - id: bats-in-container-tests |
| name: Run in container bats tests |
| language: system |
| entry: "./scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh" |
| files: ^tests/bats/in_container/.*.bats$|^scripts/in_container/.*sh |
| pass_filenames: false |
