| #!/usr/bin/env bash |
| |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails) |
| set -euo pipefail |
| |
| MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" |
| |
| export AIRFLOW_SOURCES="${MY_DIR}" |
| |
| # Directory where all CI scripts are located |
| export SCRIPTS_CI_DIR="${MY_DIR}/scripts/ci" |
| |
| BUILD_CACHE_DIR="${MY_DIR}/.build" |
| FILES_DIR="${MY_DIR}/files" |
| TMP_DIR="${MY_DIR}/tmp" |
| |
| mkdir -pv "${BUILD_CACHE_DIR}" |
| mkdir -pv "${TMP_DIR}" |
| mkdir -pv "${FILES_DIR}" |
| |
| function save_to_file { |
| # shellcheck disable=SC2005 |
| echo "$(eval echo "\$$1")" > "${BUILD_CACHE_DIR}/.$1" |
| } |
| |
| function read_from_file { |
| cat "${BUILD_CACHE_DIR}/.$1" 2>/dev/null || true |
| } |
| |
| export PYTHON_VERSION="${PYTHON_VERSION:=$(read_from_file PYTHON_VERSION)}" |
| |
| # shellcheck source=scripts/ci/_utils.sh |
| . "${SCRIPTS_CI_DIR}/_utils.sh" |
| |
| basic_sanity_checks |
| |
| script_start |
| |
| # Sets width of the screen |
| SEPARATOR_WIDTH="$(tput cols)" |
| |
| # Name of the script |
| CMDNAME="$(basename -- "$0")" |
| |
| # Update short and long options in the breeze-complete script |
| # This way autocomplete will work automagically with all options |
| # shellcheck source=breeze-complete |
| . "${MY_DIR}/breeze-complete" |
| |
| # Whether to actually run docker compose with the command set given |
| ENTER_ENVIRONMENT="true" |
| |
| |
| # Whether to cleanup local image |
| CLEANUP_IMAGES="false" |
| |
| # Skips mounting local Airflow sources |
| SKIP_MOUNTING_LOCAL_SOURCES="false" |
| |
| # If set, we initialize local virtualenv and install all dependencies |
| INITIALIZE_LOCAL_VIRTUALENV=false |
| |
| # If set, we setup autocomplete for breeze |
| SETUP_AUTOCOMPLETE=false |
| |
| # Holds chosen command if the -x flag is used. |
| RUN_COMMAND="" |
| |
| # Holds the test target if the -t flag is used. |
| TEST_TARGET="" |
| |
| # Holds docker compose command if the -d flag is used. |
| DOCKER_COMPOSE_COMMAND="" |
| |
| # If true, the docker images are rebuilt locally. |
| export AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED="false" |
| |
| # By default we only pull images if we do not have them locally. |
| # This can be overridden by -p flag |
| export AIRFLOW_CONTAINER_FORCE_PULL_IMAGES="false" |
| |
| # We use docker image caches by default to speed up the builds |
| export AIRFLOW_CONTAINER_USE_DOCKER_CACHE=${AIRFLOW_CONTAINER_USE_DOCKER_CACHE:="true"} |
| |
| # By default we do not push images. This can be overridden by -u flag. |
| export AIRFLOW_CONTAINER_PUSH_IMAGES=${AIRFLOW_CONTAINER_PUSH_IMAGES:="false"} |
| |
| # For local builds we fix file permissions only for setup-related files |
| export AIRFLOW_FIX_PERMISSIONS=${AIRFLOW_FIX_PERMISSIONS:="setup"} |
| |
| # Skip building slim image locally - we only need full CI image |
| export AIRFLOW_CONTAINER_SKIP_CI_SLIM_IMAGE="true" |
| |
| # Skip building full CI image locally - we only need slim image |
| export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false" |
| |
| # Branch name of the base image used (usually master or v1-10-test or v1-10-stable) |
| export AIRFLOW_CONTAINER_BRANCH_NAME=${AIRFLOW_CONTAINER_BRANCH_NAME:=${DEFAULT_BRANCH}} |
| |
| # Determine version of the Airflow from version.py |
| AIRFLOW_VERSION=$(cat airflow/version.py - << EOF | python |
| print(version.replace("+","")) |
| EOF |
| ) |
| export AIRFLOW_VERSION |
| |
| # Verbosity in running ci scripts |
| export VERBOSE="false" |
| |
| # Enter environment by default, rather than run tests or bash command or docker compose or static checks |
| export RUN_TESTS="false" |
| export RUN_DOCKER_COMPOSE="false" |
| export RUN_IN_BASH="false" |
| export RUN_STATIC_CHECKS="false" |
| export RUN_BUILD_DOCS="false" |
| |
| export AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD=${AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD:="false"} |
| |
| # Files determining whether asciiart/cheatsheet are suppressed |
| |
| SUPPRESS_CHEATSHEET_FILE="${MY_DIR}/.suppress_cheatsheet" |
| SUPPRESS_ASCIIART_FILE="${MY_DIR}/.suppress_asciiart" |
| |
| function print_badge { |
| if [[ ! -f "${SUPPRESS_ASCIIART_FILE}" ]]; then |
| cat <<EOF |
| |
| |
| |
| |
| @&&&&&&@ |
| @&&&&&&&&&&&@ |
| &&&&&&&&&&&&&&&& |
| &&&&&&&&&& |
| &&&&&&& |
| &&&&&&& |
| @@@@@@@@@@@@@@@@ &&&&&& |
| @&&&&&&&&&&&&&&&&&&&&&&&&&& |
| &&&&&&&&&&&&&&&&&&&&&&&&&&&& |
| &&&&&&&&&&&& |
| &&&&&&&&& |
| &&&&&&&&&&&& |
| @@&&&&&&&&&&&&&&&@ |
| @&&&&&&&&&&&&&&&&&&&&&&&&&&&& &&&&&& |
| &&&&&&&&&&&&&&&&&&&&&&&&&&&& &&&&&& |
| &&&&&&&&&&&&&&&&&&&&&&&& &&&&&& |
| &&&&&& |
| &&&&&&& |
| @&&&&&&&& |
| @&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& |
| &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& |
| &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& |
| |
| |
| |
| @&&&@ && @&&&&&&&&&&& &&&&&&&&&&&& && &&&&&&&&&& &&& &&& &&& |
| &&& &&& && @&& &&& && && &&& &&&@ &&& &&&&& &&& |
| &&& &&& && @&&&&&&&&&&&& &&&&&&&&&&& && && &&& &&& &&& &&@ &&& |
| &&&&&&&&&&& && @&&&&&&&&& && && &&@ &&& &&@&& &&@&& |
| &&& &&& && @&& &&&@ && &&&&&&&&&&& &&&&&&&&&&&& &&&& &&&& |
| |
| &&&&&&&&&&&& &&&&&&&&&&&& &&&&&&&&&&&@ &&&&&&&&&&&& &&&&&&&&&&& &&&&&&&&&&& |
| &&& &&& && &&& && &&& &&&& && |
| &&&&&&&&&&&&@ &&&&&&&&&&&& &&&&&&&&&&& &&&&&&&&&&& &&&& &&&&&&&&&& |
| &&& && && &&&& && &&& &&&& && |
| &&&&&&&&&&&&& && &&&&@ &&&&&&&&&&&@ &&&&&&&&&&&& @&&&&&&&&&&& &&&&&&&&&&& |
| |
| |
| Branch name: ${AIRFLOW_CONTAINER_BRANCH_NAME} |
| Docker image: ${AIRFLOW_CONTAINER_DOCKER_IMAGE} |
| Airflow version: ${AIRFLOW_VERSION} |
| Python version: ${PYTHON_VERSION} |
| DockerHub user: ${DOCKERHUB_USER} |
| DockerHub repo: ${DOCKERHUB_REPO} |
| Backend: ${BACKEND} |
| Env: ${ENV} |
| EOF |
| if [[ ${ENV} == "kubernetes" ]]; then |
| cat <<EOF |
| |
| Kubernetes version: ${KUBERNETES_VERSION} |
| Kubernetes mode: ${KUBERNETES_MODE} |
| |
| |
| EOF |
| fi |
| else |
| cat <<EOF |
| |
| Branch name: ${AIRFLOW_CONTAINER_BRANCH_NAME} |
| Docker image: ${AIRFLOW_CONTAINER_DOCKER_IMAGE} |
| Airflow version: ${AIRFLOW_VERSION} |
| Python version: ${PYTHON_VERSION} |
| DockerHub user: ${DOCKERHUB_USER} |
| DockerHub repo: ${DOCKERHUB_REPO} |
| Backend: ${BACKEND} |
| Env: ${ENV} |
| EOF |
| if [[ ${ENV} == "kubernetes" ]]; then |
| cat <<EOF |
| Kubernetes version: ${KUBERNETES_VERSION} |
| Kubernetes mode: ${KUBERNETES_MODE} |
| |
| |
| EOF |
| fi |
| fi |
| } |
| |
| function prepare_command_file() { |
| local FILE="${1}" |
| local CMD="${2}" |
| local TESTS="${3}" |
| local EXPANSION="${4-@}" |
| cat <<EOF > "${FILE}" |
| #!/usr/bin/env bash |
| cd "\$(pwd)" || exit |
| export DOCKERHUB_USER=${DOCKERHUB_USER} |
| export DOCKERHUB_REPO=${DOCKERHUB_REPO} |
| export COMPOSE_FILE="${COMPOSE_FILE}" |
| export PYTHON_VERSION="${PYTHON_VERSION}" |
| export BACKEND="${BACKEND}" |
| export ENV="${ENV}" |
| export KUBERNETES_VERSION="${KUBERNETES_VERSION}" |
| export KUBERNETES_MODE="${KUBERNETES_MODE}" |
| export AIRFLOW_VERSION="${AIRFLOW_VERSION}" |
| export RUN_TESTS="${TESTS}" |
| export WEBSERVER_HOST_PORT="${WEBSERVER_HOST_PORT}" |
| export POSTGRES_HOST_PORT="${POSTGRES_HOST_PORT}" |
| export MYSQL_HOST_PORT="${MYSQL_HOST_PORT}" |
| docker-compose --log-level INFO ${CMD}\$${EXPANSION}" |
| EOF |
| chmod u+x "${FILE}" |
| } |
| |
| # Default values |
| |
| _BREEZE_DEFAULT_ENV="docker" |
| _BREEZE_DEFAULT_BACKEND="sqlite" |
| _BREEZE_DEFAULT_KUBERNETES_VERSION="v1.13.0" |
| _BREEZE_DEFAULT_KUBERNETES_MODE="git_mode" |
| |
| usage() { |
| echo """ |
| |
| Usage: ${CMDNAME} [FLAGS] \\ |
| [-k]|[-S <STATIC_CHECK>]|[-F <STATIC_CHECK>]|[-O]|[-e]|[-a]|[-b]|[-t <TARGET>]|[-x <COMMAND>]|[-d <COMMAND>] \\ |
| -- <EXTRA_ARGS> |
| |
| The swiss-knife-army tool for Airflow testings. It allows to perform various test tasks: |
| |
| * Enter interactive environment when no command flags are specified (default behaviour) |
| * Stop the interactive environment with -k, --stop-environment command |
| * Run static checks - either for currently staged change or for all files with |
| -S, --static-check or -F, --static-check-all-files commanbd |
| * Build documentation with -O, --build-docs command |
| * Setup local virtualenv with -e, --setup-virtualenv command |
| * Setup autocomplete for itself with -a, --setup-autocomplete command |
| * Build docker image with -b, --build-only command |
| * Run test target specified with -t, --test-target connad |
| * Execute arbitrary command in the test environmenrt with -x, --execute-command command |
| * Execute arbitrary docker-compose command with -d, --docker-compose command |
| |
| ** Commands |
| |
| By default the script enters IT environment and drops you to bash shell, |
| but you can also choose one of the commands to run specific actions instead: |
| |
| -k, --stop-environment |
| Bring down running docker compose environment. When you start the environment, the docker |
| containers will continue running so that startup time is shorter. But they take quite a lot of |
| memory and CPU. This command stops all running containers from the environment. |
| |
| -O, --build-docs |
| Build documentation. |
| |
| -S, --static-check <STATIC_CHECK> |
| Run selected static checks for currently changed files. You should specify static check that |
| you would like to run or 'all' to run all checks. One of |
| [${_BREEZE_ALLOWED_STATIC_CHECKS:=}]. |
| You can pass extra arguments including options to to the pre-commit framework as |
| <EXTRA_ARGS> passed after --. For example: |
| |
| '${0} --static-check mypy' or |
| '${0} --static-check mypy -- --files tests/core.py' |
| |
| You can see all the options by adding --help EXTRA_ARG: |
| |
| '${0} --static-check mypy -- --help' |
| |
| -F, --static-check-all-files <STATIC_CHECK> |
| Run selected static checks for all applicable files. You should specify static check that |
| you would like to run or 'all' to run all checks. One of |
| [${_BREEZE_ALLOWED_STATIC_CHECKS:=}]. |
| You can pass extra arguments including options to the pre-commit framework as |
| <EXTRA_ARGS> passed after --. For example: |
| |
| '${0} --static-check-all-files mypy' or |
| '${0} --static-check-all-files mypy -- --verbose' |
| |
| You can see all the options by adding --help EXTRA_ARG: |
| |
| '${0} --static-check-all-files mypy -- --help' |
| |
| -e, --initialize-local-virtualenv |
| Initializes locally created virtualenv installing all dependencies of Airflow. |
| This local virtualenv can be used to aid autocompletion and IDE support as |
| well as run unit tests directly from the IDE. You need to have virtualenv |
| activated before running this command. |
| |
| -a, --setup-autocomplete |
| Sets up autocomplete for breeze commands. Once you do it you need to re-enter the bash |
| shell and when typing breeze command <TAB> will provide autocomplete for parameters and values. |
| |
| -b, --build-only |
| Only build docker images but do not enter the airflow-testing docker container. |
| |
| -t, --test-target <TARGET> |
| Run the specified unit test target. There might be multiple |
| targets specified separated with comas. The <EXTRA_ARGS> passed after -- are treated |
| as additional options passed to nosetest. For example: |
| |
| '${0} --test-target tests.core -- --logging-level=DEBUG' |
| |
| -x, --execute-command <COMMAND> |
| Run chosen command instead of entering the environment. The command is run using |
| 'bash -c \"<command with args>\" if you need to pass arguments to your command, you need |
| to pass them together with command surrounded with \" or '. Alternatively you can pass arguments as |
| <EXTRA_ARGS> passed after --. For example: |
| |
| '${0} --execute-command \"ls -la\"' or |
| '${0} --execute-command ls -- --la' |
| |
| -d, --docker-compose <COMMAND> |
| Run docker-compose command instead of entering the environment. Use 'help' command |
| to see available commands. The <EXTRA_ARGS> passed after -- are treated |
| as additional options passed to docker-compose. For example |
| |
| '${0} --docker-compose pull -- --ignore-pull-failures' |
| |
| ** General flags |
| |
| -h, --help |
| Shows this help message. |
| |
| -P, --python <PYTHON_VERSION> |
| Python version used for the image. This is always major/minor version. |
| One of [${_BREEZE_ALLOWED_PYTHON_VERSIONS:=}]. Default is the python3 or python on the path. |
| |
| -E, --env <ENVIRONMENT> |
| Environment to use for tests. It determines which types of tests can be run. |
| One of [${_BREEZE_ALLOWED_ENVS:=}]. Default: ${_BREEZE_DEFAULT_ENV:=} |
| |
| -B, --backend <BACKEND> |
| Backend to use for tests - it determines which database is used. |
| One of [${_BREEZE_ALLOWED_BACKENDS:=}]. Default: ${_BREEZE_DEFAULT_BACKEND:=} |
| |
| -K, --kubernetes-version <KUBERNETES_VERSION> |
| Kubernetes version - only used in case of 'kubernetes' environment. |
| One of [${_BREEZE_ALLOWED_KUBERNETES_VERSIONS:=}]. Default: ${_BREEZE_DEFAULT_KUBERNETES_VERSION:=} |
| |
| -M, --kubernetes-mode <KUBERNETES_MODE> |
| Kubernetes mode - only used in case of 'kubernetes' environment. |
| One of [${_BREEZE_ALLOWED_KUBERNETES_MODES:=}]. Default: ${_BREEZE_DEFAULT_KUBERNETES_MODE:=} |
| |
| -s, --skip-mounting-source-volume |
| Skips mounting local volume with sources - you get exactly what is in the |
| docker image rather than your current local sources of airflow. |
| |
| -v, --verbose |
| Show verbose information about executed commands (enabled by default for running test) |
| |
| -y, --assume-yes |
| Assume 'yes' answer to all questions. |
| |
| -n, --assume-no |
| Assume 'no' answer to all questions. |
| |
| -C, --toggle-suppress-cheatsheet |
| Toggles on/off cheatsheet displayed before starting bash shell |
| |
| -A, --toggle-suppress-asciiart |
| Toggles on/off asciiart displayed before starting bash shell |
| |
| ** Dockerfile management flags |
| |
| -D, --dockerhub-user |
| DockerHub user used to pull, push and build images. Default: ${_BREEZE_DEFAULT_DOCKERHUB_USER:=}. |
| |
| -H, --dockerhub-repo |
| DockerHub repository used to pull, push, build images. Default: ${_BREEZE_DEFAULT_DOCKERHUB_REPO:=}. |
| |
| -r, --force-build-images |
| Forces building of the local docker images. The images are rebuilt |
| automatically for the first time or when changes are detected in |
| package-related files, but you can force it using this flag. |
| |
| -R, --force-build-images-clean |
| Force build images without cache. This will remove the pulled or build images |
| and start building images from scratch. This might take a long time. |
| |
| -p, --force-pull-images |
| Forces pulling of images from DockerHub before building to populate cache. The |
| images are pulled by default only for the first time you run the |
| environment, later the locally build images are used as cache. |
| |
| -u, --push-images |
| After building - uploads the images to DockerHub |
| It is useful in case you use your own DockerHub user to store images and you want |
| to build them locally. Note that you need to use 'docker login' before you upload images. |
| |
| -c, --cleanup-images |
| Cleanup your local docker cache of the airflow docker images. This will not reclaim space in |
| docker cache. You need to 'docker system prune' (optionally with --all) to reclaim that space. |
| |
| """ |
| } |
| |
| #################### Parsing options/arguments |
| if ! PARAMS=$(getopt \ |
| -o "${_BREEZE_GETOPT_SHORT_OPTIONS:=}" \ |
| -l "${_BREEZE_GETOPT_LONG_OPTIONS:=}" \ |
| --name "$CMDNAME" -- "$@") |
| then |
| usage |
| exit 1 |
| fi |
| |
| eval set -- "${PARAMS}" |
| unset PARAMS |
| |
| # Parse Flags. |
| # Please update short and long options in the breeze-complete script |
| # This way autocomplete will work out-of-the-box |
| while true |
| do |
| case "${1}" in |
| -h|--help) |
| usage; |
| exit 0 ;; |
| -P|--python) |
| export PYTHON_VERSION="${2}"; |
| echo |
| echo "Python version: ${PYTHON_VERSION}" |
| echo |
| shift 2 ;; |
| -E|--env) |
| export ENV="${2}"; |
| echo |
| echo "Environment: ${ENV}" |
| echo |
| shift 2 ;; |
| -B|--backend) |
| export BACKEND="${2}"; |
| echo |
| echo "Backend: ${BACKEND}" |
| echo |
| shift 2 ;; |
| -K|--kubernetes-version) |
| export KUBERNETES_VERSION="${2}"; |
| echo |
| echo "Kubernetes version: ${KUBERNETES_VERSION}" |
| echo |
| shift 2 ;; |
| -M|--kubernetes-mode) |
| export KUBERNETES_MODE="${2}"; |
| echo |
| echo "Kubernetes mode: ${KUBERNETES_MODE}" |
| echo |
| shift 2 ;; |
| -s|--skip-mounting-local-sources) |
| SKIP_MOUNTING_LOCAL_SOURCES="true" |
| echo "Skip mounting local sources: ${SKIP_MOUNTING_LOCAL_SOURCES}" |
| echo |
| shift ;; |
| -b|--build-only) |
| ENTER_ENVIRONMENT="false" |
| export AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD="true" |
| echo "Only build. Do not enter airflow-testing container" |
| echo |
| shift ;; |
| -v|--verbose) |
| VERBOSE="true" |
| echo "Verbose output" |
| echo |
| shift ;; |
| -y|--assume-yes) |
| export FORCE_ANSWER_TO_QUESTIONS="yes" |
| echo "Assuming 'yes' answer to all questions." |
| echo |
| shift ;; |
| -n|--assume-no) |
| export FORCE_ANSWER_TO_QUESTIONS="no" |
| echo "Assuming 'no' answer to all questions." |
| echo |
| shift ;; |
| -C|--toggle-suppress-cheatsheet) |
| if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then |
| rm -f "${SUPPRESS_CHEATSHEET_FILE}" |
| else |
| touch "${SUPPRESS_CHEATSHEET_FILE}" |
| fi |
| echo "Toggle suppress cheatsheet" |
| echo |
| shift ;; |
| -A|--toggle-suppress-asciiart) |
| if [[ -f "${SUPPRESS_ASCIIART_FILE}" ]]; then |
| rm -f "${SUPPRESS_ASCIIART_FILE}" |
| else |
| touch "${SUPPRESS_ASCIIART_FILE}" |
| fi |
| echo "Toggle suppress asciiart" |
| echo |
| shift ;; |
| -r|--force-build-images) |
| echo |
| echo "Force build images" |
| echo |
| export AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD="true" |
| shift ;; |
| -R|--force-build-images-clean) |
| echo |
| echo "Clean build of images without cache" |
| echo |
| export AIRFLOW_CONTAINER_USE_DOCKER_CACHE=false |
| export AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE=false |
| export AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD="true" |
| CLEANUP_IMAGES="true" |
| shift ;; |
| -p|--force-pull-images) |
| echo |
| echo "Force pulling images before build. Uses pulled images as cache." |
| echo |
| export AIRFLOW_CONTAINER_FORCE_PULL_IMAGES="true" |
| export AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD="true" |
| shift ;; |
| -u|--push-images) |
| if [[ "${AIRFLOW_FIX_PERMISSIONS}" != "all" ]]; then |
| echo >&2 |
| echo >&2 "ERROR: Disable fix permissions when pushing" |
| echo >&2 |
| echo >&2 "You cannot push images if you have AIRFLOW_FIX_PERMISSIONS set to other value than 'all'" |
| echo >&2 "Your docker context is most likely wrong in this case" |
| echo >&2 "You need to set AIRFLOW_FIX_PERMISSIONS to false" |
| echo >&2 "And run the build again" |
| echo >&2 |
| exit 1 |
| fi |
| echo |
| echo "Pushing images to DockerHub" |
| echo |
| export AIRFLOW_CONTAINER_PUSH_IMAGES="true" |
| export AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD="true" |
| shift ;; |
| -c|--cleanup-images) |
| echo |
| echo "Cleanup the images" |
| echo |
| CLEANUP_IMAGES=true |
| shift ;; |
| -D|--dockerhub-user) |
| export DOCKERHUB_USER="${2}" |
| echo |
| echo "Dockerhub user ${DOCKERHUB_USER}" |
| echo |
| shift 2 ;; |
| -H|--dockerhub-repo) |
| export DOCKERHUB_REPO="${2}" |
| echo |
| echo "Dockerhub repo ${DOCKERHUB_REPO}" |
| echo |
| shift 2 ;; |
| -e|--initialize-local-virtualenv) |
| echo |
| echo Initializing local virtualenv |
| echo |
| INITIALIZE_LOCAL_VIRTUALENV="true" |
| SETUP_AUTOCOMPLETE="false" |
| ENTER_ENVIRONMENT=:"false" |
| shift ;; |
| -a|--setup-autocomplete) |
| echo |
| echo Setting up autocomplete |
| echo |
| INITIALIZE_LOCAL_VIRTUALENV="false" |
| SETUP_AUTOCOMPLETE="true" |
| ENTER_ENVIRONMENT=:"false" |
| shift ;; |
| -t|--test-target) |
| export TEST_TARGET="${2}" |
| export RUN_IN_BASH="false" |
| export RUN_TESTS="true" |
| export RUN_DOCKER_COMPOSE="false" |
| export RUN_STATIC_CHECKS="false" |
| export RUN_BUILD_DOCS="false" |
| shift 2 ;; |
| -d|--docker-compose) |
| export DOCKER_COMPOSE_COMMAND="${2}" |
| export RUN_IN_BASH="false" |
| export RUN_TESTS="false" |
| export RUN_DOCKER_COMPOSE="true" |
| export RUN_STATIC_CHECKS="false" |
| export RUN_BUILD_DOCS="false" |
| shift 2 ;; |
| -k|--stop-environment) |
| export DOCKER_COMPOSE_COMMAND="down" |
| export RUN_IN_BASH="false" |
| export RUN_TESTS="false" |
| export RUN_BUILD_DOCS="false" |
| export RUN_DOCKER_COMPOSE="true" |
| export RUN_STATIC_CHECKS="false" |
| shift ;; |
| -x|--execute-command) |
| export RUN_COMMAND="${2}" |
| export RUN_IN_BASH="true" |
| export RUN_TESTS="false" |
| export RUN_BUILD_DOCS="false" |
| export RUN_DOCKER_COMPOSE="false" |
| export RUN_STATIC_CHECKS="false" |
| shift 2 ;; |
| -S|--static-check ) |
| export ENTER_ENVIRONMENT="false" |
| export RUN_TESTS="false" |
| export RUN_DOCKER_COMPOSE="false" |
| export RUN_STATIC_CHECKS="true" |
| export RUN_BUILD_DOCS="false" |
| export STATIC_CHECK="${2}" |
| export EXTRA_STATIC_CHECK_OPTIONS=("--show-diff-on-failure") |
| export STATIC_CHECK_ALL_FILES="false" |
| shift 2 ;; |
| -F|--static-check-all-files) |
| export ENTER_ENVIRONMENT="false" |
| export RUN_TESTS="false" |
| export RUN_DOCKER_COMPOSE="false" |
| export RUN_STATIC_CHECKS="true" |
| export RUN_BUILD_DOCS="false" |
| export STATIC_CHECK="${2}" |
| export STATIC_CHECK_ALL_FILES="true" |
| export EXTRA_STATIC_CHECK_OPTIONS=("--all-files" "--show-diff-on-failure") |
| shift 2 ;; |
| -O|--build-docs) |
| export ENTER_ENVIRONMENT="false" |
| export RUN_TESTS="false" |
| export RUN_DOCKER_COMPOSE="false" |
| export RUN_STATIC_CHECKS="false" |
| export RUN_BUILD_DOCS="true" |
| shift 1 ;; |
| --) |
| shift ; |
| break ;; |
| *) |
| usage |
| echo >&2 |
| echo >&2 "ERROR: Unknown argument ${1}" |
| echo >&2 |
| exit 1 |
| ;; |
| esac |
| done |
| |
| echo |
| printf '=%.0s' $(seq "${SEPARATOR_WIDTH}") |
| echo |
| |
| CMDNAME="$(basename -- "$0")" |
| |
| # Cleans up the answer that was given last time, whether to force/ |
| cleanup_last_force_answer |
| |
| export ENV="${ENV:=$(read_from_file ENV)}" |
| export BACKEND="${BACKEND:=$(read_from_file BACKEND)}" |
| export KUBERNETES_VERSION="${KUBERNETES_VERSION:=$(read_from_file KUBERNETES_VERSION)}" |
| export KUBERNETES_MODE="${KUBERNETES_MODE:=$(read_from_file KUBERNETES_MODE)}" |
| |
| # Here you read DockerHub user/account that you use |
| # You can populate your own images in DockerHub this way and work with the, |
| # You can override it with "-d" option and it will be stored in .build directory |
| export DOCKERHUB_USER="${DOCKERHUB_USER:=$(read_from_file DOCKERHUB_USER)}" |
| export DOCKERHUB_USER="${DOCKERHUB_USER:=${_BREEZE_DEFAULT_DOCKERHUB_USER}}" |
| |
| # Here you read DockerHub repo that you use |
| # You can populate your own images in DockerHub this way and work with them |
| # You can override it with "-d" option and it will be stored in .build directory |
| export DOCKERHUB_REPO="${DOCKERHUB_REPO:=$(read_from_file DOCKERHUB_REPO)}" |
| export DOCKERHUB_REPO="${DOCKERHUB_REPO:=${_BREEZE_DEFAULT_DOCKERHUB_REPO}}" |
| |
| # Default environment for tests |
| export ENV=${ENV:-${_BREEZE_DEFAULT_ENV}} |
| |
| # Default backend for tests |
| export BACKEND=${BACKEND:-${_BREEZE_DEFAULT_BACKEND}} |
| |
| # Default version of Kubernetes to use |
| export KUBERNETES_VERSION=${KUBERNETES_VERSION:=${_BREEZE_DEFAULT_KUBERNETES_VERSION}} |
| # Default mode of Kubernetes to use |
| export KUBERNETES_MODE=${KUBERNETES_MODE:=${_BREEZE_DEFAULT_KUBERNETES_MODE}} |
| |
| #################### Check python version ########################################## |
| if [[ ${_BREEZE_ALLOWED_PYTHON_VERSIONS:=} != *" ${PYTHON_VERSION} "* ]]; then |
| echo >&2 |
| echo >&2 "ERROR: Allowed Python versions: [${_BREEZE_ALLOWED_PYTHON_VERSIONS}]. Is: '${PYTHON_VERSION}'." |
| echo >&2 |
| echo >&2 "Switch to virtualenv with the supported python version or specify python with --python flag." |
| echo >&2 |
| exit 1 |
| fi |
| |
| #################### Check environments ########################################## |
| if [[ ${_BREEZE_ALLOWED_ENVS:=} != *" ${ENV} "* ]]; then |
| echo >&2 |
| echo >&2 "ERROR: Allowed environments are [${_BREEZE_ALLOWED_ENVS}]. Used: '${ENV}'" |
| echo >&2 |
| exit 1 |
| fi |
| |
| #################### Check backends ########################################## |
| if [[ ${_BREEZE_ALLOWED_BACKENDS:=} != *" ${BACKEND} "* ]]; then |
| echo >&2 |
| echo >&2 "ERROR: Allowed backends are [${_BREEZE_ALLOWED_BACKENDS}]. Used: '${BACKEND}'" |
| echo >&2 |
| exit 1 |
| fi |
| |
| #################### Check environments ########################################## |
| if [[ ${_BREEZE_ALLOWED_KUBERNETES_VERSIONS} != *" ${KUBERNETES_VERSION} "* ]]; then |
| echo >&2 |
| echo >&2 "ERROR: Allowed kubernetes versions" \ |
| "are [${_BREEZE_ALLOWED_KUBERNETES_VERSIONS}]. Used: '${KUBERNETES_VERSION}'" |
| echo >&2 |
| exit 1 |
| fi |
| |
| #################### Check environments ########################################## |
| if [[ ${_BREEZE_ALLOWED_KUBERNETES_MODES} != *" ${KUBERNETES_MODE} "* ]]; then |
| echo >&2 |
| echo >&2 "ERROR: Allowed kubernetes modes" \ |
| "are [${_BREEZE_ALLOWED_KUBERNETES_MODES}]. Used: '${KUBERNETES_MODE}'" |
| echo >&2 |
| exit 1 |
| fi |
| |
| |
| # Those files are mounted into container when run locally |
| # .bash_history is preserved and you can modify .bash_aliases and .inputrc |
| # according to your liking |
| touch "${MY_DIR}/.bash_history" |
| touch "${MY_DIR}/.bash_aliases" |
| touch "${MY_DIR}/.inputrc" |
| |
| save_to_file BACKEND |
| save_to_file ENV |
| save_to_file KUBERNETES_VERSION |
| save_to_file KUBERNETES_MODE |
| save_to_file PYTHON_VERSION |
| save_to_file DOCKERHUB_USER |
| save_to_file DOCKERHUB_REPO |
| |
| #################### Cleanup image if requested ######################################## |
| if [[ "${CLEANUP_IMAGES}" == "true" ]]; then |
| export AIRFLOW_CONTAINER_CLEANUP_IMAGES=true |
| "${MY_DIR}/scripts/ci/local_ci_cleanup.sh" |
| exit 0 |
| fi |
| |
| #################### Initializes local virtualenv ######################################## |
| if [[ ${INITIALIZE_LOCAL_VIRTUALENV} == "true" ]]; then |
| # Check if we are in virtualenv |
| set +e |
| echo -e "import sys\nif not hasattr(sys,'real_prefix'):\n sys.exit(1)" | "python${PYTHON_VERSION}" |
| RES=$? |
| set -e |
| if [[ ${RES} != "0" ]]; then |
| echo >&2 |
| echo >&2 "ERROR: Initializing local virtualenv only works when you have virtualenv activated" |
| echo >&2 |
| echo >&2 "Please enter your local virtualenv before (for example using 'workon') " |
| echo >&2 |
| exit 1 |
| else |
| # If no Airflow Home defined - fallback to ${HOME}/airflow |
| AIRFLOW_HOME_DIR=${AIRFLOW_HOME:=${HOME}/airflow} |
| echo |
| echo "Initializing the virtualenv: $(command -v python)!" |
| echo |
| echo "This will wipe out ${AIRFLOW_HOME_DIR} and reset all the databases!" |
| echo |
| "${MY_DIR}/confirm" "Proceeding with the initialization" |
| echo |
| pushd "${MY_DIR}" |
| SYSTEM=$(uname -s) |
| echo "#######################################################################" |
| echo " If you have trouble installing all dependencies you might need to run:" |
| echo |
| if [[ ${SYSTEM} == "Darwin" ]]; then |
| echo " brew install sqlite mysql postgresql" |
| else |
| echo " sudo apt-get install openssl sqlite libmysqlclient-dev libmysqld-dev postgresql --confirm" |
| fi |
| echo |
| echo "#######################################################################" |
| pip install -e ".[devel]" |
| popd |
| echo |
| echo "Wiping and recreating ${AIRFLOW_HOME_DIR}" |
| echo |
| rm -rvf "${AIRFLOW_HOME_DIR}" |
| mkdir -p "${AIRFLOW_HOME_DIR}" |
| echo |
| echo "Resetting AIRFLOW sqlite database" |
| echo |
| unset AIRFLOW__CORE__UNIT_TEST_MODE |
| airflow db reset -y |
| echo |
| echo "Resetting AIRFLOW sqlite unit test database" |
| echo |
| export AIRFLOW__CORE__UNIT_TEST_MODE=True |
| airflow db reset -y |
| exit 0 |
| fi |
| fi |
| |
| |
| #################### Sets up autocomplete for breeze commands ######################################## |
| if [[ ${SETUP_AUTOCOMPLETE} == "true" ]]; then |
| echo |
| echo "Installing bash/zsh completion for local user" |
| echo "Note that completion for zsh is just limited to flags - without their values" |
| echo |
| set +e |
| grep ".bash_completion.d" "${HOME}/.bashrc" >/dev/null 2>&1 |
| RES=$? |
| set -e |
| if [[ "${RES}" == "0" ]]; then |
| echo >&2 |
| echo >&2 "ERROR: Bash completion already setup before." |
| echo >&2 |
| exit 1 |
| fi |
| "${MY_DIR}/confirm" "This will create ~/.bash_completion.d/ directory and modify ~/.bashrc and ~/.zshrc file" |
| echo |
| echo |
| mkdir -pv ~/.bash_completion.d |
| ln -sf "${MY_DIR}/breeze-complete" "${HOME}/.bash_completion.d/" |
| touch ~/.bashrc |
| cat >>~/.bashrc <<"EOF" |
| for BCFILE in ~/.bash_completion.d/* ; do |
| . ${BCFILE} |
| done |
| EOF |
| cat >>~/.zshrc <<"EOF" |
| autoload compinit && compinit |
| autoload bashcompinit && bashcompinit |
| source ~/.bash_completion.d/breeze-complete |
| EOF |
| if [[ "${OSTYPE}" == "darwin"* ]]; then |
| # For MacOS we have to handle the special case where terminal app DOES NOT run .bashrc by default |
| # But re-runs .bash_profile :( |
| # See https://scriptingosx.com/2017/04/about-bash_profile-and-bashrc-on-macos/ |
| set +e |
| grep ".bashrc" "${HOME}/.bash_profile" |
| RES=$? |
| set -e |
| if [[ "${RES}" == "0" ]]; then |
| echo " Seems you already source .bashrc in your .bash_profile so not adding it." |
| else |
| "${MY_DIR}/confirm" "This will modify ~/.bash_profile and source .bashrc from it" |
| echo |
| echo |
| cat >>~/.bash_profile <<"EOF" |
| if [ -r ~/.bashrc ]; then |
| source ~/.bashrc |
| fi |
| EOF |
| fi |
| fi |
| echo |
| echo |
| echo "Breeze bash completion installed to ~/.bash_completion.d/breeze-complete" |
| echo |
| echo |
| echo "Please re-enter bash or run '. ~/.bash_completion.d/breeze-complete'" |
| echo |
| exit 0 |
| fi |
| |
| MAIN_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose.yml |
| KUBERNETES_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose-kubernetes.yml |
| BACKEND_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose-${BACKEND}.yml |
| LOCAL_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose-local.yml |
| |
| COMPOSE_FILE=${MAIN_DOCKER_COMPOSE_FILE}:${BACKEND_DOCKER_COMPOSE_FILE} |
| |
| if [[ "${ENV}" == "kubernetes" ]]; then |
| COMPOSE_FILE=${COMPOSE_FILE}:${KUBERNETES_DOCKER_COMPOSE_FILE} |
| fi |
| |
| |
| if [[ "${SKIP_MOUNTING_LOCAL_SOURCES}" != "true" ]]; then |
| COMPOSE_FILE=${COMPOSE_FILE}:${LOCAL_DOCKER_COMPOSE_FILE} |
| fi |
| |
| export COMPOSE_FILE |
| |
| CI_ENTRYPOINT_FILE="/opt/airflow/scripts/ci/in_container/entrypoint_ci.sh" |
| |
| DC_RUN_COMMAND="run --service-ports --rm airflow-testing \"${CI_ENTRYPOINT_FILE} " |
| DC_RUN_COMMAND_ONLY_AT="run --no-deps --service-ports --rm airflow-testing \"${CI_ENTRYPOINT_FILE} " |
| |
| LAST_DC_RUN_FILE="cmd_run" |
| LAST_DC_RUN_ONLY_AT_FILE="cmd_only_at_run" |
| LAST_DC_TEST_FILE="test_run" |
| LAST_DC_FILE="dc" |
| |
| # Prepare script for "run command" |
| prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_RUN_FILE}" "${DC_RUN_COMMAND}" "false" '*' |
| |
| # Prepare script for "run command" |
| prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_RUN_ONLY_AT_FILE}" "${DC_RUN_COMMAND_ONLY_AT}" "false" '*' |
| |
| # Prepare script for "run test" |
| prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_TEST_FILE}" "${DC_RUN_COMMAND}" "true" '*' |
| |
| # Prepare script for "run docker compose command" |
| prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_FILE}" '"' "false" |
| |
| rebuild_ci_image_if_needed |
| rebuild_ci_slim_image_if_needed |
| rebuild_checklicence_image_if_needed |
| |
| export AIRFLOW_CONTAINER_DOCKER_IMAGE=\ |
| ${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CONTAINER_BRANCH_NAME}-python${PYTHON_VERSION}-ci |
| |
| printf '=%.0s' $(seq "${SEPARATOR_WIDTH}") |
| |
| if [[ "${TEST_TARGET}" == "." ]]; then |
| TEST_TARGET="" |
| fi |
| |
| print_badge |
| |
| function print_line { |
| printf '#%.0s' $(seq "${SEPARATOR_WIDTH}") |
| } |
| |
| if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} ]]; then |
| echo |
| echo |
| print_line |
| echo |
| echo " Airflow Breeze CHEATSHEET" |
| echo |
| print_line |
| echo |
| echo |
| print_line |
| echo |
| echo " Quick scripts:" |
| echo " * Enter the full environment : ${BUILD_CACHE_DIR}/${LAST_DC_RUN_FILE}" |
| echo " * Run command in full environment : ${BUILD_CACHE_DIR}/${LAST_DC_RUN_FILE} "\ |
| "[command with args] [bash options]" |
| echo " * Run command airflow-testing only : ${BUILD_CACHE_DIR}/${LAST_DC_RUN_ONLY_AT_FILE} "\ |
| "[command with args] [bash options]" |
| echo " * Run tests in the full environment : ${BUILD_CACHE_DIR}/${LAST_DC_TEST_FILE} "\ |
| "[test target] [nosetest options]" |
| echo " * Run Docker compose command : ${BUILD_CACHE_DIR}/${LAST_DC_FILE} "\ |
| "[docker compose command] [docker-compose options]" |
| echo |
| |
| set +e |
| if ! command -v breeze; then |
| print_line |
| echo |
| echo " Adding breeze to your path:" |
| echo " When you exit the environment, you can add sources of airflow to the path - you can" |
| echo " run breeze or the scripts above from any directory by calling 'breeze' commands directly" |
| echo |
| echo " export PATH=\${PATH}:\"${MY_DIR}\"" |
| echo |
| fi |
| set -e |
| print_line |
| |
| echo |
| echo " Port forwarding:" |
| echo |
| echo " Ports are forwarded to the running docker containers for webserver and database" |
| echo " * ${WEBSERVER_HOST_PORT} -> forwarded to airflow webserver -> airflow-testing:8080" |
| echo " * ${POSTGRES_HOST_PORT} -> forwarded to postgres database -> postgres:5432" |
| echo " * ${MYSQL_HOST_PORT} -> forwarded to mysql database -> mysql:3306" |
| echo |
| echo " Here are links to those services that you can use on host:" |
| echo " * Webserver: http://127.0.0.1:28080" |
| echo " * Postgres: jdbc:postgresql://127.0.0.1:25433/airflow?user=postgres&password=airflow" |
| echo " * Mysql: jdbc:mysql://localhost:23306/airflow?user=root" |
| echo |
| else |
| echo |
| fi |
| |
| # shellcheck disable=SC2034 # Unused variables left for comp_breeze usage |
| if ! typeset -f "_comp_breeze" > /dev/null; then |
| print_line |
| echo |
| echo " You can setup autocomplete by running '$0 --setup-autocomplete'" |
| echo |
| echo |
| fi |
| print_line |
| echo |
| echo " You can toggle ascii/cheatsheet by adding this flag:" |
| echo " * --toggle-suppress-cheatsheet" |
| echo " * --toggle-suppress-asciiart" |
| echo |
| print_line |
| echo |
| echo |
| echo |
| echo |
| |
| if [[ ${ENTER_ENVIRONMENT} == "true" ]]; then |
| if [[ ${RUN_TESTS} == "true" ]]; then |
| "${BUILD_CACHE_DIR}/${LAST_DC_TEST_FILE}" "\"${TEST_TARGET}\"" "$@" |
| elif [[ ${RUN_DOCKER_COMPOSE} == "true" ]]; then |
| "${BUILD_CACHE_DIR}/${LAST_DC_FILE}" "${DOCKER_COMPOSE_COMMAND}" "$@" |
| elif [[ ${RUN_IN_BASH} == "true" ]]; then |
| "${BUILD_CACHE_DIR}/${LAST_DC_RUN_FILE}" "${RUN_COMMAND}" "$@" |
| else |
| "${BUILD_CACHE_DIR}/${LAST_DC_RUN_FILE}" |
| fi |
| else |
| if [[ ${RUN_STATIC_CHECKS} == "true" ]]; then |
| echo |
| echo "Making sure pre-commit is installed" |
| echo |
| if command -v pip3 >/dev/null; then |
| PIP_BIN=pip3 |
| elif command -v pip >/dev/null; then |
| PIP_BIN=pip |
| else |
| echo >&2 |
| echo >&2 "ERROR: You need to have pip or pip3 in your PATH" |
| echo >&2 |
| S |
| exit 1 |
| fi |
| "${PIP_BIN}" install --upgrade pre-commit >/dev/null 2>&1 |
| # Add ~/.local/bin to the path in case pip is run outside of virtualenv |
| export PATH="${PATH}":~/.local/bin |
| if [[ ${STATIC_CHECK} == "all" ]]; then |
| echo |
| echo "Running: pre-commit run" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@" |
| echo |
| pre-commit run "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@" |
| else |
| echo |
| echo "Running: pre-commit run" "${STATIC_CHECK}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@" |
| echo |
| pre-commit run "${STATIC_CHECK}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@" |
| fi |
| elif [[ ${RUN_BUILD_DOCS} == "true" ]]; then |
| run_docs |
| fi |
| fi |
| |
| script_end |
