| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT. |
| # |
| ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" |
| FROM ${PYTHON_BASE_IMAGE} as main |
| |
| SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] |
| |
| ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" |
| ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} |
| |
| ARG AIRFLOW_VERSION="2.0.0.dev0" |
| ENV AIRFLOW_VERSION=$AIRFLOW_VERSION |
| |
| ARG PYTHON_MAJOR_MINOR_VERSION="3.6" |
| ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION} |
| |
| # Print versions |
| RUN echo "Base image: ${PYTHON_BASE_IMAGE}" |
| RUN echo "Airflow version: ${AIRFLOW_VERSION}" |
| |
| # Make sure noninteractive debian install is used and language variables set |
| ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ |
| LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 |
| |
| # By increasing this number we can do force build of all dependencies |
| ARG DEPENDENCIES_EPOCH_NUMBER="4" |
| # Increase the value below to force renstalling of all dependencies |
| ENV DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} |
| |
| # Install curl and gnupg2 - needed to download nodejs in the next step |
| RUN apt-get update \ |
| && apt-get install -y --no-install-recommends \ |
| curl \ |
| gnupg2 \ |
| && apt-get autoremove -yqq --purge \ |
| && apt-get clean \ |
| && rm -rf /var/lib/apt/lists/* |
| |
| ARG ADDITIONAL_DEV_DEPS="" |
| ENV ADDITIONAL_DEV_DEPS=${ADDITIONAL_DEV_DEPS} |
| |
| # Install basic and additional apt dependencies |
| RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ |
| && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ |
| && echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list \ |
| && apt-get update \ |
| && apt-get install -y --no-install-recommends \ |
| apt-utils \ |
| build-essential \ |
| dirmngr \ |
| dumb-init \ |
| freetds-bin \ |
| freetds-dev \ |
| git \ |
| graphviz \ |
| gosu \ |
| libffi-dev \ |
| libkrb5-dev \ |
| libpq-dev \ |
| libsasl2-2 \ |
| libsasl2-dev \ |
| libsasl2-modules \ |
| libssl-dev \ |
| locales \ |
| netcat \ |
| nodejs \ |
| rsync \ |
| sasl2-bin \ |
| sudo \ |
| yarn \ |
| ${ADDITIONAL_DEV_DEPS} \ |
| && apt-get autoremove -yqq --purge \ |
| && apt-get clean \ |
| && rm -rf /var/lib/apt/lists/* |
| |
| # Install MySQL client from Oracle repositories (Debian installs mariadb) |
| RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \ |
| && GNUPGHOME="$(mktemp -d)" \ |
| && export GNUPGHOME \ |
| && for KEYSERVER in $(shuf -e \ |
| ha.pool.sks-keyservers.net \ |
| hkp://p80.pool.sks-keyservers.net:80 \ |
| keyserver.ubuntu.com \ |
| hkp://keyserver.ubuntu.com:80 \ |
| pgp.mit.edu) ; do \ |
| gpg --keyserver "${KEYSERVER}" --recv-keys "${KEY}" && break || true ; \ |
| done \ |
| && gpg --export "${KEY}" | apt-key add - \ |
| && gpgconf --kill all \ |
| rm -rf "${GNUPGHOME}"; \ |
| apt-key list > /dev/null \ |
| && echo "deb http://repo.mysql.com/apt/debian/ stretch mysql-5.6" | tee -a /etc/apt/sources.list.d/mysql.list \ |
| && apt-get update \ |
| && apt-get install --no-install-recommends -y \ |
| libmysqlclient-dev \ |
| mysql-client \ |
| && apt-get autoremove -yqq --purge \ |
| && apt-get clean && rm -rf /var/lib/apt/lists/* |
| |
| RUN adduser airflow \ |
| && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ |
| && chmod 0440 /etc/sudoers.d/airflow |
| |
| ARG ADDITIONAL_RUNTIME_DEPS="" |
| ENV ADDITIONAL_RUNTIME_DEPS=${ADDITIONAL_RUNTIME_DEPS} |
| |
| # Note missing man directories on debian-buster |
| # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 |
| RUN mkdir -pv /usr/share/man/man1 \ |
| && mkdir -pv /usr/share/man/man7 \ |
| && apt-get update \ |
| && apt-get install --no-install-recommends -y \ |
| gnupg \ |
| libgcc-8-dev \ |
| apt-transport-https \ |
| bash-completion \ |
| ca-certificates \ |
| software-properties-common \ |
| krb5-user \ |
| ldap-utils \ |
| less \ |
| # The latest buster images do not have libpython 2.7 installed and it is needed |
| # To run virtualenv tests with python 2 |
| libpython2.7-stdlib \ |
| lsb-release \ |
| net-tools \ |
| openssh-client \ |
| openssh-server \ |
| postgresql-client \ |
| sqlite3 \ |
| tmux \ |
| unzip \ |
| vim \ |
| ${ADDITIONAL_RUNTIME_DEPS} \ |
| && apt-get autoremove -yqq --purge \ |
| && apt-get clean \ |
| && rm -rf /var/lib/apt/lists/* |
| |
| ARG DOCKER_CLI_VERSION=19.03.9 |
| ENV DOCKER_CLI_VERSION=${DOCKER_CLI_VERSION} |
| |
| RUN curl https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz \ |
| | tar -C /usr/bin --strip-components=1 -xvzf - docker/docker |
| |
| # Setup PIP |
| # By default PIP install run without cache to make image smaller |
| ARG PIP_NO_CACHE_DIR="true" |
| ENV PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} |
| RUN echo "Pip no cache dir: ${PIP_NO_CACHE_DIR}" |
| |
| # PIP version used to install dependencies |
| ARG PIP_VERSION="19.0.2" |
| ENV PIP_VERSION=${PIP_VERSION} |
| RUN echo "Pip version: ${PIP_VERSION}" |
| |
| RUN pip install --upgrade pip==${PIP_VERSION} |
| |
| ARG HOME=/root |
| ENV HOME=${HOME} |
| |
| ARG AIRFLOW_HOME=/root/airflow |
| ENV AIRFLOW_HOME=${AIRFLOW_HOME} |
| |
| ARG AIRFLOW_SOURCES=/opt/airflow |
| ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES} |
| |
| WORKDIR ${AIRFLOW_SOURCES} |
| |
| RUN mkdir -pv ${AIRFLOW_HOME} && \ |
| mkdir -pv ${AIRFLOW_HOME}/dags && \ |
| mkdir -pv ${AIRFLOW_HOME}/logs |
| |
| # Increase the value here to force reinstalling Apache Airflow pip dependencies |
| ARG PIP_DEPENDENCIES_EPOCH_NUMBER="4" |
| ENV PIP_DEPENDENCIES_EPOCH_NUMBER=${PIP_DEPENDENCIES_EPOCH_NUMBER} |
| |
| # Optimizing installation of Cassandra driver |
| # Speeds up building the image - cassandra driver without CYTHON saves around 10 minutes |
| ARG CASS_DRIVER_NO_CYTHON="1" |
| # Build cassandra driver on multiple CPUs |
| ARG CASS_DRIVER_BUILD_CONCURRENCY="8" |
| |
| ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY} |
| ENV CASS_DRIVER_NO_CYTHON=${CASS_DRIVER_NO_CYTHON} |
| |
| ARG AIRFLOW_REPO=apache/airflow |
| ENV AIRFLOW_REPO=${AIRFLOW_REPO} |
| |
| ARG AIRFLOW_BRANCH=master |
| ENV AIRFLOW_BRANCH=${AIRFLOW_BRANCH} |
| |
| # Airflow Extras installed |
| ARG AIRFLOW_EXTRAS="all" |
| ARG ADDITIONAL_AIRFLOW_EXTRAS="" |
| ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} |
| |
| RUN echo "Installing with extras: ${AIRFLOW_EXTRAS}." |
| |
| ARG AIRFLOW_CONSTRAINTS_REFERENCE="constraints-master" |
| ARG AIRFLOW_CONSTRAINTS_URL="https://raw.githubusercontent.com/apache/airflow/${AIRFLOW_CONSTRAINTS_REFERENCE}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt" |
| ENV AIRFLOW_CONSTRAINTS_URL=${AIRFLOW_CONSTRAINTS_URL} |
| |
| # By changing the CI build epoch we can force reinstalling Arflow from the current master |
| # It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable. |
| ARG AIRFLOW_CI_BUILD_EPOCH="1" |
| ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} |
| |
| # In case of CI builds we want to pre-install master version of airflow dependencies so that |
| # We do not have to always reinstall it from the scratch. |
| # This can be reinstalled from latest master by increasing PIP_DEPENDENCIES_EPOCH_NUMBER. |
| # And is automatically reinstalled from the scratch every time patch release of python gets released |
| RUN pip install \ |
| "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ |
| --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; |
| |
| |
| # Link dumb-init for backwards compatibility (so that older images also work) |
| RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init |
| |
| # Install NPM dependencies here. The NPM dependencies don't change that often and we already have pip |
| # installed dependencies in case of CI optimised build, so it is ok to install NPM deps here |
| # Rather than after setup.py is added. |
| COPY airflow/www_rbac/yarn.lock airflow/www_rbac/package.json ${AIRFLOW_SOURCES}/airflow/www_rbac/ |
| |
| RUN yarn --cwd airflow/www_rbac install --frozen-lockfile --no-cache |
| |
| # Note! We are copying everything with airflow:airflow user:group even if we use root to run the scripts |
| # This is fine as root user will be able to use those dirs anyway. |
| |
| # Airflow sources change frequently but dependency configuration won't change that often |
| # We copy setup.py and other files needed to perform setup of dependencies |
| # So in case setup.py changes we can install latest dependencies required. |
| COPY setup.py ${AIRFLOW_SOURCES}/setup.py |
| COPY setup.cfg ${AIRFLOW_SOURCES}/setup.cfg |
| |
| COPY airflow/version.py ${AIRFLOW_SOURCES}/airflow/version.py |
| COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py |
| COPY airflow/bin/airflow ${AIRFLOW_SOURCES}/airflow/bin/airflow |
| |
| ARG UPGRADE_TO_LATEST_CONSTRAINTS="false" |
| ENV UPGRADE_TO_LATEST_CONSTRAINTS=${UPGRADE_TO_LATEST_CONSTRAINTS} |
| |
| # The goal of this line is to install the dependencies from the most current setup.py from sources |
| # This will be usually incremental small set of packages in CI optimized build, so it will be very fast |
| # In non-CI optimized build this will install all dependencies before installing sources. |
| # Usually we will install versions constrained to the current constraints file |
| # But in cron job we will install latest versions matching setup.py to see if there is no breaking change |
| # and push the constraints if everything is successful |
| RUN \ |
| if [[ "${UPGRADE_TO_LATEST_CONSTRAINTS}" == "true" ]]; then \ |
| pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade --upgrade-strategy eager; \ |
| else \ |
| pip install -e ".[${AIRFLOW_EXTRAS}]" \ |
| --constraint "${AIRFLOW_CONSTRAINTS_URL}" ; \ |
| fi |
| |
| # Copy all the www/ files we need to compile assets. Done as two separate COPY |
| # commands so as otherwise it copies the _contents_ of static/ in to www_rbac/ |
| COPY airflow/www_rbac/webpack.config.js ${AIRFLOW_SOURCES}/airflow/www_rbac/ |
| COPY airflow/www_rbac/static ${AIRFLOW_SOURCES}/airflow/www_rbac/static/ |
| |
| # Package JS/css for production |
| RUN yarn --cwd airflow/www_rbac run prod |
| |
| COPY scripts/ci/in_container/entrypoint_ci.sh /entrypoint |
| RUN chmod a+x /entrypoint |
| |
| # Copy selected subdirectories only |
| COPY .github/ ${AIRFLOW_SOURCES}/.github/ |
| COPY dags/ ${AIRFLOW_SOURCES}/dags/ |
| COPY common/ ${AIRFLOW_SOURCES}/common/ |
| COPY licenses/ ${AIRFLOW_SOURCES}/licenses/ |
| COPY scripts/ ${AIRFLOW_SOURCES}/scripts/ |
| COPY docs/ ${AIRFLOW_SOURCES}/docs/ |
| COPY tests/ ${AIRFLOW_SOURCES}/tests/ |
| COPY airflow/ ${AIRFLOW_SOURCES}/airflow/ |
| COPY .coveragerc .rat-excludes .flake8 LICENSE MANIFEST.in NOTICE CHANGELOG.txt \ |
| .github pytest.ini \ |
| setup.cfg setup.py \ |
| ${AIRFLOW_SOURCES}/ |
| |
| # Install autocomplete for airflow |
| RUN register-python-argcomplete airflow >> ~/.bashrc |
| |
| # Install autocomplete for Kubeclt |
| RUN echo "source /etc/bash_completion" >> ~/.bashrc |
| |
| WORKDIR ${AIRFLOW_SOURCES} |
| |
| # Additional python deps to install |
| ARG ADDITIONAL_PYTHON_DEPS="" |
| |
| RUN if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ |
| pip install ${ADDITIONAL_PYTHON_DEPS}; \ |
| fi |
| |
| WORKDIR ${AIRFLOW_SOURCES} |
| |
| ENV PATH="${HOME}:${PATH}" |
| |
| # Needed to stop Gunicorn from crashing when /tmp is now mounted from host |
| ENV GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm/" |
| |
| EXPOSE 8080 |
| |
| ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] |