| ; |
| ; Licensed to the Apache Software Foundation (ASF) under one or more |
| ; contributor license agreements. See the NOTICE file distributed with |
| ; this work for additional information regarding copyright ownership. |
| ; The ASF licenses this file to You under the Apache License, Version 2.0 |
| ; (the "License"); you may not use this file except in compliance with |
| ; the License. You may obtain a copy of the License at |
| ; |
| ; http://www.apache.org/licenses/LICENSE-2.0 |
| ; |
| ; Unless required by applicable law or agreed to in writing, software |
| ; distributed under the License is distributed on an "AS IS" BASIS, |
| ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| ; See the License for the specific language governing permissions and |
| ; limitations under the License. |
| ; |
| |
| [tox] |
| # new environments will be excluded by default unless explicitly added to envlist. |
| envlist = py27,py35,py36,py37,py27-{gcp,cython,lint,lint3},py35-{gcp,cython},py36-{gcp,cython},py37-{gcp,cython,lint,mypy},docs |
| toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox} |
| |
| [pycodestyle] |
| # Disable all errors and warnings except for the ones related to blank lines. |
| # pylint does not check the number of blank lines. |
| select = E3 |
| |
| # Shared environment options. |
| [testenv] |
| # Set [] options for pip installation of apache-beam tarball. |
| extras = test |
| # Don't warn that these commands aren't installed. |
| whitelist_externals = |
| false |
| time |
| deps = |
| cython: cython==0.28.1 |
| -r build-requirements.txt |
| setenv = |
| RUN_SKIPPED_PY3_TESTS=0 |
| # Use an isolated tmp dir for tests that get slowed down by scanning /tmp. |
| TMPDIR={envtmpdir} |
| |
| # These 2 magic command overrides are required for Jenkins builds. |
| # Otherwise we get "OSError: [Errno 2] No such file or directory" errors. |
| # Source: |
| # https://github.com/tox-dev/tox/issues/123#issuecomment-284714629 |
| install_command = {envbindir}/python {envbindir}/pip install --retries 10 {opts} {packages} |
| list_dependencies_command = {envbindir}/python {envbindir}/pip freeze |
| commands_pre = |
| python --version |
| pip --version |
| pip check |
| {toxinidir}/scripts/run_tox_cleanup.sh |
| commands_post = |
| {toxinidir}/scripts/run_tox_cleanup.sh |
| commands = false {envname} is misconfigured |
| |
| [testenv:py27] |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| python setup.py nosetests --ignore-files '.*py3\d?\.py$' {posargs} |
| |
| [testenv:py27-pytest] |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py35] |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| python setup.py nosetests --ignore-files '.*py3[6-9]\.py$' {posargs} |
| |
| [testenv:py35-pytest] |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py36] |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| python setup.py nosetests --ignore-files '.*py3[7-9]\.py$' {posargs} |
| |
| [testenv:py36-pytest] |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py37] |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| python setup.py nosetests --ignore-files '.*py3[8-9]\.py$' {posargs} |
| |
| [testenv:py37-pytest] |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py27-cython] |
| # cython tests are only expected to work in linux (2.x and 3.x) |
| # If we want to add other platforms in the future, it should be: |
| # `platform = linux2|darwin|...` |
| # See https://docs.python.org/2/library/sys.html#sys.platform for platform codes |
| platform = linux2 |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| python setup.py nosetests --ignore-files '.*py3\d?\.py$' {posargs} |
| |
| [testenv:py27-cython-pytest] |
| # cython tests are only expected to work in linux (2.x and 3.x) |
| # If we want to add other platforms in the future, it should be: |
| # `platform = linux2|darwin|...` |
| # See https://docs.python.org/2/library/sys.html#sys.platform for platform codes |
| platform = linux2 |
| commands = |
| # TODO(#10038): Remove this build_ext invocation once local source no longer |
| # shadows the installed apache_beam. |
| python setup.py build_ext --inplace |
| python apache_beam/examples/complete/autocomplete_test.py |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py35-cython] |
| # cython tests are only expected to work in linux (2.x and 3.x) |
| # If we want to add other platforms in the future, it should be: |
| # `platform = linux2|darwin|...` |
| # See https://docs.python.org/2/library/sys.html#sys.platform for platform codes |
| platform = linux |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| python setup.py nosetests --ignore-files '.*py3[5-9]\.py$' {posargs} |
| |
| [testenv:py35-cython-pytest] |
| # cython tests are only expected to work in linux (2.x and 3.x) |
| # If we want to add other platforms in the future, it should be: |
| # `platform = linux2|darwin|...` |
| # See https://docs.python.org/2/library/sys.html#sys.platform for platform codes |
| platform = linux |
| commands = |
| # TODO(BEAM-8954): Remove this build_ext invocation once local source no longer |
| # shadows the installed apache_beam. |
| python setup.py build_ext --inplace |
| python apache_beam/examples/complete/autocomplete_test.py |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py36-cython] |
| # cython tests are only expected to work in linux (2.x and 3.x) |
| # If we want to add other platforms in the future, it should be: |
| # `platform = linux2|darwin|...` |
| # See https://docs.python.org/2/library/sys.html#sys.platform for platform codes |
| platform = linux |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| python setup.py nosetests --ignore-files '.*py3[7-9]\.py$' {posargs} |
| |
| [testenv:py36-cython-pytest] |
| # cython tests are only expected to work in linux (2.x and 3.x) |
| # If we want to add other platforms in the future, it should be: |
| # `platform = linux2|darwin|...` |
| # See https://docs.python.org/2/library/sys.html#sys.platform for platform codes |
| platform = linux |
| commands = |
| # TODO(BEAM-8954): Remove this build_ext invocation once local source no longer |
| # shadows the installed apache_beam. |
| python setup.py build_ext --inplace |
| python apache_beam/examples/complete/autocomplete_test.py |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py37-cython] |
| # cython tests are only expected to work in linux (2.x and 3.x) |
| # If we want to add other platforms in the future, it should be: |
| # `platform = linux2|darwin|...` |
| # See https://docs.python.org/2/library/sys.html#sys.platform for platform codes |
| platform = linux |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| python setup.py nosetests --ignore-files '.*py3[8-9]\.py$' {posargs} |
| |
| [testenv:py37-cython-pytest] |
| # cython tests are only expected to work in linux (2.x and 3.x) |
| # If we want to add other platforms in the future, it should be: |
| # `platform = linux2|darwin|...` |
| # See https://docs.python.org/2/library/sys.html#sys.platform for platform codes |
| platform = linux |
| commands = |
| # TODO(BEAM-8954): Remove this build_ext invocation once local source no longer |
| # shadows the installed apache_beam. |
| python setup.py build_ext --inplace |
| python apache_beam/examples/complete/autocomplete_test.py |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py27-gcp] |
| extras = test,gcp |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| python setup.py nosetests --ignore-files '.*py3\d?\.py$' {posargs} |
| # Old and new Datastore client unit tests cannot be run in the same process |
| # due to conflicting protobuf modules. |
| # TODO(BEAM-4543): Remove these separate nosetests invocations once the |
| # googledatastore dependency is removed. |
| python setup.py nosetests {posargs} --tests apache_beam.io.gcp.datastore.v1 --xunit-file nosetests-v1.xml |
| python setup.py nosetests {posargs} --tests apache_beam.io.gcp.datastore.v1new --xunit-file nosetests-v1new.xml |
| |
| [testenv:py27-gcp-pytest] |
| extras = test,gcp |
| commands = |
| python apache_beam/examples/complete/autocomplete_test.py |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| # Old and new Datastore client unit tests cannot be run in the same process |
| # due to conflicting protobuf modules. |
| # TODO(BEAM-4543): Remove these separate nosetests invocations once the |
| # googledatastore dependency is removed. |
| pytest -o junit_suite_name={envname}_v1 --junitxml=pytest_{envname}_v1.xml apache_beam/io/gcp/datastore/v1 |
| pytest -o junit_suite_name={envname}_v1new --junitxml=pytest_{envname}_v1new.xml apache_beam/io/gcp/datastore/v1new |
| |
| [testenv:py35-gcp] |
| extras = test,gcp |
| commands = |
| python setup.py nosetests --ignore-files '.*py3[6-9]\.py$' {posargs} |
| |
| [testenv:py35-gcp-pytest] |
| extras = test,gcp |
| commands = |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py36-gcp] |
| extras = test,gcp,interactive |
| commands = |
| python setup.py nosetests --ignore-files '.*py3[7-9]\.py$' {posargs} |
| |
| [testenv:py36-gcp-pytest] |
| extras = test,gcp,interactive |
| commands = |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py37-gcp] |
| extras = test,gcp,interactive |
| commands = |
| python setup.py nosetests --ignore-files '.*py3[8-9]\.py$' {posargs} |
| |
| [testenv:py37-gcp-pytest] |
| extras = test,gcp,interactive |
| commands = |
| {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" |
| |
| [testenv:py37-aws] |
| extras = test,aws |
| commands = |
| python setup.py nosetests --ignore-files '.*py3[8-9]\.py$' {posargs} |
| |
| [testenv:py27-lint] |
| # Checks for py2 syntax errors |
| deps = |
| -r build-requirements.txt |
| flake8==3.5.0 |
| commands = |
| time {toxinidir}/scripts/run_mini_py2lint.sh |
| |
| [testenv:py27-lint3] |
| # Checks for py2/3 compatibility issues |
| # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in pylint. |
| setenv = |
| deps = |
| -r build-requirements.txt |
| pycodestyle==2.3.1 |
| pylint==1.9.3 |
| isort==4.2.15 |
| flake8==3.5.0 |
| commands = |
| pylint --version |
| time {toxinidir}/scripts/run_pylint_2to3.sh |
| |
| [testenv:py37-lint] |
| # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in pylint. |
| setenv = |
| deps = |
| -r build-requirements.txt |
| astroid<2.4,>=2.3.0 |
| pycodestyle==2.3.1 |
| pylint==2.4.3 |
| isort==4.2.15 |
| flake8==3.5.0 |
| commands = |
| pylint --version |
| time {toxinidir}/scripts/run_pylint.sh |
| |
| [testenv:py37-mypy] |
| deps = |
| -r build-requirements.txt |
| mypy==0.730 |
| # make extras available in case any of these libs are typed |
| extras = |
| gcp |
| # TODO: enable c test failures |
| commands = |
| mypy --version |
| - python setup.py mypy |
| |
| [testenv:docs] |
| extras = test,gcp,docs,interactive |
| deps = |
| Sphinx==1.6.5 |
| sphinx_rtd_theme==0.2.4 |
| commands = |
| time {toxinidir}/scripts/generate_pydoc.sh |
| |
| [testenv:cover] |
| # This is not included in envlist which is defined in [tox]. |
| deps = |
| coverage==4.4.1 |
| commands = |
| # Clean up previously collected data. |
| coverage erase |
| coverage run setup.py test |
| # Print coverage report to STDOUT |
| coverage report --skip-covered |
| # Generate report in xml format |
| coverage xml |
| |
| [testenv:hdfs_integration_test] |
| # Used by hdfs_integration_test.sh. Do not run this directly, as it depends on |
| # nodes defined in hdfs_integration_test/docker-compose.yml. |
| deps = |
| -r build-requirements.txt |
| gsutil==4.47 |
| holdup==1.8.0 |
| extras = |
| gcp |
| whitelist_externals = |
| echo |
| sleep |
| passenv = HDFSCLI_CONFIG |
| commands = |
| holdup -t 45 http://namenode:50070 http://datanode:50075 |
| echo "Waiting for safe mode to end." |
| sleep 45 |
| gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt . |
| hdfscli -v -v -v upload -f kinglear.txt / |
| python -m apache_beam.examples.wordcount \ |
| --input hdfs://kinglear* \ |
| --output hdfs://py-wordcount-integration \ |
| --hdfs_host namenode --hdfs_port 50070 --hdfs_user root |
| # Disable pip check. TODO: remove this once gsutil does not conflict with |
| # apache_beam (oauth2client). |
| commands_pre = |