sdks/python/tox.ini - beam - Git at Google

 ;
 ;    Licensed to the Apache Software Foundation (ASF) under one or more
 ;    contributor license agreements.  See the NOTICE file distributed with
 ;    this work for additional information regarding copyright ownership.
 ;    The ASF licenses this file to You under the Apache License, Version 2.0
 ;    (the "License"); you may not use this file except in compliance with
 ;    the License.  You may obtain a copy of the License at
 ;
 ;       http://www.apache.org/licenses/LICENSE-2.0
 ;
 ;    Unless required by applicable law or agreed to in writing, software
 ;    distributed under the License is distributed on an "AS IS" BASIS,
 ;    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 ;    See the License for the specific language governing permissions and
 ;    limitations under the License.
 ;

 [tox]
 # new environments will be excluded by default unless explicitly added to envlist.
 envlist = py38,py39,py310,py311,py312,py38-{cloud,cloudcoverage,dask},py39-{cloud},py310-{cloud,dask},py311-{cloud,dask},py312-{cloud,dask},docs,lint,mypy,whitespacelint
 toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox}

 [pycodestyle]
 # Disable all errors and warnings except for the ones related to blank lines.
 # pylint does not check the number of blank lines.
 select = E3

 # Shared environment options.
 [testenv]
 # Run the tests using pre-released dependencies.
 # https://github.com/apache/beam/issues/25668
 pip_pre = True
 # allow apps that support color to use it.
 passenv=TERM,CLOUDSDK_CONFIG
 # Set [] options for pip installation of apache-beam tarball.
 extras = test,dataframe
 # Don't warn that these commands aren't installed.
 allowlist_externals =
   false
   time
   bash
   rm
   /bin/sh
   curl
   ./codecov
   chmod
 setenv =
   RUN_SKIPPED_PY3_TESTS=0
   # Use an isolated tmp dir for tests that get slowed down by scanning /tmp.
   TMPDIR={envtmpdir}
   # Silence warning about ignoring PYTHONPATH.
   PYTHONPATH=

 # These 2 magic command overrides are required for Jenkins builds.
 # Otherwise we get "OSError: [Errno 2] No such file or directory" errors.
 # Source:
 # https://github.com/tox-dev/tox/issues/123#issuecomment-284714629
 install_command = {envbindir}/python {envbindir}/pip install --retries 10 {opts} {packages}
 list_dependencies_command = {envbindir}/python {envbindir}/pip freeze
 commands_pre =
   python --version
   pip --version
   pip check
   bash {toxinidir}/scripts/run_tox_cleanup.sh
 commands_post =
   bash {toxinidir}/scripts/run_tox_cleanup.sh

 commands = false {envname} is misconfigured

 [testenv:py{38,39,310,311,312}]
 commands =
   python apache_beam/examples/complete/autocomplete_test.py
   bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"

 [testenv:py{38,39,310,311,312}-win]
 commands =
   python apache_beam/examples/complete/autocomplete_test.py
   bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"
 install_command = {envbindir}/python.exe {envbindir}/pip.exe install --retries 10 {opts} {packages}
 list_dependencies_command = {envbindir}/python.exe {envbindir}/pip.exe freeze

 [testenv:py{38,39,310,311,312}-cloud]
 ; extras = test,gcp,interactive,dataframe,aws,azure
 extras = test,gcp,interactive,dataframe,aws,azure
 commands =
   python apache_beam/examples/complete/autocomplete_test.py
   bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"

 [testenv:py{38,39,310,311,312}-ml]
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
 setenv =
 extras = test,gcp,dataframe,ml_test
 commands =
   bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"

 [testenv:py{38,39,310,311,312}-dask]
 extras = test,dask
 commands =
   bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"

 [testenv:py38-cloudcoverage]
 deps =
   pytest-cov==3.0.0
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
 setenv =
   PYTHONPATH = {toxinidir}
 platform = linux
 passenv = GIT_*,BUILD_*,ghprb*,CHANGE_ID,BRANCH_NAME,JENKINS_*,CODECOV_*,GITHUB_*
 # NOTE: we could add ml_test to increase the collected code coverage metrics, but it would make the suite slower.
 extras = test,gcp,interactive,dataframe,aws
 commands =
   bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" "--cov-report=xml --cov=. --cov-append"

 [testenv:lint]
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in pylint.
 setenv =
 # keep the version of pylint in sync with the 'rev' in .pre-commit-config.yaml
 deps =
   astroid<2.17.0,>=2.15.6
   pycodestyle==2.8.0
   pylint==2.17.5
   isort==4.2.15
   flake8==4.0.1
 commands =
   pylint --version
   time {toxinidir}/scripts/run_pylint.sh

 [testenv:whitespacelint]
 setenv =
 deps =
   whitespacelint==1.1.0
 commands =
   time {toxinidir}/scripts/run_whitespacelint.sh

 [testenv:mypy]
 deps =
   mypy==0.790
   dask==2022.01.0
   distributed==2022.01.0
 # make extras available in case any of these libs are typed
 extras =
   gcp
 commands =
   mypy --version
   python setup.py mypy


 [testenv:docs]
 extras = test,gcp,docs,interactive,dataframe,dask
 deps =
   Sphinx==1.8.5
   sphinx_rtd_theme==0.4.3
   docutils<0.18
   Jinja2==3.0.3 # TODO(https://github.com/apache/beam/issues/21587): Sphinx version is too old.
 commands =
   time {toxinidir}/scripts/generate_pydoc.sh

 [testenv:hdfs_integration_test]
 # Used by hdfs_integration_test.sh. Do not run this directly, as it depends on
 # nodes defined in hdfs_integration_test/docker-compose.yml.
 deps =
   holdup==1.8.0
 extras =
   gcp
 allowlist_externals =
   bash
   echo
   sleep
   wget
   hdfscli
 passenv = HDFSCLI_CONFIG
 commands =
   holdup -t 45 http://namenode:50070 http://datanode:50075
   echo "Waiting for safe mode to end."
   sleep 45
   wget storage.googleapis.com/dataflow-samples/shakespeare/kinglear.txt
   hdfscli -v -v -v upload -f kinglear.txt /
   python -m apache_beam.examples.wordcount \
       --input hdfs://kinglear* \
       --output hdfs://py-wordcount-integration \
       --hdfs_host namenode --hdfs_port 50070 --hdfs_user root
   python -m apache_beam.examples.wordcount \
       --input hdfs://unused_server/kinglear* \
       --output hdfs://unused_server/py-wordcount-integration \
       --hdfs_host namenode --hdfs_port 50070 --hdfs_user root --hdfs_full_urls
 commands_pre =
   pip check

 [testenv:azure_integration_test]
 # Used by azure/integration_test/azure_integration_test.sh.
 # Do not run this directly, as it depends on nodes defined in
 # azure/integration_test/docker-compose.yml.
 deps =
 extras =
   azure
 passenv = REQUESTS_CA_BUNDLE
 allowlist_externals =
   wget
   az
   bash
 setenv =
   CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://azurite:10000/devstoreaccount1;
 commands_pre =
   pip check
   wget storage.googleapis.com/dataflow-samples/shakespeare/kinglear.txt
   # Create container for storing files.
   az storage container create -n container --connection-string {env:CONNECTION_STRING}
   # Upload test file.
   az storage blob upload -f kinglear.txt -c container -n kinglear.txt --connection-string {env:CONNECTION_STRING}
 commands =
   # Test --azure_connection_string
   python -m apache_beam.examples.wordcount \
       --input azfs://devstoreaccount1/container/* \
       --output azfs://devstoreaccount1/container/py-wordcount-integration \
       --azure_connection_string {env:CONNECTION_STRING}
   # This doesn't work because there's no way to send a fake bearer token to
   # Azurite when using DefaultAzureCredential.
   # See https://github.com/Azure/Azurite/issues/389#issuecomment-615298432
   # and https://github.com/Azure/Azurite/issues/1750#issue-1449778593
   #python -m apache_beam.examples.wordcount \
   #    --input azfs://devstoreaccount1/container/* \
   #    --output azfs://devstoreaccount1/container/py-wordcount-integration \
   #    --blob_service_endpoint https://azurite:10000/devstoreaccount1/container-name \
   #    --azure_managed_identity_client_id "abc123"

 [testenv:py3-yapf]
 # keep the version of yapf in sync with the 'rev' in .pre-commit-config.yaml and pyproject.toml
 deps =
   yapf==0.29.0
 commands =
   yapf --version
   time yapf --in-place --parallel --recursive apache_beam

 [testenv:py3-yapf-check]
 # keep the version of yapf in sync with the 'rev' in .pre-commit-config.yaml and pyproject.toml
 deps =
   yapf==0.29.0
 commands =
   yapf --version
   time yapf --diff --parallel --recursive apache_beam

 [testenv:jest]
 setenv =
 deps =
   jupyterlab==3.1.18
 commands =
   time {toxinidir}/scripts/setup_nodejs.sh
   time {toxinidir}/scripts/run_jest.sh

 [testenv:eslint]
 setenv =
 deps =
   jupyterlab==3.1.18
 commands =
   time {toxinidir}/scripts/setup_nodejs.sh
   time {toxinidir}/scripts/run_eslint.sh

 [testenv:flink-runner-test]
 extras = test
 commands =
   bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/flink_runner_test.py {posargs}

 [testenv:samza-runner-test]
 extras = test
 commands =
   bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/samza_runner_test.py {posargs}

 [testenv:spark-runner-test]
 extras = test
 commands =
   bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs}


 [testenv:py{38,39}-pyarrow-{3,9,10,11,12,13,14,15,16}]
 deps =
   # As a courtesy to users, test against the oldest allowed version of Pyarrow.
   # We'd have to increase the pyarrow lower bound when Python 3.9 is deprecated.
   # Since Pandas 2 requires pyarrow>=7, downgrade pandas for this test.
   3: pyarrow>=3,<4
   3: pandas<2
   # Test against versions of pyarrow released in last ~2 years.
   9: pyarrow>=9,<10
   10: pyarrow>=10,<11
   11: pyarrow>=11,<12
   12: pyarrow>=12,<13
   13: pyarrow>=13,<14
   14: pyarrow>=14,<15
   15: pyarrow>=15,<16
   16: pyarrow>=16,<17
 commands =
   # Log pyarrow and numpy version for debugging
   /bin/sh -c "pip freeze | grep -E '(pyarrow|numpy)'"
   # Run pytest directly rather using run_pytest.sh. It doesn't handle
   # selecting tests with -m (BEAM-12985).
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'


 [testenv:py{38,39}-pandas-{14,15,20}]
 deps =
   14: pandas>=1.4.3,<1.5.0
   # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725
   15: pandas>=1.5.2,<1.6.0
   20: pandas>=2.0.0,<2.1.0
   20: pyarrow>=7
 commands =
   # Log pandas and numpy version for debugging
   /bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"
   # Run all DataFrame API unit tests
   bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/dataframe'

 [testenv:py{38,39}-tft-{113,114}]
 deps =
   # Help pip resolve conflict with typing-extensions due to an old version of tensorflow https://github.com/apache/beam/issues/30852
   113: pydantic<2.0
   114: tensorflow_transform>=1.14.0,<1.15.0
 commands =
   bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py'

 [testenv:py{38,39}-pytorch-{19,110,111,112,113}]
 deps =
   19: torch>=1.9.0,<1.10.0
   110: torch>=1.10.0,<1.11.0
   111: torch>=1.11.0,<1.12.0
   112: torch>=1.12.0,<1.13.0
   113: torch>=1.13.0,<1.14.0
 extras = test,gcp
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
 setenv =
 commands =
   # Log torch version for debugging
   /bin/sh -c "pip freeze | grep -E torch"
   # Run all PyTorch<2 unit tests
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'

 [testenv:py{38,39}-pytorch-200]
 deps =
   200:
     torch>=2.0.0,<2.1.0
     mpmath==1.3.0
 extras = test,gcp
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
 setenv =
 commands =
   # Log torch version for debugging
   /bin/sh -c "pip freeze | grep -E torch"
   # Run all PyTorch>=2  unit tests
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'

 # TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py38/build.gradle once onnx supports protobuf 4.x.x
 [testenv:py{38,39}-onnx-113]
 # TODO(https://github.com/apache/beam/issues/25443)
 # apparently tox has problem when substitution key has single value. Change back to -onnx-{113,...}
 # when multiple onnx versions are tested.
 deps =
   onnxruntime==1.13.1
   pandas==1.5.2
   torch==1.13.1
   tensorflow==2.11.0
   tf2onnx==1.13.0
   skl2onnx==1.13
   transformers==4.25.1
 extras = test,gcp
 commands =
   # Log onnx version for debugging
   /bin/sh -c "pip freeze | grep -E onnx"
   # Run all ONNX unit tests
   pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_onnx {posargs}

 [testenv:py{38,39}-tensorflow-212]
 deps =
   212: tensorflow>=2.12rc1,<2.13
   # Help pip resolve conflict with typing-extensions for old version of TF https://github.com/apache/beam/issues/30852
   212: pydantic<2.7
 extras = test,gcp
 commands =
   # Log tensorflow version for debugging
   /bin/sh -c "pip freeze | grep -E tensorflow"
   # Run all Tensorflow unit tests
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'

 [testenv:py{38,39}-xgboost-{160,170}]
 deps =
   160:
     xgboost>=1.6.0,<1.7.0
     datatable==1.0.0
   170:
     xgboost>=1.7.0
     datatable==1.0.0
 extras = test,gcp
 commands =
   # Log XGBoost version for debugging
   /bin/sh -c "pip freeze | grep -E xgboost"
   # Run all XGBoost unit tests
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_xgboost {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'

 [testenv:py{38,39}-transformers-{428,429,430}]
 deps =
   428: transformers>=4.28.0,<4.29.0
   429: transformers>=4.29.0,<4.30.0
   430: transformers>=4.30.0,<4.31.0
   torch>=1.9.0,<1.14.0
   tensorflow==2.12.0
 extras = test,gcp
 commands =
   # Log transformers and its dependencies version for debugging
   /bin/sh -c "pip freeze | grep -E transformers"
   /bin/sh -c "pip freeze | grep -E torch"
   /bin/sh -c "pip freeze | grep -E tensorflow"
   # Run all Transformers unit tests
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_transformers {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'

 [testenv:py{38,311}-vertex-ai]
 deps =
   tensorflow==2.12.0
 extras = test,gcp
 commands =
   # Log aiplatform and its dependencies version for debugging
   /bin/sh -c "pip freeze | grep -E google-cloud-aiplatform"
   /bin/sh -c "pip freeze | grep -E tensorflow"
   # Run all Vertex AI unit tests
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_vertex_ai {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'


 [testenv:py{38,39}-embeddings]
 deps =
   sentence-transformers==2.2.2
 passenv = HF_INFERENCE_TOKEN
 extras = test,gcp
 commands =
   # Log aiplatform and its dependencies version for debugging
   /bin/sh -c "pip freeze | grep -E sentence-transformers"
   /bin/sh -c "pip freeze | grep -E google-cloud-aiplatform"
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest apache_beam/ml/transforms/embeddings -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'


 [testenv:py{38,39}-TFHubEmbeddings-{014,015}]
 deps =
   014: tensorflow-hub>=0.14.0,<0.15.0
   # Help pip resolve conflict with typing-extensions due to an old version of tensorboard https://github.com/apache/beam/issues/30852
   014: pydantic<2.7
   015: tensorflow-hub>=0.15.0,<0.16.0
   # Help pip resolve conflict with typing-extensions due to an old version of tensorboard https://github.com/apache/beam/issues/30852
   015: pydantic<2.7
   tensorflow-text # required to register ops for text embedding models.

 extras = test,gcp
 commands =
   # Log aiplatform and its dependencies version for debugging
   /bin/sh -c "pip freeze | grep -E tensorflow"
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms/embeddings'
	;
	; Licensed to the Apache Software Foundation (ASF) under one or more
	; contributor license agreements. See the NOTICE file distributed with
	; this work for additional information regarding copyright ownership.
	; The ASF licenses this file to You under the Apache License, Version 2.0
	; (the "License"); you may not use this file except in compliance with
	; the License. You may obtain a copy of the License at
	;
	; http://www.apache.org/licenses/LICENSE-2.0
	;
	; Unless required by applicable law or agreed to in writing, software
	; distributed under the License is distributed on an "AS IS" BASIS,
	; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	; See the License for the specific language governing permissions and
	; limitations under the License.
	;

	[tox]
	# new environments will be excluded by default unless explicitly added to envlist.
	envlist = py38,py39,py310,py311,py312,py38-{cloud,cloudcoverage,dask},py39-{cloud},py310-{cloud,dask},py311-{cloud,dask},py312-{cloud,dask},docs,lint,mypy,whitespacelint
	toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox}

	[pycodestyle]
	# Disable all errors and warnings except for the ones related to blank lines.
	# pylint does not check the number of blank lines.
	select = E3

	# Shared environment options.
	[testenv]
	# Run the tests using pre-released dependencies.
	# https://github.com/apache/beam/issues/25668
	pip_pre = True
	# allow apps that support color to use it.
	passenv=TERM,CLOUDSDK_CONFIG
	# Set [] options for pip installation of apache-beam tarball.
	extras = test,dataframe
	# Don't warn that these commands aren't installed.
	allowlist_externals =
	false
	time
	bash
	rm
	/bin/sh
	curl
	./codecov
	chmod
	setenv =
	RUN_SKIPPED_PY3_TESTS=0
	# Use an isolated tmp dir for tests that get slowed down by scanning /tmp.
	TMPDIR={envtmpdir}
	# Silence warning about ignoring PYTHONPATH.
	PYTHONPATH=

	# These 2 magic command overrides are required for Jenkins builds.
	# Otherwise we get "OSError: [Errno 2] No such file or directory" errors.
	# Source:
	# https://github.com/tox-dev/tox/issues/123#issuecomment-284714629
	install_command = {envbindir}/python {envbindir}/pip install --retries 10 {opts} {packages}
	list_dependencies_command = {envbindir}/python {envbindir}/pip freeze
	commands_pre =
	python --version
	pip --version
	pip check
	bash {toxinidir}/scripts/run_tox_cleanup.sh
	commands_post =
	bash {toxinidir}/scripts/run_tox_cleanup.sh

	commands = false {envname} is misconfigured

	[testenv:py{38,39,310,311,312}]
	commands =
	python apache_beam/examples/complete/autocomplete_test.py
	bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"

	[testenv:py{38,39,310,311,312}-win]
	commands =
	python apache_beam/examples/complete/autocomplete_test.py
	bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"
	install_command = {envbindir}/python.exe {envbindir}/pip.exe install --retries 10 {opts} {packages}
	list_dependencies_command = {envbindir}/python.exe {envbindir}/pip.exe freeze

	[testenv:py{38,39,310,311,312}-cloud]
	; extras = test,gcp,interactive,dataframe,aws,azure
	extras = test,gcp,interactive,dataframe,aws,azure
	commands =
	python apache_beam/examples/complete/autocomplete_test.py
	bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"

	[testenv:py{38,39,310,311,312}-ml]
	# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
	setenv =
	extras = test,gcp,dataframe,ml_test
	commands =
	bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"

	[testenv:py{38,39,310,311,312}-dask]
	extras = test,dask
	commands =
	bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"

	[testenv:py38-cloudcoverage]
	deps =
	pytest-cov==3.0.0
	# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
	setenv =
	PYTHONPATH = {toxinidir}
	platform = linux
	passenv = GIT_,BUILD_,ghprb,CHANGE_ID,BRANCH_NAME,JENKINS_,CODECOV_,GITHUB_
	# NOTE: we could add ml_test to increase the collected code coverage metrics, but it would make the suite slower.
	extras = test,gcp,interactive,dataframe,aws
	commands =
	bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" "--cov-report=xml --cov=. --cov-append"

	[testenv:lint]
	# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in pylint.
	setenv =
	# keep the version of pylint in sync with the 'rev' in .pre-commit-config.yaml
	deps =
	astroid<2.17.0,>=2.15.6
	pycodestyle==2.8.0
	pylint==2.17.5
	isort==4.2.15
	flake8==4.0.1
	commands =
	pylint --version
	time {toxinidir}/scripts/run_pylint.sh

	[testenv:whitespacelint]
	setenv =
	deps =
	whitespacelint==1.1.0
	commands =
	time {toxinidir}/scripts/run_whitespacelint.sh

	[testenv:mypy]
	deps =
	mypy==0.790
	dask==2022.01.0
	distributed==2022.01.0
	# make extras available in case any of these libs are typed
	extras =
	gcp
	commands =
	mypy --version
	python setup.py mypy


	[testenv:docs]
	extras = test,gcp,docs,interactive,dataframe,dask
	deps =
	Sphinx==1.8.5
	sphinx_rtd_theme==0.4.3
	docutils<0.18
	Jinja2==3.0.3 # TODO(https://github.com/apache/beam/issues/21587): Sphinx version is too old.
	commands =
	time {toxinidir}/scripts/generate_pydoc.sh

	[testenv:hdfs_integration_test]
	# Used by hdfs_integration_test.sh. Do not run this directly, as it depends on
	# nodes defined in hdfs_integration_test/docker-compose.yml.
	deps =
	holdup==1.8.0
	extras =
	gcp
	allowlist_externals =
	bash
	echo
	sleep
	wget
	hdfscli
	passenv = HDFSCLI_CONFIG
	commands =
	holdup -t 45 http://namenode:50070 http://datanode:50075
	echo "Waiting for safe mode to end."
	sleep 45
	wget storage.googleapis.com/dataflow-samples/shakespeare/kinglear.txt
	hdfscli -v -v -v upload -f kinglear.txt /
	python -m apache_beam.examples.wordcount \
	--input hdfs://kinglear* \
	--output hdfs://py-wordcount-integration \
	--hdfs_host namenode --hdfs_port 50070 --hdfs_user root
	python -m apache_beam.examples.wordcount \
	--input hdfs://unused_server/kinglear* \
	--output hdfs://unused_server/py-wordcount-integration \
	--hdfs_host namenode --hdfs_port 50070 --hdfs_user root --hdfs_full_urls
	commands_pre =
	pip check

	[testenv:azure_integration_test]
	# Used by azure/integration_test/azure_integration_test.sh.
	# Do not run this directly, as it depends on nodes defined in
	# azure/integration_test/docker-compose.yml.
	deps =
	extras =
	azure
	passenv = REQUESTS_CA_BUNDLE
	allowlist_externals =
	wget
	az
	bash
	setenv =
	CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://azurite:10000/devstoreaccount1;
	commands_pre =
	pip check
	wget storage.googleapis.com/dataflow-samples/shakespeare/kinglear.txt
	# Create container for storing files.
	az storage container create -n container --connection-string {env:CONNECTION_STRING}
	# Upload test file.
	az storage blob upload -f kinglear.txt -c container -n kinglear.txt --connection-string {env:CONNECTION_STRING}
	commands =
	# Test --azure_connection_string
	python -m apache_beam.examples.wordcount \
	--input azfs://devstoreaccount1/container/* \
	--output azfs://devstoreaccount1/container/py-wordcount-integration \
	--azure_connection_string {env:CONNECTION_STRING}
	# This doesn't work because there's no way to send a fake bearer token to
	# Azurite when using DefaultAzureCredential.
	# See https://github.com/Azure/Azurite/issues/389#issuecomment-615298432
	# and https://github.com/Azure/Azurite/issues/1750#issue-1449778593
	#python -m apache_beam.examples.wordcount \
	# --input azfs://devstoreaccount1/container/* \
	# --output azfs://devstoreaccount1/container/py-wordcount-integration \
	# --blob_service_endpoint https://azurite:10000/devstoreaccount1/container-name \
	# --azure_managed_identity_client_id "abc123"

	[testenv:py3-yapf]
	# keep the version of yapf in sync with the 'rev' in .pre-commit-config.yaml and pyproject.toml
	deps =
	yapf==0.29.0
	commands =
	yapf --version
	time yapf --in-place --parallel --recursive apache_beam

	[testenv:py3-yapf-check]
	# keep the version of yapf in sync with the 'rev' in .pre-commit-config.yaml and pyproject.toml
	deps =
	yapf==0.29.0
	commands =
	yapf --version
	time yapf --diff --parallel --recursive apache_beam

	[testenv:jest]
	setenv =
	deps =
	jupyterlab==3.1.18
	commands =
	time {toxinidir}/scripts/setup_nodejs.sh
	time {toxinidir}/scripts/run_jest.sh

	[testenv:eslint]
	setenv =
	deps =
	jupyterlab==3.1.18
	commands =
	time {toxinidir}/scripts/setup_nodejs.sh
	time {toxinidir}/scripts/run_eslint.sh

	[testenv:flink-runner-test]
	extras = test
	commands =
	bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/flink_runner_test.py {posargs}

	[testenv:samza-runner-test]
	extras = test
	commands =
	bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/samza_runner_test.py {posargs}

	[testenv:spark-runner-test]
	extras = test
	commands =
	bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs}


	[testenv:py{38,39}-pyarrow-{3,9,10,11,12,13,14,15,16}]
	deps =
	# As a courtesy to users, test against the oldest allowed version of Pyarrow.
	# We'd have to increase the pyarrow lower bound when Python 3.9 is deprecated.
	# Since Pandas 2 requires pyarrow>=7, downgrade pandas for this test.
	3: pyarrow>=3,<4
	3: pandas<2
	# Test against versions of pyarrow released in last ~2 years.
	9: pyarrow>=9,<10
	10: pyarrow>=10,<11
	11: pyarrow>=11,<12
	12: pyarrow>=12,<13
	13: pyarrow>=13,<14
	14: pyarrow>=14,<15
	15: pyarrow>=15,<16
	16: pyarrow>=16,<17
	commands =
	# Log pyarrow and numpy version for debugging
	/bin/sh -c "pip freeze \| grep -E '(pyarrow\|numpy)'"
	# Run pytest directly rather using run_pytest.sh. It doesn't handle
	# selecting tests with -m (BEAM-12985).
	# Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
	/bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 \|\| exit $ret'


	[testenv:py{38,39}-pandas-{14,15,20}]
	deps =
	14: pandas>=1.4.3,<1.5.0
	# Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725
	15: pandas>=1.5.2,<1.6.0
	20: pandas>=2.0.0,<2.1.0
	20: pyarrow>=7
	commands =
	# Log pandas and numpy version for debugging
	/bin/sh -c "pip freeze \| grep -E '(pandas\|numpy)'"
	# Run all DataFrame API unit tests
	bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/dataframe'

	[testenv:py{38,39}-tft-{113,114}]
	deps =
	# Help pip resolve conflict with typing-extensions due to an old version of tensorflow https://github.com/apache/beam/issues/30852
	113: pydantic<2.0
	114: tensorflow_transform>=1.14.0,<1.15.0
	commands =
	bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py'

	[testenv:py{38,39}-pytorch-{19,110,111,112,113}]
	deps =
	19: torch>=1.9.0,<1.10.0
	110: torch>=1.10.0,<1.11.0
	111: torch>=1.11.0,<1.12.0
	112: torch>=1.12.0,<1.13.0
	113: torch>=1.13.0,<1.14.0
	extras = test,gcp
	# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
	setenv =
	commands =
	# Log torch version for debugging
	/bin/sh -c "pip freeze \| grep -E torch"
	# Run all PyTorch<2 unit tests
	# Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
	/bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 \|\| exit $ret'

	[testenv:py{38,39}-pytorch-200]
	deps =
	200:
	torch>=2.0.0,<2.1.0
	mpmath==1.3.0
	extras = test,gcp
	# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
	setenv =
	commands =
	# Log torch version for debugging
	/bin/sh -c "pip freeze \| grep -E torch"
	# Run all PyTorch>=2 unit tests
	# Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
	/bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 \|\| exit $ret'

	# TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py38/build.gradle once onnx supports protobuf 4.x.x
	[testenv:py{38,39}-onnx-113]
	# TODO(https://github.com/apache/beam/issues/25443)
	# apparently tox has problem when substitution key has single value. Change back to -onnx-{113,...}
	# when multiple onnx versions are tested.
	deps =
	onnxruntime==1.13.1
	pandas==1.5.2
	torch==1.13.1
	tensorflow==2.11.0
	tf2onnx==1.13.0
	skl2onnx==1.13
	transformers==4.25.1
	extras = test,gcp
	commands =
	# Log onnx version for debugging
	/bin/sh -c "pip freeze \| grep -E onnx"
	# Run all ONNX unit tests
	pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_onnx {posargs}

	[testenv:py{38,39}-tensorflow-212]
	deps =
	212: tensorflow>=2.12rc1,<2.13
	# Help pip resolve conflict with typing-extensions for old version of TF https://github.com/apache/beam/issues/30852
	212: pydantic<2.7
	extras = test,gcp
	commands =
	# Log tensorflow version for debugging
	/bin/sh -c "pip freeze \| grep -E tensorflow"
	# Run all Tensorflow unit tests
	# Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
	/bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 \|\| exit $ret'

	[testenv:py{38,39}-xgboost-{160,170}]
	deps =
	160:
	xgboost>=1.6.0,<1.7.0
	datatable==1.0.0
	170:
	xgboost>=1.7.0
	datatable==1.0.0
	extras = test,gcp
	commands =
	# Log XGBoost version for debugging
	/bin/sh -c "pip freeze \| grep -E xgboost"
	# Run all XGBoost unit tests
	# Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
	/bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_xgboost {posargs}; ret=$?; [ $ret = 5 ] && exit 0 \|\| exit $ret'

	[testenv:py{38,39}-transformers-{428,429,430}]
	deps =
	428: transformers>=4.28.0,<4.29.0
	429: transformers>=4.29.0,<4.30.0
	430: transformers>=4.30.0,<4.31.0
	torch>=1.9.0,<1.14.0
	tensorflow==2.12.0
	extras = test,gcp
	commands =
	# Log transformers and its dependencies version for debugging
	/bin/sh -c "pip freeze \| grep -E transformers"
	/bin/sh -c "pip freeze \| grep -E torch"
	/bin/sh -c "pip freeze \| grep -E tensorflow"
	# Run all Transformers unit tests
	# Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
	/bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_transformers {posargs}; ret=$?; [ $ret = 5 ] && exit 0 \|\| exit $ret'

	[testenv:py{38,311}-vertex-ai]
	deps =
	tensorflow==2.12.0
	extras = test,gcp
	commands =
	# Log aiplatform and its dependencies version for debugging
	/bin/sh -c "pip freeze \| grep -E google-cloud-aiplatform"
	/bin/sh -c "pip freeze \| grep -E tensorflow"
	# Run all Vertex AI unit tests
	# Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
	/bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_vertex_ai {posargs}; ret=$?; [ $ret = 5 ] && exit 0 \|\| exit $ret'


	[testenv:py{38,39}-embeddings]
	deps =
	sentence-transformers==2.2.2
	passenv = HF_INFERENCE_TOKEN
	extras = test,gcp
	commands =
	# Log aiplatform and its dependencies version for debugging
	/bin/sh -c "pip freeze \| grep -E sentence-transformers"
	/bin/sh -c "pip freeze \| grep -E google-cloud-aiplatform"
	# Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
	/bin/sh -c 'pytest apache_beam/ml/transforms/embeddings -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 {posargs}; ret=$?; [ $ret = 5 ] && exit 0 \|\| exit $ret'


	[testenv:py{38,39}-TFHubEmbeddings-{014,015}]
	deps =
	014: tensorflow-hub>=0.14.0,<0.15.0
	# Help pip resolve conflict with typing-extensions due to an old version of tensorboard https://github.com/apache/beam/issues/30852
	014: pydantic<2.7
	015: tensorflow-hub>=0.15.0,<0.16.0
	# Help pip resolve conflict with typing-extensions due to an old version of tensorboard https://github.com/apache/beam/issues/30852
	015: pydantic<2.7
	tensorflow-text # required to register ops for text embedding models.

	extras = test,gcp
	commands =
	# Log aiplatform and its dependencies version for debugging
	/bin/sh -c "pip freeze \| grep -E tensorflow"
	# Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
	bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms/embeddings'