| #!/usr/bin/env bash |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| # define test binaries + versions |
| FLAKE8_BUILD="flake8" |
| MINIMUM_FLAKE8="3.5.0" |
| MYPY_BUILD="mypy" |
| PYCODESTYLE_BUILD="pycodestyle" |
| MINIMUM_PYCODESTYLE="2.6.0" |
| |
| SPHINX_BUILD="sphinx-build" |
| |
| PYTHON_EXECUTABLE="python3" |
| |
| function satisfies_min_version { |
| local provided_version="$1" |
| local expected_version="$2" |
| echo "$( |
| "$PYTHON_EXECUTABLE" << EOM |
| from setuptools.extern.packaging import version |
| print(version.parse('$provided_version') >= version.parse('$expected_version')) |
| EOM |
| )" |
| } |
| |
| function compile_python_test { |
| local COMPILE_STATUS= |
| local COMPILE_REPORT= |
| |
| if [[ ! "$1" ]]; then |
| echo "No python files found! Something is very wrong -- exiting." |
| exit 1; |
| fi |
| |
| # compileall: https://docs.python.org/3/library/compileall.html |
| echo "starting python compilation test..." |
| COMPILE_REPORT=$( ("$PYTHON_EXECUTABLE" -B -mcompileall -q -l -x "[/\\\\][.]git" $1) 2>&1) |
| COMPILE_STATUS=$? |
| |
| if [ $COMPILE_STATUS -ne 0 ]; then |
| echo "Python compilation failed with the following errors:" |
| echo "$COMPILE_REPORT" |
| echo "$COMPILE_STATUS" |
| exit "$COMPILE_STATUS" |
| else |
| echo "python compilation succeeded." |
| echo |
| fi |
| } |
| |
| function pycodestyle_test { |
| local PYCODESTYLE_STATUS= |
| local PYCODESTYLE_REPORT= |
| local RUN_LOCAL_PYCODESTYLE= |
| local PYCODESTYLE_VERSION= |
| local EXPECTED_PYCODESTYLE= |
| local PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$MINIMUM_PYCODESTYLE.py" |
| local PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$MINIMUM_PYCODESTYLE/pycodestyle.py" |
| |
| if [[ ! "$1" ]]; then |
| echo "No python files found! Something is very wrong -- exiting." |
| exit 1; |
| fi |
| |
| # check for locally installed pycodestyle & version |
| RUN_LOCAL_PYCODESTYLE="False" |
| if hash "$PYCODESTYLE_BUILD" 2> /dev/null; then |
| PYCODESTYLE_VERSION="$($PYCODESTYLE_BUILD --version)" |
| EXPECTED_PYCODESTYLE="$(satisfies_min_version $PYCODESTYLE_VERSION $MINIMUM_PYCODESTYLE)" |
| if [ "$EXPECTED_PYCODESTYLE" == "True" ]; then |
| RUN_LOCAL_PYCODESTYLE="True" |
| fi |
| fi |
| |
| # download the right version or run locally |
| if [ $RUN_LOCAL_PYCODESTYLE == "False" ]; then |
| # Get pycodestyle at runtime so that we don't rely on it being installed on the build server. |
| # See: https://github.com/apache/spark/pull/1744#issuecomment-50982162 |
| # Updated to the latest official version of pep8. pep8 is formally renamed to pycodestyle. |
| echo "downloading pycodestyle from $PYCODESTYLE_SCRIPT_REMOTE_PATH..." |
| if [ ! -e "$PYCODESTYLE_SCRIPT_PATH" ]; then |
| curl --silent -o "$PYCODESTYLE_SCRIPT_PATH" "$PYCODESTYLE_SCRIPT_REMOTE_PATH" |
| local curl_status="$?" |
| |
| if [ "$curl_status" -ne 0 ]; then |
| echo "Failed to download pycodestyle.py from $PYCODESTYLE_SCRIPT_REMOTE_PATH" |
| exit "$curl_status" |
| fi |
| fi |
| |
| echo "starting pycodestyle test..." |
| PYCODESTYLE_REPORT=$( ("$PYTHON_EXECUTABLE" "$PYCODESTYLE_SCRIPT_PATH" --config=dev/tox.ini $1) 2>&1) |
| PYCODESTYLE_STATUS=$? |
| else |
| # we have the right version installed, so run locally |
| echo "starting pycodestyle test..." |
| PYCODESTYLE_REPORT=$( ($PYCODESTYLE_BUILD --config=dev/tox.ini $1) 2>&1) |
| PYCODESTYLE_STATUS=$? |
| fi |
| |
| if [ $PYCODESTYLE_STATUS -ne 0 ]; then |
| echo "pycodestyle checks failed:" |
| echo "$PYCODESTYLE_REPORT" |
| exit "$PYCODESTYLE_STATUS" |
| else |
| echo "pycodestyle checks passed." |
| echo |
| fi |
| } |
| |
| function mypy_test { |
| local MYPY_REPORT= |
| local MYPY_STATUS= |
| |
| # TODO(SPARK-32797): Install mypy on the Jenkins CI workers |
| if ! hash "$MYPY_BUILD" 2> /dev/null; then |
| echo "The $MYPY_BUILD command was not found. Skipping for now." |
| return |
| fi |
| |
| echo "starting $MYPY_BUILD test..." |
| MYPY_REPORT=$( ($MYPY_BUILD --config-file python/mypy.ini python/pyspark) 2>&1) |
| MYPY_STATUS=$? |
| |
| if [ "$MYPY_STATUS" -ne 0 ]; then |
| echo "mypy checks failed:" |
| echo "$MYPY_REPORT" |
| echo "$MYPY_STATUS" |
| exit "$MYPY_STATUS" |
| else |
| echo "mypy checks passed." |
| echo |
| fi |
| } |
| |
| function flake8_test { |
| local FLAKE8_VERSION= |
| local EXPECTED_FLAKE8= |
| local FLAKE8_REPORT= |
| local FLAKE8_STATUS= |
| |
| if ! hash "$FLAKE8_BUILD" 2> /dev/null; then |
| echo "The flake8 command was not found." |
| echo "flake8 checks failed." |
| exit 1 |
| fi |
| |
| _FLAKE8_VERSION=($($FLAKE8_BUILD --version)) |
| FLAKE8_VERSION="${_FLAKE8_VERSION[0]}" |
| EXPECTED_FLAKE8="$(satisfies_min_version $FLAKE8_VERSION $MINIMUM_FLAKE8)" |
| |
| if [[ "$EXPECTED_FLAKE8" == "False" ]]; then |
| echo "\ |
| The minimum flake8 version needs to be $MINIMUM_FLAKE8. Your current version is $FLAKE8_VERSION |
| |
| flake8 checks failed." |
| exit 1 |
| fi |
| |
| echo "starting $FLAKE8_BUILD test..." |
| FLAKE8_REPORT=$( ($FLAKE8_BUILD --append-config dev/tox.ini --count --show-source --statistics .) 2>&1) |
| FLAKE8_STATUS=$? |
| |
| if [ "$FLAKE8_STATUS" -ne 0 ]; then |
| echo "flake8 checks failed:" |
| echo "$FLAKE8_REPORT" |
| echo "$FLAKE8_STATUS" |
| exit "$FLAKE8_STATUS" |
| else |
| echo "flake8 checks passed." |
| echo |
| fi |
| } |
| |
| function sphinx_test { |
| local SPHINX_REPORT= |
| local SPHINX_STATUS= |
| |
| # Check that the documentation builds acceptably, skip check if sphinx is not installed. |
| if ! hash "$SPHINX_BUILD" 2> /dev/null; then |
| echo "The $SPHINX_BUILD command was not found. Skipping Sphinx build for now." |
| echo |
| return |
| fi |
| |
| PYTHON_HAS_SPHINX=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("sphinx") is not None)') |
| if [[ "$PYTHON_HAS_SPHINX" == "False" ]]; then |
| echo "$PYTHON_EXECUTABLE does not have Sphinx installed. Skipping Sphinx build for now." |
| echo |
| return |
| fi |
| |
| # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. |
| # See also https://github.com/sphinx-doc/sphinx/issues/7551. |
| PYTHON_HAS_SPHINX_3_0=$("$PYTHON_EXECUTABLE" -c 'from distutils.version import LooseVersion; import sphinx; print(LooseVersion(sphinx.__version__) < LooseVersion("3.1.0"))') |
| if [[ "$PYTHON_HAS_SPHINX_3_0" == "False" ]]; then |
| echo "$PYTHON_EXECUTABLE has Sphinx 3.1+ installed but it requires lower than 3.1. Skipping Sphinx build for now." |
| echo |
| return |
| fi |
| |
| # TODO(SPARK-32391): Install pydata_sphinx_theme in Jenkins machines |
| PYTHON_HAS_THEME=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("pydata_sphinx_theme") is not None)') |
| if [[ "$PYTHON_HAS_THEME" == "False" ]]; then |
| echo "$PYTHON_EXECUTABLE does not have pydata_sphinx_theme installed. Skipping Sphinx build for now." |
| echo |
| return |
| fi |
| |
| # TODO(SPARK-32666): Install nbsphinx in Jenkins machines |
| PYTHON_HAS_NBSPHINX=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("nbsphinx") is not None)') |
| if [[ "$PYTHON_HAS_NBSPHINX" == "False" ]]; then |
| echo "$PYTHON_EXECUTABLE does not have nbsphinx installed. Skipping Sphinx build for now." |
| echo |
| return |
| fi |
| |
| # TODO(SPARK-32666): Install ipython in Jenkins machines |
| PYTHON_HAS_IPYTHON=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("IPython") is not None)') |
| if [[ "$PYTHON_HAS_IPYTHON" == "False" ]]; then |
| echo "$PYTHON_EXECUTABLE does not have ipython installed. Skipping Sphinx build for now." |
| echo |
| return |
| fi |
| |
| # TODO(SPARK-33242): Install numpydoc in Jenkins machines |
| PYTHON_HAS_NUMPYDOC=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("numpydoc") is not None)') |
| if [[ "$PYTHON_HAS_NUMPYDOC" == "False" ]]; then |
| echo "$PYTHON_EXECUTABLE does not have numpydoc installed. Skipping Sphinx build for now." |
| echo |
| return |
| fi |
| |
| echo "starting $SPHINX_BUILD tests..." |
| pushd python/docs &> /dev/null |
| make clean &> /dev/null |
| # Treat warnings as errors so we stop correctly |
| SPHINX_REPORT=$( (SPHINXOPTS="-a -W" make html) 2>&1) |
| SPHINX_STATUS=$? |
| |
| if [ "$SPHINX_STATUS" -ne 0 ]; then |
| echo "$SPHINX_BUILD checks failed:" |
| echo "$SPHINX_REPORT" |
| echo |
| echo "re-running make html to print full warning list:" |
| make clean &> /dev/null |
| SPHINX_REPORT=$( (SPHINXOPTS="-a" make html) 2>&1) |
| echo "$SPHINX_REPORT" |
| exit "$SPHINX_STATUS" |
| else |
| echo "$SPHINX_BUILD checks passed." |
| echo |
| fi |
| |
| popd &> /dev/null |
| } |
| |
| SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" |
| SPARK_ROOT_DIR="$(dirname "${SCRIPT_DIR}")" |
| |
| pushd "$SPARK_ROOT_DIR" &> /dev/null |
| |
| PYTHON_SOURCE="$(find . -name "*.py")" |
| |
| compile_python_test "$PYTHON_SOURCE" |
| pycodestyle_test "$PYTHON_SOURCE" |
| flake8_test |
| mypy_test |
| sphinx_test |
| |
| echo |
| echo "all lint-python tests passed!" |
| |
| popd &> /dev/null |