dev/lint-python - spark - Git at Google

 #!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
 # define test binaries + versions
 FLAKE8_BUILD="flake8"
 MINIMUM_FLAKE8="3.9.0"
 MINIMUM_MYPY="1.8.0"
 MYPY_BUILD="mypy"
 PYTEST_BUILD="pytest"

 PYTHON_EXECUTABLE="${PYTHON_EXECUTABLE:-python3}"

 BLACK_BUILD="$PYTHON_EXECUTABLE -m black"

 function exit_with_usage {
   set +x
   echo "lint-python - linter for Python"
   echo ""
   echo "usage:"
   cl_options="[--compile] [--black] [--custom-pyspark-error] [--flake8] [--mypy] [--mypy-examples] [--mypy-data]"
   echo "lint-python $cl_options"
   echo ""
   exit 1
 }

 # Parse arguments
 while (( "$#" )); do
   case $1 in
     --compile)
       COMPILE_TEST=true
       ;;
     --black)
       BLACK_TEST=true
       ;;
     --custom-pyspark-error)
       PYSPARK_CUSTOM_ERRORS_CHECK_TEST=true
       ;;
     --flake8)
       FLAKE8_TEST=true
       ;;
     --mypy)
       MYPY_TEST=true
       ;;
     --mypy-examples)
       MYPY_EXAMPLES_TEST=true
       ;;
     --mypy-data)
       MYPY_DATA_TEST=true
       ;;
     *)
       echo "Error: $1 is not supported"
       exit_with_usage
       ;;
   esac
   shift
 done

 if [[ -z "$COMPILE_TEST$BLACK_TEST$PYSPARK_CUSTOM_ERRORS_CHECK_TEST$FLAKE8_TEST$MYPY_TEST$MYPY_EXAMPLES_TEST$MYPY_DATA_TEST" ]]; then
   COMPILE_TEST=true
   BLACK_TEST=true
   PYSPARK_CUSTOM_ERRORS_CHECK_TEST=true
   FLAKE8_TEST=true
   MYPY_TEST=true
   MYPY_EXAMPLES_TEST=true
   MYPY_DATA_TEST=true
 fi

 function satisfies_min_version {
     local provided_version="$1"
     local expected_version="$2"
     echo "$(
         "$PYTHON_EXECUTABLE" << EOM
 try:
     from setuptools.extern.packaging import version
 except ModuleNotFoundError:
     from packaging import version
 print(version.parse('$provided_version') >= version.parse('$expected_version'))
 EOM
     )"
 }

 function compile_python_test {
     local COMPILE_STATUS=
     local COMPILE_REPORT=

     if [[ ! "$1" ]]; then
         echo "No python files found!  Something is very wrong -- exiting."
         exit 1;
     fi

     # compileall: https://docs.python.org/3/library/compileall.html
     echo "starting python compilation test..."
     COMPILE_REPORT=$( ("$PYTHON_EXECUTABLE" -B -mcompileall -q -l -x "[/\\\\][.]git" $1) 2>&1)
     COMPILE_STATUS=$?

     if [ $COMPILE_STATUS -ne 0 ]; then
         echo "Python compilation failed with the following errors:"
         echo "$COMPILE_REPORT"
         echo "$COMPILE_STATUS"
         exit "$COMPILE_STATUS"
     else
         echo "python compilation succeeded."
         echo
     fi
 }


 function mypy_annotation_test {
     local MYPY_REPORT=
     local MYPY_STATUS=

     echo "starting mypy annotations test..."
     MYPY_REPORT=$( ($MYPY_BUILD \
       --python-executable $PYTHON_EXECUTABLE \
       --namespace-packages \
       --config-file python/mypy.ini \
       --cache-dir /tmp/.mypy_cache/ \
       python/pyspark) 2>&1)
     MYPY_STATUS=$?

     if [ "$MYPY_STATUS" -ne 0 ]; then
         echo "annotations failed mypy checks:"
         echo "$MYPY_REPORT"
         echo "$MYPY_STATUS"
         exit "$MYPY_STATUS"
     else
         echo "annotations passed mypy checks."
         echo
     fi
 }


 function mypy_data_test {
     local PYTEST_REPORT=
     local PYTEST_STATUS=

     echo "starting mypy data test..."

     $PYTHON_EXECUTABLE -c "import importlib.util; import sys; \
                sys.exit(0 if importlib.util.find_spec('pytest_mypy_plugins') else 1)"

     if [ $? -ne 0 ]; then
       echo "pytest-mypy-plugins missing. Skipping for now."
       return
     fi

     PYTEST_REPORT=$( (MYPYPATH=python $PYTEST_BUILD \
       -c dev/pyproject.toml \
       --rootdir python \
       --mypy-only-local-stub \
       --mypy-ini-file python/mypy.ini \
       python/pyspark ) 2>&1)

     PYTEST_STATUS=$?

     if [ "$PYTEST_STATUS" -ne 0 ]; then
         echo "annotations failed data checks:"
         echo "$PYTEST_REPORT"
         echo "$PYTEST_STATUS"
         exit "$PYTEST_STATUS"
     else
       echo "annotations passed data checks."
       echo
     fi
 }

 function mypy_examples_test {
     local MYPY_REPORT=
     local MYPY_STATUS=

     echo "starting mypy examples test..."

     MYPY_REPORT=$( (MYPYPATH=python $MYPY_BUILD \
       --python-executable $PYTHON_EXECUTABLE \
       --namespace-packages \
       --config-file python/mypy.ini \
       --exclude "mllib/*" \
       examples/src/main/python/) 2>&1)

     MYPY_STATUS=$?

     if [ "$MYPY_STATUS" -ne 0 ]; then
         echo "examples failed mypy checks:"
         echo "$MYPY_REPORT"
         echo "$MYPY_STATUS"
         exit "$MYPY_STATUS"
     else
       echo "examples passed mypy checks."
       echo
     fi
 }


 function mypy_test {
     if ! hash "$MYPY_BUILD" 2> /dev/null; then
         echo "The $MYPY_BUILD command was not found. Skipping for now."
         return
     fi

     _MYPY_VERSION=($($MYPY_BUILD --version))
     MYPY_VERSION="${_MYPY_VERSION[1]}"
     EXPECTED_MYPY="$(satisfies_min_version $MYPY_VERSION $MINIMUM_MYPY)"

     if [[ "$EXPECTED_MYPY" == "False" ]]; then
         echo "The minimum mypy version needs to be $MINIMUM_MYPY. Your current version is $MYPY_VERSION. Skipping for now."
         return
     fi

     if [[ "$MYPY_TEST" == "true" ]]; then
         mypy_annotation_test
     fi
     if [[ "$MYPY_EXAMPLES_TEST" == "true" ]]; then
         mypy_examples_test
     fi
     if [[ "$MYPY_DATA_TEST" == "true" ]]; then
         mypy_data_test
     fi
 }


 function flake8_test {
     local FLAKE8_VERSION=
     local EXPECTED_FLAKE8=
     local FLAKE8_REPORT=
     local FLAKE8_STATUS=

     if ! hash "$FLAKE8_BUILD" 2> /dev/null; then
         echo "The flake8 command was not found. Skipping for now."
         return
     fi

     _FLAKE8_VERSION=($($FLAKE8_BUILD --version))
     FLAKE8_VERSION="${_FLAKE8_VERSION[0]}"
     EXPECTED_FLAKE8="$(satisfies_min_version $FLAKE8_VERSION $MINIMUM_FLAKE8)"

     if [[ "$EXPECTED_FLAKE8" == "False" ]]; then
         echo "\
 The minimum flake8 version needs to be $MINIMUM_FLAKE8. Your current version is $FLAKE8_VERSION

 flake8 checks failed."
         exit 1
     fi

     echo "starting $FLAKE8_BUILD test..."
     FLAKE8_REPORT=$( ($FLAKE8_BUILD --append-config dev/tox.ini --count --show-source --statistics .) 2>&1)
     FLAKE8_STATUS=$?

     if [ "$FLAKE8_STATUS" -ne 0 ]; then
         echo "flake8 checks failed:"
         echo "$FLAKE8_REPORT"
         echo "$FLAKE8_STATUS"
         exit "$FLAKE8_STATUS"
     else
         echo "flake8 checks passed."
         echo
     fi
 }

 function black_test {
     local BLACK_REPORT=
     local BLACK_STATUS=

     # Skip check if black is not installed.
     $PYTHON_EXECUTABLE -c 'import black' &> /dev/null
     if [ $? -ne 0 ]; then
         echo "The Python library providing 'black' module was not found. Skipping black checks for now."
         echo
         return
     fi

     echo "starting black test..."
     BLACK_REPORT=$( ($BLACK_BUILD  --config dev/pyproject.toml --check python/pyspark dev python/packaging) 2>&1)
     BLACK_STATUS=$?

     if [ "$BLACK_STATUS" -ne 0 ]; then
         echo "black checks failed:"
         echo "$BLACK_REPORT"
         echo "Please run 'dev/reformat-python' script."
         echo "$BLACK_STATUS"
         exit "$BLACK_STATUS"
     else
         echo "black checks passed."
         echo
     fi
 }

 function pyspark_custom_errors_check_test {
     local PYSPARK_CUSTOM_ERRORS_CHECK_REPORT=
     local PYSPARK_CUSTOM_ERRORS_CHECK_STATUS=

     echo "starting PySpark custom errors check..."
     PYSPARK_CUSTOM_ERRORS_CHECK_REPORT=$( "$PYTHON_EXECUTABLE" dev/check_pyspark_custom_errors.py 2>&1 )
     PYSPARK_CUSTOM_ERRORS_CHECK_STATUS=$?

     if [ "$PYSPARK_CUSTOM_ERRORS_CHECK_STATUS" -ne 0 ]; then
         echo "PySpark custom errors check failed!"
         echo "$PYSPARK_CUSTOM_ERRORS_CHECK_REPORT"
         exit "$PYSPARK_CUSTOM_ERRORS_CHECK_STATUS"
     else
         echo "PySpark custom errors check passed."
         echo
     fi
 }

 SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 SPARK_ROOT_DIR="$(dirname "${SCRIPT_DIR}")"

 pushd "$SPARK_ROOT_DIR" &> /dev/null

 PYTHON_SOURCE="$(git ls-files '*.py')"

 if [[ "$COMPILE_TEST" == "true" ]]; then
     compile_python_test "$PYTHON_SOURCE"
 fi
 if [[ "$BLACK_TEST" == "true" ]]; then
     black_test
 fi
 if [[ "$PYSPARK_CUSTOM_ERRORS_CHECK_TEST" == "true" ]]; then
     pyspark_custom_errors_check_test
 fi
 if [[ "$FLAKE8_TEST" == "true" ]]; then
     flake8_test
 fi
 if [[ "$MYPY_TEST" == "true" ]] || [[ "$MYPY_EXAMPLES_TEST" == "true" ]] || [[ "$MYPY_DATA_TEST" == "true" ]]; then
     mypy_test
 fi

 echo
 echo "all lint-python tests passed!"

 popd &> /dev/null
	#!/usr/bin/env bash
	#
	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright ownership.
	# The ASF licenses this file to You under the Apache License, Version 2.0
	# (the "License"); you may not use this file except in compliance with
	# the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#
	# define test binaries + versions
	FLAKE8_BUILD="flake8"
	MINIMUM_FLAKE8="3.9.0"
	MINIMUM_MYPY="1.8.0"
	MYPY_BUILD="mypy"
	PYTEST_BUILD="pytest"

	PYTHON_EXECUTABLE="${PYTHON_EXECUTABLE:-python3}"

	BLACK_BUILD="$PYTHON_EXECUTABLE -m black"

	function exit_with_usage {
	set +x
	echo "lint-python - linter for Python"
	echo ""
	echo "usage:"
	cl_options="[--compile] [--black] [--custom-pyspark-error] [--flake8] [--mypy] [--mypy-examples] [--mypy-data]"
	echo "lint-python $cl_options"
	echo ""
	exit 1
	}

	# Parse arguments
	while (( "$#" )); do
	case $1 in
	--compile)
	COMPILE_TEST=true
	;;
	--black)
	BLACK_TEST=true
	;;
	--custom-pyspark-error)
	PYSPARK_CUSTOM_ERRORS_CHECK_TEST=true
	;;
	--flake8)
	FLAKE8_TEST=true
	;;
	--mypy)
	MYPY_TEST=true
	;;
	--mypy-examples)
	MYPY_EXAMPLES_TEST=true
	;;
	--mypy-data)
	MYPY_DATA_TEST=true
	;;
	*)
	echo "Error: $1 is not supported"
	exit_with_usage
	;;
	esac
	shift
	done

	if [[ -z "$COMPILE_TEST$BLACK_TEST$PYSPARK_CUSTOM_ERRORS_CHECK_TEST$FLAKE8_TEST$MYPY_TEST$MYPY_EXAMPLES_TEST$MYPY_DATA_TEST" ]]; then
	COMPILE_TEST=true
	BLACK_TEST=true
	PYSPARK_CUSTOM_ERRORS_CHECK_TEST=true
	FLAKE8_TEST=true
	MYPY_TEST=true
	MYPY_EXAMPLES_TEST=true
	MYPY_DATA_TEST=true
	fi

	function satisfies_min_version {
	local provided_version="$1"
	local expected_version="$2"
	echo "$(
	"$PYTHON_EXECUTABLE" << EOM
	try:
	from setuptools.extern.packaging import version
	except ModuleNotFoundError:
	from packaging import version
	print(version.parse('$provided_version') >= version.parse('$expected_version'))
	EOM
	)"
	}

	function compile_python_test {
	local COMPILE_STATUS=
	local COMPILE_REPORT=

	if [[ ! "$1" ]]; then
	echo "No python files found! Something is very wrong -- exiting."
	exit 1;
	fi

	# compileall: https://docs.python.org/3/library/compileall.html
	echo "starting python compilation test..."
	COMPILE_REPORT=$( ("$PYTHON_EXECUTABLE" -B -mcompileall -q -l -x "[/\\\\][.]git" $1) 2>&1)
	COMPILE_STATUS=$?

	if [ $COMPILE_STATUS -ne 0 ]; then
	echo "Python compilation failed with the following errors:"
	echo "$COMPILE_REPORT"
	echo "$COMPILE_STATUS"
	exit "$COMPILE_STATUS"
	else
	echo "python compilation succeeded."
	echo
	fi
	}


	function mypy_annotation_test {
	local MYPY_REPORT=
	local MYPY_STATUS=

	echo "starting mypy annotations test..."
	MYPY_REPORT=$( ($MYPY_BUILD \
	--python-executable $PYTHON_EXECUTABLE \
	--namespace-packages \
	--config-file python/mypy.ini \
	--cache-dir /tmp/.mypy_cache/ \
	python/pyspark) 2>&1)
	MYPY_STATUS=$?

	if [ "$MYPY_STATUS" -ne 0 ]; then
	echo "annotations failed mypy checks:"
	echo "$MYPY_REPORT"
	echo "$MYPY_STATUS"
	exit "$MYPY_STATUS"
	else
	echo "annotations passed mypy checks."
	echo
	fi
	}


	function mypy_data_test {
	local PYTEST_REPORT=
	local PYTEST_STATUS=

	echo "starting mypy data test..."

	$PYTHON_EXECUTABLE -c "import importlib.util; import sys; \
	sys.exit(0 if importlib.util.find_spec('pytest_mypy_plugins') else 1)"

	if [ $? -ne 0 ]; then
	echo "pytest-mypy-plugins missing. Skipping for now."
	return
	fi

	PYTEST_REPORT=$( (MYPYPATH=python $PYTEST_BUILD \
	-c dev/pyproject.toml \
	--rootdir python \
	--mypy-only-local-stub \
	--mypy-ini-file python/mypy.ini \
	python/pyspark ) 2>&1)

	PYTEST_STATUS=$?

	if [ "$PYTEST_STATUS" -ne 0 ]; then
	echo "annotations failed data checks:"
	echo "$PYTEST_REPORT"
	echo "$PYTEST_STATUS"
	exit "$PYTEST_STATUS"
	else
	echo "annotations passed data checks."
	echo
	fi
	}

	function mypy_examples_test {
	local MYPY_REPORT=
	local MYPY_STATUS=

	echo "starting mypy examples test..."

	MYPY_REPORT=$( (MYPYPATH=python $MYPY_BUILD \
	--python-executable $PYTHON_EXECUTABLE \
	--namespace-packages \
	--config-file python/mypy.ini \
	--exclude "mllib/*" \
	examples/src/main/python/) 2>&1)

	MYPY_STATUS=$?

	if [ "$MYPY_STATUS" -ne 0 ]; then
	echo "examples failed mypy checks:"
	echo "$MYPY_REPORT"
	echo "$MYPY_STATUS"
	exit "$MYPY_STATUS"
	else
	echo "examples passed mypy checks."
	echo
	fi
	}


	function mypy_test {
	if ! hash "$MYPY_BUILD" 2> /dev/null; then
	echo "The $MYPY_BUILD command was not found. Skipping for now."
	return
	fi

	_MYPY_VERSION=($($MYPY_BUILD --version))
	MYPY_VERSION="${_MYPY_VERSION[1]}"
	EXPECTED_MYPY="$(satisfies_min_version $MYPY_VERSION $MINIMUM_MYPY)"

	if [[ "$EXPECTED_MYPY" == "False" ]]; then
	echo "The minimum mypy version needs to be $MINIMUM_MYPY. Your current version is $MYPY_VERSION. Skipping for now."
	return
	fi

	if [[ "$MYPY_TEST" == "true" ]]; then
	mypy_annotation_test
	fi
	if [[ "$MYPY_EXAMPLES_TEST" == "true" ]]; then
	mypy_examples_test
	fi
	if [[ "$MYPY_DATA_TEST" == "true" ]]; then
	mypy_data_test
	fi
	}


	function flake8_test {
	local FLAKE8_VERSION=
	local EXPECTED_FLAKE8=
	local FLAKE8_REPORT=
	local FLAKE8_STATUS=

	if ! hash "$FLAKE8_BUILD" 2> /dev/null; then
	echo "The flake8 command was not found. Skipping for now."
	return
	fi

	_FLAKE8_VERSION=($($FLAKE8_BUILD --version))
	FLAKE8_VERSION="${_FLAKE8_VERSION[0]}"
	EXPECTED_FLAKE8="$(satisfies_min_version $FLAKE8_VERSION $MINIMUM_FLAKE8)"

	if [[ "$EXPECTED_FLAKE8" == "False" ]]; then
	echo "\
	The minimum flake8 version needs to be $MINIMUM_FLAKE8. Your current version is $FLAKE8_VERSION

	flake8 checks failed."
	exit 1
	fi

	echo "starting $FLAKE8_BUILD test..."
	FLAKE8_REPORT=$( ($FLAKE8_BUILD --append-config dev/tox.ini --count --show-source --statistics .) 2>&1)
	FLAKE8_STATUS=$?

	if [ "$FLAKE8_STATUS" -ne 0 ]; then
	echo "flake8 checks failed:"
	echo "$FLAKE8_REPORT"
	echo "$FLAKE8_STATUS"
	exit "$FLAKE8_STATUS"
	else
	echo "flake8 checks passed."
	echo
	fi
	}

	function black_test {
	local BLACK_REPORT=
	local BLACK_STATUS=

	# Skip check if black is not installed.
	$PYTHON_EXECUTABLE -c 'import black' &> /dev/null
	if [ $? -ne 0 ]; then
	echo "The Python library providing 'black' module was not found. Skipping black checks for now."
	echo
	return
	fi

	echo "starting black test..."
	BLACK_REPORT=$( ($BLACK_BUILD --config dev/pyproject.toml --check python/pyspark dev python/packaging) 2>&1)
	BLACK_STATUS=$?

	if [ "$BLACK_STATUS" -ne 0 ]; then
	echo "black checks failed:"
	echo "$BLACK_REPORT"
	echo "Please run 'dev/reformat-python' script."
	echo "$BLACK_STATUS"
	exit "$BLACK_STATUS"
	else
	echo "black checks passed."
	echo
	fi
	}

	function pyspark_custom_errors_check_test {
	local PYSPARK_CUSTOM_ERRORS_CHECK_REPORT=
	local PYSPARK_CUSTOM_ERRORS_CHECK_STATUS=

	echo "starting PySpark custom errors check..."
	PYSPARK_CUSTOM_ERRORS_CHECK_REPORT=$( "$PYTHON_EXECUTABLE" dev/check_pyspark_custom_errors.py 2>&1 )
	PYSPARK_CUSTOM_ERRORS_CHECK_STATUS=$?

	if [ "$PYSPARK_CUSTOM_ERRORS_CHECK_STATUS" -ne 0 ]; then
	echo "PySpark custom errors check failed!"
	echo "$PYSPARK_CUSTOM_ERRORS_CHECK_REPORT"
	exit "$PYSPARK_CUSTOM_ERRORS_CHECK_STATUS"
	else
	echo "PySpark custom errors check passed."
	echo
	fi
	}

	SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
	SPARK_ROOT_DIR="$(dirname "${SCRIPT_DIR}")"

	pushd "$SPARK_ROOT_DIR" &> /dev/null

	PYTHON_SOURCE="$(git ls-files '*.py')"

	if [[ "$COMPILE_TEST" == "true" ]]; then
	compile_python_test "$PYTHON_SOURCE"
	fi
	if [[ "$BLACK_TEST" == "true" ]]; then
	black_test
	fi
	if [[ "$PYSPARK_CUSTOM_ERRORS_CHECK_TEST" == "true" ]]; then
	pyspark_custom_errors_check_test
	fi
	if [[ "$FLAKE8_TEST" == "true" ]]; then
	flake8_test
	fi
	if [[ "$MYPY_TEST" == "true" ]] \|\| [[ "$MYPY_EXAMPLES_TEST" == "true" ]] \|\| [[ "$MYPY_DATA_TEST" == "true" ]]; then
	mypy_test
	fi

	echo
	echo "all lint-python tests passed!"

	popd &> /dev/null