bin/check-pylint-py3k.sh

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

set -euo pipefail

BINDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# To allow incrementally banning individual pylint checks, this uses grep
# expressions to match banned pylint warnings. The grep expressions are stored
# in the bin/banned_py3k_warnings.txt file.
BANNED_PY3K_WARNINGS="${BINDIR}/banned_py3k_warnings.txt"

function print_usage {
    echo "check-pylink-py3k.sh : Checks eligible python files for pylint py3k compliance."
    echo "Fails if the python files have py3k warnings that match the patterns in "
    echo "bin/banned_py3k_warnings.txt."
    echo "[--error_output_file] : (optional) Also output the errors to a file"
    echo "[--warning_output_file] : (optional) Also output the warnings to a file"
}

ERROR_OUTPUT_FILE=""
WARNING_OUTPUT_FILE=""
while [ -n "$*" ]
do
    case "$1" in
        --error_output_file)
            ERROR_OUTPUT_FILE="${2-}"
            shift;
            ;;
        --warning_output_file)
            WARNING_OUTPUT_FILE="${2-}"
            shift;
            ;;
        --help|*)
            print_usage
            exit 1
            ;;
    esac
    shift
done

pushd ${IMPALA_HOME} > /dev/null 2>&1

OUTPUT_TMP_DIR=$(mktemp -d)
PYLINT_OUTPUT_FILE="${OUTPUT_TMP_DIR}/pylint_output.txt"
ERROR_OUTPUT_TMP_FILE="${OUTPUT_TMP_DIR}/error_output_tmp.txt"
WARNING_OUTPUT_TMP_FILE="${OUTPUT_TMP_DIR}/warning_output_tmp.txt"

RETCODE=0
for file in $(git ls-files '**/*.py'); do
    # Skip the shell entirely (but cover tests/shell)
    if [[ "${file}" =~ "shell/" && ! "${file}" =~ "tests/shell" ]]; then
        continue
    fi
    # Ignore files that are created to run with python3.
    FIRST_LINE=$(head -n1 ${file})
    if [[ "${file}: ${FIRST_LINE}" =~ "#!" ]]; then
        if [[ "${FIRST_LINE}" =~ "python3" ]]; then
            >&2 echo "SKIPPING: ${file} is already using python3: ${FIRST_LINE}"
            continue
        fi
        if [[ "${FIRST_LINE}" =~ "/bin/bash" ]]; then
            >&2 echo "SKIPPING: ${file} is a weird bash/python hybrid: ${FIRST_LINE}"
            continue
        fi
    fi

    >&2 echo "PROCESSING: ${file}"

    # -s n (skip score for each file)
    # --exit-zero: don't fail
    impala-pylint -s n --exit-zero --py3k ${file} >> ${PYLINT_OUTPUT_FILE}
done

touch "${ERROR_OUTPUT_TMP_FILE}"
touch "${WARNING_OUTPUT_TMP_FILE}"

# Hitting a banned py3k warning will cause this to return an error
echo ""
echo ""
if grep -f "${BANNED_PY3K_WARNINGS}" "${PYLINT_OUTPUT_FILE}" > /dev/null 2>&1 ; then
    echo "ERROR: Some python files contain these banned pylint warnings:" | \
        tee "${ERROR_OUTPUT_TMP_FILE}"
    grep -f "${BANNED_PY3K_WARNINGS}" "${PYLINT_OUTPUT_FILE}" | \
        tee -a "${ERROR_OUTPUT_TMP_FILE}"
    RETCODE=1
else
    echo "No errors found" | tee "${ERROR_OUTPUT_TMP_FILE}"
fi

if [[ -n "${ERROR_OUTPUT_FILE}" ]]; then
    cp "${ERROR_OUTPUT_TMP_FILE}" "${ERROR_OUTPUT_FILE}"
fi

# The remaining py3k warnings are interesting, but they are not yet enforced.
# Pylint produces annoying lines like "************* Module X", so try to filter those out
echo ""
echo ""
if grep -v -e '\*\*\*\*' -f "${BANNED_PY3K_WARNINGS}" \
        "${PYLINT_OUTPUT_FILE}" > /dev/null 2>&1 ; then
    echo "WARNING: Some python files contain these unenforced pylint warnings:" | \
        tee "${WARNING_OUTPUT_TMP_FILE}"
    grep -v -e '\*\*\*\*' -f "${BANNED_PY3K_WARNINGS}" "${PYLINT_OUTPUT_FILE}" | \
        tee -a "${WARNING_OUTPUT_TMP_FILE}"

    echo "WARNING SUMMARY table:"
    cat "${WARNING_OUTPUT_TMP_FILE}" | grep -v "WARNING" | cut -d: -f4- | \
        sed 's#^ ##' | sort | uniq -c
else
    echo "No warnings found" | tee "${WARNING_OUTPUT_TMP_FILE}"
fi

if [[ -n "${WARNING_OUTPUT_FILE}" ]]; then
    cp "${WARNING_OUTPUT_TMP_FILE}" "${WARNING_OUTPUT_FILE}"
fi

rm -rf "${OUTPUT_TMP_DIR}"

popd > /dev/null 2>&1

exit ${RETCODE}