| #!/bin/bash |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # Do some error checking and generate junit symptoms after running a build. |
| |
| set -euo pipefail |
| trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR |
| |
| START_TIME="" |
| if [ "$#" -eq 1 ] |
| then |
| # START_TIME is an optional parameter which gives the start time of the test run. |
| START_TIME="$1" |
| fi |
| |
| if test -v CMAKE_BUILD_TYPE && [[ "${CMAKE_BUILD_TYPE}" =~ 'UBSAN' ]] \ |
| && [ "${UBSAN_FAIL}" = "error" ] \ |
| && { grep -rI ": runtime error: " "${IMPALA_HOME}/logs" 2>&1 | sort | uniq \ |
| | tee logs/ubsan.txt ; } |
| then |
| "${IMPALA_HOME}"/bin/generate_junitxml.py --step UBSAN \ |
| --stderr "${IMPALA_HOME}"/logs/ubsan.txt --error "Undefined C++ behavior" |
| fi |
| |
| rm -rf "${IMPALA_HOME}"/logs_system |
| mkdir -p "${IMPALA_HOME}"/logs_system |
| |
| # Check dmesg output for OOMs and generate a symptom if present. |
| DID_JOURNALCTL=false |
| if [ -n "${START_TIME}" ] |
| then |
| # Restrict the dmesg output by the start time of the test run. |
| if sudo journalctl --dmesg --since="${START_TIME}" > \ |
| "${IMPALA_HOME}"/logs_system/journalctl 2>/dev/null; then |
| DID_JOURNALCTL=true |
| fi |
| else |
| if sudo journalctl --dmesg > \ |
| "${IMPALA_HOME}"/logs_system/journalctl 2>/dev/null; then |
| DID_JOURNALCTL=true |
| fi |
| fi |
| |
| if [[ "${DID_JOURNALCTL}" == "true" ]]; then |
| if [[ $(grep "Out of memory" "${IMPALA_HOME}"/logs_system/journalctl) ]]; then |
| "${IMPALA_HOME}"/bin/generate_junitxml.py --phase finalize --step dmesg \ |
| --stdout "${IMPALA_HOME}"/logs_system/journalctl --error "Process was OOM killed." |
| fi |
| else |
| echo "Failed to run journalctl, not checking for OOMs" |
| fi |
| |
| # Check for any minidumps and symbolize and dump them. |
| LOGS_DIR="${IMPALA_HOME}"/logs |
| if [[ $(find $LOGS_DIR -path "*minidumps*" -name "*dmp") ]]; then |
| SYM_DIR=$(mktemp -d) |
| dump_breakpad_symbols.py -b $IMPALA_HOME/be/build/latest -d $SYM_DIR |
| for minidump in $(find $LOGS_DIR -path "*minidumps*" -name "*dmp"); do |
| $IMPALA_TOOLCHAIN_PACKAGES_HOME/breakpad-$IMPALA_BREAKPAD_VERSION/bin/minidump_stackwalk \ |
| ${minidump} $SYM_DIR > ${minidump}_dumped 2> ${minidump}_dumped.log |
| "${IMPALA_HOME}"/bin/generate_junitxml.py --phase finalize --step minidumps \ |
| --error "Minidump generated: $minidump" \ |
| --stderr "$(head -n 100 ${minidump}_dumped)" |
| done |
| rm -rf $SYM_DIR |
| fi |
| |
| # Do a second pass over the minidumps with the resolve_minidump.py script. |
| # This means that we now generate two JUnitXMLs per minidump. This should |
| # be temporary. |
| # TODO: Once we verify everything works, we can remove the first loop. |
| if [[ $(find $LOGS_DIR -path "*minidumps*" -name "*dmp") ]]; then |
| for minidump in $(find $LOGS_DIR -path "*minidumps*" -name "*dmp"); do |
| # Since this is experimental, use it inside an if so that any error code doesn't |
| # abort this script. |
| if ! "${IMPALA_HOME}"/bin/resolve_minidumps.py --minidump_file ${minidump} \ |
| --output_file ${minidump}_dumpedv2 ; then |
| echo "bin/resolve_minidumps.py failed!" |
| else |
| "${IMPALA_HOME}"/bin/generate_junitxml.py --phase finalize --step minidumpsv2 \ |
| --error "Minidump generated: $minidump" \ |
| --stderr "resolve_minidumps.py output:\n$(head -n 100 ${minidump}_dumpedv2)" |
| fi |
| done |
| fi |
| |
| function check_for_asan_error { |
| ERROR_LOG=${1} |
| if grep -q "ERROR: AddressSanitizer:" ${ERROR_LOG} ; then |
| # Extract out the ASAN message from the log file into a temp file. |
| tmp_asan_output=$(mktemp) |
| sed -n '/AddressSanitizer:/,/ABORTING/p' ${ERROR_LOG} > "${tmp_asan_output}" |
| # Make each ASAN issue use its own JUnitXML file by including the log filename |
| # in the step. |
| base=$(basename ${ERROR_LOG}) |
| "${IMPALA_HOME}"/bin/generate_junitxml.py --phase finalize \ |
| --step "asan_error_${base}" \ |
| --error "Address Sanitizer message detected in ${ERROR_LOG}" \ |
| --stderr "$(cat ${tmp_asan_output})" |
| rm "${tmp_asan_output}" |
| fi |
| } |
| |
| function check_for_tsan_error { |
| ERROR_LOG=${1} |
| if grep -q "WARNING: ThreadSanitizer:" ${ERROR_LOG} ; then |
| # Extract out the TSAN message from the log file into a temp file. |
| # Starts with WARNING: ThreadSanitizer and then ends with a line with several '=' |
| # characters (currently 18, we match 10). |
| tmp_tsan_output=$(mktemp) |
| sed -n '/ThreadSanitizer:/,/==========/p' ${ERROR_LOG} > "${tmp_tsan_output}" |
| # Make each TSAN issue use its own JUnitXML file by including the log filename |
| # in the step. |
| base=$(basename ${ERROR_LOG}) |
| "${IMPALA_HOME}"/bin/generate_junitxml.py --phase finalize \ |
| --step "tsan_error_${base}" \ |
| --error "Thread Sanitizer message detected in ${ERROR_LOG}" \ |
| --stderr "$(cat ${tmp_tsan_output})" |
| rm "${tmp_tsan_output}" |
| fi |
| } |
| |
| # Check for AddressSanitizer/ThreadSanitizer messages. ASAN/TSAN errors can show up |
| # in ERROR logs (particularly for impalad). Some backend tests generate ERROR logs. |
| for error_log in $(find $LOGS_DIR -name "*ERROR*"); do |
| check_for_asan_error ${error_log} |
| check_for_tsan_error ${error_log} |
| done |
| # Backend tests can also generate output in logs/be_tests/LastTest.log |
| if [[ -f ${LOGS_DIR}/be_tests/LastTest.log ]]; then |
| check_for_asan_error ${LOGS_DIR}/be_tests/LastTest.log |
| check_for_tsan_error ${LOGS_DIR}/be_tests/LastTest.log |
| fi |
| |
| # Check for DCHECK messages. DCHECKs translate into CHECKs, which log at FATAL level |
| # and start the message with "Check failed:". |
| # Some backend tests do death tests that are designed to trigger DCHECKs. Ignore |
| # the be_tests directory to avoid flagging these as errors. |
| for fatal_log in $(find $LOGS_DIR -name "*FATAL*" ! -path "*/be_tests/*"); do |
| if grep -q "Check failed:" "${fatal_log}"; then |
| # Generate JUnitXML with the entire FATAL log included. It should be small. |
| base=$(basename ${fatal_log}) |
| "${IMPALA_HOME}"/bin/generate_junitxml.py --phase finalize \ |
| --step "dcheck_${base}" \ |
| --error "DCHECK found in log file: ${fatal_log}" \ |
| --stderr "${fatal_log}" |
| fi |
| done |