build-support/run-test.sh - kudu - Git at Google

 #!/bin/bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
 #
 # Script which wraps running a test and redirects its output to a
 # test log directory.
 #
 # If KUDU_COMPRESS_TEST_OUTPUT is non-empty, then the logs will be
 # gzip-compressed while they are written.
 #
 # If KUDU_FLAKY_TEST_ATTEMPTS is set, and either the test being run matches one
 # of the lines in the file KUDU_FLAKY_TEST_LIST or KUDU_RETRY_ALL_FAILED_TESTS
 # is non-zero, then the test will be retried on failure up to the specified
 # number of times. This can be used in the gerrit workflow to prevent annoying
 # false -1s caused by tests that are known to be flaky in master.
 #
 # If KUDU_REPORT_TEST_RESULTS is non-zero, then tests are reported to the
 # central test server.
 #
 # If KUDU_MEASURE_TEST_CPU_CONSUMPTION is non-zero, then tests will be wrapped
 # in a 'perf stat' command which measures the CPU consumption on a periodic
 # basis, writing the output into '<test>.<shard>.perf.txt' logs in the test log
 # directory. This is handy in order to determine an appropriate value for the
 # 'PROCESSORS' property passed to 'ADD_KUDU_TEST' in CMakeLists.txt.

 # Path to the test executable or script to be run.
 # May be relative or absolute.
 TEST_PATH=$1

 # Absolute path to the root source directory. This script is expected to live within it.
 SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/.. ; pwd)

 # Absolute path to the root build directory. The test path is expected to be within it.
 BUILD_ROOT=$(cd $(dirname "$TEST_PATH")/.. ; pwd)

 TEST_LOGDIR=$BUILD_ROOT/test-logs
 mkdir -p $TEST_LOGDIR

 TEST_DEBUGDIR=$BUILD_ROOT/test-debug
 mkdir -p $TEST_DEBUGDIR

 TEST_DIRNAME=$(cd $(dirname $TEST_PATH); pwd)
 TEST_FILENAME=$(basename $TEST_PATH)
 ABS_TEST_PATH=$TEST_DIRNAME/$TEST_FILENAME
 shift
 # Remove path and extension (if any).

 # The "short" test name doesn't include the shard number.
 SHORT_TEST_NAME=$(echo $TEST_FILENAME | perl -pe 's/\..+?$//')

 # The full test name does include the shard number if the test is sharded.
 if [ ${GTEST_TOTAL_SHARDS:-0} -gt 1 ]; then
   TEST_NAME=${SHORT_TEST_NAME}.${GTEST_SHARD_INDEX:?}
 else
   TEST_NAME=${SHORT_TEST_NAME}
 fi

 # Determine whether the user has chosen to retry all failed tests, or whether
 # the test is a known flaky by comparing against the user-specified list.
 TEST_EXECUTION_ATTEMPTS=1
 if [ -n "$KUDU_RETRY_ALL_FAILED_TESTS" ]; then
   echo "Will retry all failed tests"
   TEST_IS_RETRYABLE=1
 elif [ -n "$KUDU_FLAKY_TEST_LIST" ]; then
   if [ -f "$KUDU_FLAKY_TEST_LIST" ] && grep -q --count --line-regexp "$SHORT_TEST_NAME" "$KUDU_FLAKY_TEST_LIST"; then
     TEST_IS_RETRYABLE=1
   else
     echo "Flaky test list file $KUDU_FLAKY_TEST_LIST missing"
   fi
 fi

 if [ -n "$TEST_IS_RETRYABLE" ]; then
   TEST_EXECUTION_ATTEMPTS=${KUDU_FLAKY_TEST_ATTEMPTS:-1}
   echo $TEST_NAME is a retryable test. Will attempt running it
   echo up to $TEST_EXECUTION_ATTEMPTS times.
 fi


 # We run each test in its own subdir to avoid core file related races.
 TEST_WORKDIR=$BUILD_ROOT/test-work/$TEST_NAME
 mkdir -p $TEST_WORKDIR
 pushd $TEST_WORKDIR >/dev/null || exit 1
 rm -f *

 set -o pipefail

 LOGFILE=$TEST_LOGDIR/$TEST_NAME.txt
 XMLFILE=$TEST_LOGDIR/$TEST_NAME.xml
 export GTEST_OUTPUT="xml:$XMLFILE" # Enable JUnit-compatible XML output.

 # Remove both the compressed and uncompressed output, so the developer
 # doesn't accidentally get confused and read output from a prior test
 # run.
 rm -f $LOGFILE $LOGFILE.gz

 if [ -n "$KUDU_COMPRESS_TEST_OUTPUT" ] && [ "$KUDU_COMPRESS_TEST_OUTPUT" -ne 0 ] ; then
   pipe_cmd=gzip
   grep_cmd=$(which zgrep)
   LOGFILE=${LOGFILE}.gz
 else
   pipe_cmd=cat
   grep_cmd=$(which grep)
 fi

 # Set a 15-minute timeout for tests run via 'make test'.
 # This keeps our jenkins builds from hanging in the case that there's
 # a deadlock or anything.
 #
 # NOTE: this should be kept in sync with the default value of ARG_TIMEOUT
 # in the definition of ADD_KUDU_TEST in the top-level CMakeLists.txt.
 KUDU_TEST_TIMEOUT=${KUDU_TEST_TIMEOUT:-900}

 # Allow for collecting core dumps.
 KUDU_TEST_ULIMIT_CORE=${KUDU_TEST_ULIMIT_CORE:-0}
 ulimit -c $KUDU_TEST_ULIMIT_CORE

 # Run the actual test.
 for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
   if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
     # If the test fails, the test output may or may not be left behind,
     # depending on whether the test cleaned up or exited immediately. Either
     # way we need to clean it up. We do this by comparing the data directory
     # contents before and after the test runs, and deleting anything new.
     #
     # The comm program requires that its two inputs be sorted.
     TEST_TMPDIR_BEFORE=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
   fi

   # gtest won't overwrite old junit test files, resulting in a build failure
   # even when retries are successful.
   rm -f $XMLFILE

   if [[ $OSTYPE =~ ^darwin ]]; then
     #
     # For builds on MacOS X 10.11, neither (g)addr2line nor atos translates
     # address into line number.  The atos utility is able to do that only
     # if load address (-l <addr>) or reference to currently running process
     # (-p <pid>) is given.  This is so even for binaries linked with
     # PIE disabled.
     #
     addr2line_filter=cat
   else
     addr2line_filter="$SOURCE_ROOT/build-support/stacktrace_addr2line.pl $ABS_TEST_PATH"
   fi
   echo "Running $TEST_NAME, redirecting output into $LOGFILE" \
        "(attempt ${ATTEMPT_NUMBER}/$TEST_EXECUTION_ATTEMPTS)"
   test_command=$ABS_TEST_PATH
   if [ -n "$KUDU_MEASURE_TEST_CPU_CONSUMPTION" ] && [ "$KUDU_MEASURE_TEST_CPU_CONSUMPTION" -ne 0 ] ; then
     perf_file=$TEST_LOGDIR/$TEST_NAME.perf.txt
     echo "Will wrap test in perf-stat, output to ${perf_file}..."
     test_command="perf stat -I1000 -e task-clock -o $perf_file $test_command"
   fi
   $test_command "$@" --test_timeout_after $KUDU_TEST_TIMEOUT 2>&1 \
     | $addr2line_filter \
     | $pipe_cmd > $LOGFILE
   STATUS=$?

   # An empty gzipped file is corrupt because even a logically empty gzipped file
   # should contain at least the gzip header. If we detect this, treat the test
   # log output as a plain text file.
   if [[ -f $LOGFILE && ! -s $LOGFILE && $LOGFILE =~ .gz$ ]]; then
     RENAMED_LOGFILE=${LOGFILE%%.gz}
     mv $LOGFILE $RENAMED_LOGFILE
     LOGFILE=$RENAMED_LOGFILE
   fi

   # TSAN doesn't always exit with a non-zero exit code due to a bug:
   # mutex errors don't get reported through the normal error reporting infrastructure.
   # So we make sure to detect this and exit 1.
   #
   # Additionally, certain types of failures won't show up in the standard JUnit
   # XML output from gtest. We assume that gtest knows better than us and our
   # regexes in most cases, but for certain errors we delete the resulting xml
   # file and let our own post-processing step regenerate it.
   export GREP=$(which egrep)
   if zgrep --silent "ThreadSanitizer|Leak check.*detected leaks" $LOGFILE ; then
     echo ThreadSanitizer or leak check failures in $LOGFILE
     STATUS=1
     rm -f $XMLFILE
   fi

   if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
     # Now delete any new test output.
     TEST_TMPDIR_AFTER=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
     DIFF=$(comm -13 <(echo "$TEST_TMPDIR_BEFORE") \
                     <(echo "$TEST_TMPDIR_AFTER"))
     for DIR in $DIFF; do
       # Multiple tests may be running concurrently. To avoid deleting the
       # wrong directories, constrain to only directories beginning with the
       # test name.
       #
       # This may delete old test directories belonging to this test, but
       # that's not typically a concern when rerunning flaky tests.
       if [[ $DIR =~ ^$TEST_TMPDIR/$TEST_NAME ]]; then
         echo Deleting leftover flaky test directory "$DIR"
         rm -Rf "$DIR"
       fi
     done
   fi

   if [ -n "$KUDU_REPORT_TEST_RESULTS" ] && [ "$KUDU_REPORT_TEST_RESULTS" -ne 0 ]; then
     echo Reporting results
     $SOURCE_ROOT/build-support/report-test.sh "$ABS_TEST_PATH" "$LOGFILE" "$STATUS"
   fi

   if [ "$STATUS" -eq "0" ]; then
     break
   elif [ "$ATTEMPT_NUMBER" -lt "$TEST_EXECUTION_ATTEMPTS" ]; then
     echo Test failed attempt number $ATTEMPT_NUMBER
     echo Will retry...
   fi
 done

 # If the tests failed, we have a LeakSanitizer report, and XML reporting is
 # configured, add a new test case result to the XML file for the leak report.
 # Otherwise Jenkins won't show us that the tests had LSAN errors.
 if [[ "$STATUS" -ne "0" ]] && $grep_cmd --silent "ERROR: LeakSanitizer: detected memory leaks" $LOGFILE ; then
     echo Test had memory leaks. Editing XML
     perl -p -i -e '
     if (m#</testsuite>#) {
       print "<testcase name=\"LeakSanitizer\" status=\"run\" classname=\"LSAN\">\n";
       print "  <failure message=\"LeakSanitizer failed\" type=\"\">\n";
       print "    See txt log file for details\n";
       print "  </failure>\n";
       print "</testcase>\n";
     }' $XMLFILE
 fi

 # Capture and compress core file and binary.
 COREFILES=$(ls | grep ^core)
 if [ -n "$COREFILES" ]; then
   echo Found core dump. Saving executable and core files.
   gzip < $ABS_TEST_PATH > "$TEST_DEBUGDIR/$TEST_NAME.gz" || exit $?
   for COREFILE in $COREFILES; do
     gzip < $COREFILE > "$TEST_DEBUGDIR/$TEST_NAME.$COREFILE.gz" || exit $?
   done
   # Pull in any .so files as well.
   for LIB in $(ldd $ABS_TEST_PATH | grep $BUILD_ROOT | awk '{print $3}'); do
     LIB_NAME=$(basename $LIB)
     gzip < $LIB > "$TEST_DEBUGDIR/$LIB_NAME.gz" || exit $?
   done
 fi

 popd >/dev/null
 rm -Rf $TEST_WORKDIR

 exit $STATUS
	#!/bin/bash
	#
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.
	#
	# Script which wraps running a test and redirects its output to a
	# test log directory.
	#
	# If KUDU_COMPRESS_TEST_OUTPUT is non-empty, then the logs will be
	# gzip-compressed while they are written.
	#
	# If KUDU_FLAKY_TEST_ATTEMPTS is set, and either the test being run matches one
	# of the lines in the file KUDU_FLAKY_TEST_LIST or KUDU_RETRY_ALL_FAILED_TESTS
	# is non-zero, then the test will be retried on failure up to the specified
	# number of times. This can be used in the gerrit workflow to prevent annoying
	# false -1s caused by tests that are known to be flaky in master.
	#
	# If KUDU_REPORT_TEST_RESULTS is non-zero, then tests are reported to the
	# central test server.
	#
	# If KUDU_MEASURE_TEST_CPU_CONSUMPTION is non-zero, then tests will be wrapped
	# in a 'perf stat' command which measures the CPU consumption on a periodic
	# basis, writing the output into '<test>.<shard>.perf.txt' logs in the test log
	# directory. This is handy in order to determine an appropriate value for the
	# 'PROCESSORS' property passed to 'ADD_KUDU_TEST' in CMakeLists.txt.

	# Path to the test executable or script to be run.
	# May be relative or absolute.
	TEST_PATH=$1

	# Absolute path to the root source directory. This script is expected to live within it.
	SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/.. ; pwd)

	# Absolute path to the root build directory. The test path is expected to be within it.
	BUILD_ROOT=$(cd $(dirname "$TEST_PATH")/.. ; pwd)

	TEST_LOGDIR=$BUILD_ROOT/test-logs
	mkdir -p $TEST_LOGDIR

	TEST_DEBUGDIR=$BUILD_ROOT/test-debug
	mkdir -p $TEST_DEBUGDIR

	TEST_DIRNAME=$(cd $(dirname $TEST_PATH); pwd)
	TEST_FILENAME=$(basename $TEST_PATH)
	ABS_TEST_PATH=$TEST_DIRNAME/$TEST_FILENAME
	shift
	# Remove path and extension (if any).

	# The "short" test name doesn't include the shard number.
	SHORT_TEST_NAME=$(echo $TEST_FILENAME \| perl -pe 's/\..+?$//')

	# The full test name does include the shard number if the test is sharded.
	if [ ${GTEST_TOTAL_SHARDS:-0} -gt 1 ]; then
	TEST_NAME=${SHORT_TEST_NAME}.${GTEST_SHARD_INDEX:?}
	else
	TEST_NAME=${SHORT_TEST_NAME}
	fi

	# Determine whether the user has chosen to retry all failed tests, or whether
	# the test is a known flaky by comparing against the user-specified list.
	TEST_EXECUTION_ATTEMPTS=1
	if [ -n "$KUDU_RETRY_ALL_FAILED_TESTS" ]; then
	echo "Will retry all failed tests"
	TEST_IS_RETRYABLE=1
	elif [ -n "$KUDU_FLAKY_TEST_LIST" ]; then
	if [ -f "$KUDU_FLAKY_TEST_LIST" ] && grep -q --count --line-regexp "$SHORT_TEST_NAME" "$KUDU_FLAKY_TEST_LIST"; then
	TEST_IS_RETRYABLE=1
	else
	echo "Flaky test list file $KUDU_FLAKY_TEST_LIST missing"
	fi
	fi

	if [ -n "$TEST_IS_RETRYABLE" ]; then
	TEST_EXECUTION_ATTEMPTS=${KUDU_FLAKY_TEST_ATTEMPTS:-1}
	echo $TEST_NAME is a retryable test. Will attempt running it
	echo up to $TEST_EXECUTION_ATTEMPTS times.
	fi


	# We run each test in its own subdir to avoid core file related races.
	TEST_WORKDIR=$BUILD_ROOT/test-work/$TEST_NAME
	mkdir -p $TEST_WORKDIR
	pushd $TEST_WORKDIR >/dev/null \|\| exit 1
	rm -f *

	set -o pipefail

	LOGFILE=$TEST_LOGDIR/$TEST_NAME.txt
	XMLFILE=$TEST_LOGDIR/$TEST_NAME.xml
	export GTEST_OUTPUT="xml:$XMLFILE" # Enable JUnit-compatible XML output.

	# Remove both the compressed and uncompressed output, so the developer
	# doesn't accidentally get confused and read output from a prior test
	# run.
	rm -f $LOGFILE $LOGFILE.gz

	if [ -n "$KUDU_COMPRESS_TEST_OUTPUT" ] && [ "$KUDU_COMPRESS_TEST_OUTPUT" -ne 0 ] ; then
	pipe_cmd=gzip
	grep_cmd=$(which zgrep)
	LOGFILE=${LOGFILE}.gz
	else
	pipe_cmd=cat
	grep_cmd=$(which grep)
	fi

	# Set a 15-minute timeout for tests run via 'make test'.
	# This keeps our jenkins builds from hanging in the case that there's
	# a deadlock or anything.
	#
	# NOTE: this should be kept in sync with the default value of ARG_TIMEOUT
	# in the definition of ADD_KUDU_TEST in the top-level CMakeLists.txt.
	KUDU_TEST_TIMEOUT=${KUDU_TEST_TIMEOUT:-900}

	# Allow for collecting core dumps.
	KUDU_TEST_ULIMIT_CORE=${KUDU_TEST_ULIMIT_CORE:-0}
	ulimit -c $KUDU_TEST_ULIMIT_CORE

	# Run the actual test.
	for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
	if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
	# If the test fails, the test output may or may not be left behind,
	# depending on whether the test cleaned up or exited immediately. Either
	# way we need to clean it up. We do this by comparing the data directory
	# contents before and after the test runs, and deleting anything new.
	#
	# The comm program requires that its two inputs be sorted.
	TEST_TMPDIR_BEFORE=$(find $TEST_TMPDIR -maxdepth 1 -type d \| sort)
	fi

	# gtest won't overwrite old junit test files, resulting in a build failure
	# even when retries are successful.
	rm -f $XMLFILE

	if [[ $OSTYPE =~ ^darwin ]]; then
	#
	# For builds on MacOS X 10.11, neither (g)addr2line nor atos translates
	# address into line number. The atos utility is able to do that only
	# if load address (-l <addr>) or reference to currently running process
	# (-p <pid>) is given. This is so even for binaries linked with
	# PIE disabled.
	#
	addr2line_filter=cat
	else
	addr2line_filter="$SOURCE_ROOT/build-support/stacktrace_addr2line.pl $ABS_TEST_PATH"
	fi
	echo "Running $TEST_NAME, redirecting output into $LOGFILE" \
	"(attempt ${ATTEMPT_NUMBER}/$TEST_EXECUTION_ATTEMPTS)"
	test_command=$ABS_TEST_PATH
	if [ -n "$KUDU_MEASURE_TEST_CPU_CONSUMPTION" ] && [ "$KUDU_MEASURE_TEST_CPU_CONSUMPTION" -ne 0 ] ; then
	perf_file=$TEST_LOGDIR/$TEST_NAME.perf.txt
	echo "Will wrap test in perf-stat, output to ${perf_file}..."
	test_command="perf stat -I1000 -e task-clock -o $perf_file $test_command"
	fi
	$test_command "$@" --test_timeout_after $KUDU_TEST_TIMEOUT 2>&1 \
	\| $addr2line_filter \
	\| $pipe_cmd > $LOGFILE
	STATUS=$?

	# An empty gzipped file is corrupt because even a logically empty gzipped file
	# should contain at least the gzip header. If we detect this, treat the test
	# log output as a plain text file.
	if [[ -f $LOGFILE && ! -s $LOGFILE && $LOGFILE =~ .gz$ ]]; then
	RENAMED_LOGFILE=${LOGFILE%%.gz}
	mv $LOGFILE $RENAMED_LOGFILE
	LOGFILE=$RENAMED_LOGFILE
	fi

	# TSAN doesn't always exit with a non-zero exit code due to a bug:
	# mutex errors don't get reported through the normal error reporting infrastructure.
	# So we make sure to detect this and exit 1.
	#
	# Additionally, certain types of failures won't show up in the standard JUnit
	# XML output from gtest. We assume that gtest knows better than us and our
	# regexes in most cases, but for certain errors we delete the resulting xml
	# file and let our own post-processing step regenerate it.
	export GREP=$(which egrep)
	if zgrep --silent "ThreadSanitizer\|Leak check.*detected leaks" $LOGFILE ; then
	echo ThreadSanitizer or leak check failures in $LOGFILE
	STATUS=1
	rm -f $XMLFILE
	fi

	if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
	# Now delete any new test output.
	TEST_TMPDIR_AFTER=$(find $TEST_TMPDIR -maxdepth 1 -type d \| sort)
	DIFF=$(comm -13 <(echo "$TEST_TMPDIR_BEFORE") \
	<(echo "$TEST_TMPDIR_AFTER"))
	for DIR in $DIFF; do
	# Multiple tests may be running concurrently. To avoid deleting the
	# wrong directories, constrain to only directories beginning with the
	# test name.
	#
	# This may delete old test directories belonging to this test, but
	# that's not typically a concern when rerunning flaky tests.
	if [[ $DIR =~ ^$TEST_TMPDIR/$TEST_NAME ]]; then
	echo Deleting leftover flaky test directory "$DIR"
	rm -Rf "$DIR"
	fi
	done
	fi

	if [ -n "$KUDU_REPORT_TEST_RESULTS" ] && [ "$KUDU_REPORT_TEST_RESULTS" -ne 0 ]; then
	echo Reporting results
	$SOURCE_ROOT/build-support/report-test.sh "$ABS_TEST_PATH" "$LOGFILE" "$STATUS"
	fi

	if [ "$STATUS" -eq "0" ]; then
	break
	elif [ "$ATTEMPT_NUMBER" -lt "$TEST_EXECUTION_ATTEMPTS" ]; then
	echo Test failed attempt number $ATTEMPT_NUMBER
	echo Will retry...
	fi
	done

	# If the tests failed, we have a LeakSanitizer report, and XML reporting is
	# configured, add a new test case result to the XML file for the leak report.
	# Otherwise Jenkins won't show us that the tests had LSAN errors.
	if [[ "$STATUS" -ne "0" ]] && $grep_cmd --silent "ERROR: LeakSanitizer: detected memory leaks" $LOGFILE ; then
	echo Test had memory leaks. Editing XML
	perl -p -i -e '
	if (m#</testsuite>#) {
	print "<testcase name=\"LeakSanitizer\" status=\"run\" classname=\"LSAN\">\n";
	print " <failure message=\"LeakSanitizer failed\" type=\"\">\n";
	print " See txt log file for details\n";
	print " </failure>\n";
	print "</testcase>\n";
	}' $XMLFILE
	fi

	# Capture and compress core file and binary.
	COREFILES=$(ls \| grep ^core)
	if [ -n "$COREFILES" ]; then
	echo Found core dump. Saving executable and core files.
	gzip < $ABS_TEST_PATH > "$TEST_DEBUGDIR/$TEST_NAME.gz" \|\| exit $?
	for COREFILE in $COREFILES; do
	gzip < $COREFILE > "$TEST_DEBUGDIR/$TEST_NAME.$COREFILE.gz" \|\| exit $?
	done
	# Pull in any .so files as well.
	for LIB in $(ldd $ABS_TEST_PATH \| grep $BUILD_ROOT \| awk '{print $3}'); do
	LIB_NAME=$(basename $LIB)
	gzip < $LIB > "$TEST_DEBUGDIR/$LIB_NAME.gz" \|\| exit $?
	done
	fi

	popd >/dev/null
	rm -Rf $TEST_WORKDIR

	exit $STATUS