IMPALA-9196: Dump jstack and collect logs when tests timeout
This patch augments script-timeout-check.sh to also dump the jstack of
FE when tests timeout.
Tests:
- Manually test the script with sudo privilege
- Tested the script in private Jenkins jobs
Change-Id: Ib8a5b140024c236209c7e44149660189890b9d06
Reviewed-on: http://gerrit.cloudera.org:8080/14794
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
diff --git a/bin/script-timeout-check.sh b/bin/script-timeout-check.sh
index ad5fd04..4aec3af 100755
--- a/bin/script-timeout-check.sh
+++ b/bin/script-timeout-check.sh
@@ -32,6 +32,15 @@
# pkill -P $TIMEOUT_PID || true
# kill $TIMEOUT_PID
+function collect_stacktraces() {
+ name=$1
+ pid=$2
+ echo "**** Generating stacktrace of $name with process id: $pid ****"
+ gdb -ex "thread apply all bt" --batch -p $pid > \
+ "${IMPALA_TIMEOUT_LOGS_DIR}/${name}_${pid}.txt"
+ $JAVA_HOME/bin/jstack -F $pid > "${IMPALA_TIMEOUT_LOGS_DIR}/${name}_${pid}_jstack.txt"
+}
+
SCRIPT_NAME=""
SLEEP_TIMEOUT_MIN=""
@@ -96,10 +105,16 @@
# Impala might have a thread stuck. Print the stacktrace to the console output.
mkdir -p "$IMPALA_TIMEOUT_LOGS_DIR"
for pid in $(pgrep impalad); do
- echo "**** Generating stacktrace of impalad with process id: $pid ****"
- gdb -ex "thread apply all bt" --batch -p $pid > "${IMPALA_TIMEOUT_LOGS_DIR}/${pid}.txt"
+ collect_stacktraces impalad $pid
done
+# Catalogd's process name may change. Use 'ps' directly to search the binary name.
+CATALOGD_PID=$(ps aux | grep [c]atalogd | awk '{print $2}')
+collect_stacktraces catalogd $CATALOGD_PID
+
+STATESTORED_PID=$(pgrep statestored)
+collect_stacktraces statestored $STATESTORED_PID
+
# Now kill the caller
kill $PPID
@@ -108,4 +123,3 @@
thread which can be confirmed by looking at the stacktrace of running impalad
processes at ${IMPALA_TIMEOUT_LOGS_DIR}"
-
