IMPALA-7517. Fix hang in scanner threads when soft limit is exceeded
As described in the JIRA, when scanner threads see that the soft limit
has been exceeded, they try to shut down. In some particular
interleavings, this would cause all of the scanner threads to exit
without any of them marking the scan as completed.
This patch adds a new fault point to inject fake soft limit errors, and
adds this fault point to the scanner test. With the previous placement
of the soft limit check, this caused query hangs pretty reliably. With
the new placement of the memory limit check, it now passes.
Change-Id: I3dc1a2ec79c823575d7d40e7b52216dea5b0ddde
Reviewed-on: http://gerrit.cloudera.org:8080/11369
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Todd Lipcon <todd@apache.org>
diff --git a/be/src/exec/hdfs-scan-node.cc b/be/src/exec/hdfs-scan-node.cc
index 3755e50..e8adc56 100644
--- a/be/src/exec/hdfs-scan-node.cc
+++ b/be/src/exec/hdfs-scan-node.cc
@@ -417,11 +417,6 @@
break;
}
- // Stop extra threads if we're over a soft limit in order to free up memory.
- if (!first_thread && mem_tracker_->AnyLimitExceeded(MemLimit::SOFT)) {
- break;
- }
-
// Done with range and it completed successfully
if (progress_.done()) {
// All ranges are finished. Indicate we are done.
@@ -437,6 +432,15 @@
all_ranges_started_ = true;
break;
}
+
+ // Stop extra threads if we're over a soft limit in order to free up memory.
+ if (!first_thread &&
+ (mem_tracker_->AnyLimitExceeded(MemLimit::SOFT) ||
+ !DebugAction(runtime_state_->query_options(),
+ "HDFS_SCANNER_THREAD_CHECK_SOFT_MEM_LIMIT").ok())) {
+ VLOG_QUERY << "Soft memory limit exceeded. Extra scanner thread exiting.";
+ break;
+ }
}
{
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index c9ad888..b026475 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -64,6 +64,9 @@
'-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.5',
'-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0']
+# Trigger injected soft limit failures when scanner threads check memory limit.
+DEBUG_ACTION_DIMS.append('HDFS_SCANNER_THREAD_CHECK_SOFT_MEM_LIMIT:FAIL@0.5')
+
class TestScannersAllTableFormats(ImpalaTestSuite):
BATCH_SIZES = [0, 1, 16]
