IMPALA-6393: Add support for live_summary and live_progress in impalarc
This patch adds support for live_summary and live_progress in impalarc.
Testing:
1) Added unit-test cases in test_shell_commandline.py and
test_shell_interactive.py for live_summary and live_progress.
2) Successfully ran all other tests in test_shell_interactive.py and
test_shell_commandline.py
Change-Id: If4549b775a7966ad89d661d0349cc78754e13a86
Reviewed-on: http://gerrit.cloudera.org:8080/14927
Reviewed-by: Bikramjeet Vig <bikramjeet.vig@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index 913e40b..39fb3ad 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -141,7 +141,10 @@
tests/shell/bad_impalarc
tests/shell/good_impalarc
tests/shell/good_impalarc2
+tests/shell/good_impalarc3
+tests/shell/good_impalarc4
tests/shell/impalarc_with_error
+tests/shell/impalarc_with_error2
tests/shell/impalarc_with_query_options
tests/shell/impalarc_with_warnings
tests/shell/shell.cmds
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index 1e67415..1e0c6d6 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -151,8 +151,8 @@
QUIT_COMMANDS = ("quit", "exit")
VALID_SHELL_OPTIONS = {
- 'LIVE_PROGRESS' : (lambda x: x in ImpalaShell.TRUE_STRINGS, "print_progress"),
- 'LIVE_SUMMARY' : (lambda x: x in ImpalaShell.TRUE_STRINGS, "print_summary"),
+ 'LIVE_PROGRESS': (lambda x: x in ImpalaShell.TRUE_STRINGS, "live_progress"),
+ 'LIVE_SUMMARY': (lambda x: x in ImpalaShell.TRUE_STRINGS, "live_summary"),
'WRITE_DELIMITED' : (lambda x: x in ImpalaShell.TRUE_STRINGS, "write_delimited"),
'VERBOSE' : (lambda x: x in ImpalaShell.TRUE_STRINGS, "verbose"),
'DELIMITER' : (lambda x: " " if x == '\\s' else x, "output_delimiter"),
@@ -215,8 +215,8 @@
# Tracks query handle of the last query executed. Used by the 'profile' command.
self.last_query_handle = None;
- self.print_summary = options.print_summary
- self.print_progress = options.print_progress
+ self.live_summary = options.live_summary
+ self.live_progress = options.live_progress
self.ignore_query_failure = options.ignore_query_failure
@@ -1011,12 +1011,12 @@
query live progress for COMPUTE STATS query. Disable live
progress/summary callback for COMPUTE STATS query."""
query = self._build_query_string(self.last_leading_comment, self.orig_cmd, args)
- (prev_print_progress, prev_print_summary) = self.print_progress, self.print_summary
- (self.print_progress, self.print_summary) = False, False;
+ (prev_live_progress, prev_live_summary) = self.live_progress, self.live_summary
+ (self.live_progress, self.live_summary) = False, False
try:
ret = self._execute_stmt(query)
finally:
- (self.print_progress, self.print_summary) = prev_print_progress, prev_print_summary
+ (self.live_progress, self.live_summary) = prev_live_progress, prev_live_summary
return ret
def _format_outputstream(self):
@@ -1037,7 +1037,7 @@
execute the RPC to get the query exec summary and depending on the set options
print either the progress or the summary or both to stderr.
"""
- if not self.print_progress and not self.print_summary: return
+ if not self.live_progress and not self.live_summary: return
checkpoint = time.time()
if checkpoint - self.last_summary > self.PROGRESS_UPDATE_INTERVAL:
@@ -1060,13 +1060,13 @@
return
data = ""
- if self.print_progress and progress.total_scan_ranges > 0:
+ if self.live_progress and progress.total_scan_ranges > 0:
val = ((summary.progress.num_completed_scan_ranges * 100) /
summary.progress.total_scan_ranges)
fragment_text = "[%s%s] %s%%\n" % ("#" * val, " " * (100 - val), val)
data += fragment_text
- if self.print_summary:
+ if self.live_summary:
table = self._default_summary_table()
output = []
self.imp_client.build_summary_table(summary, 0, False, 0, False, output)
@@ -1835,7 +1835,7 @@
[(k.upper(), v) for k, v in parse_variables(options.query_options).items()])
if options.query or options.query_file:
- if options.print_progress or options.print_summary:
+ if options.live_progress or options.live_summary:
print_to_stderr("Error: Live reporting is available for interactive mode only.")
raise FatalShellException()
diff --git a/shell/impala_shell_config_defaults.py b/shell/impala_shell_config_defaults.py
index 2b649b7..98342db 100644
--- a/shell/impala_shell_config_defaults.py
+++ b/shell/impala_shell_config_defaults.py
@@ -39,8 +39,8 @@
'output_delimiter': '\\t',
'output_file': None,
'print_header': False,
- 'print_progress' : False,
- 'print_summary' : False,
+ 'live_progress': False,
+ 'live_summary': False,
'query': None,
'query_file': None,
'show_profiles': False,
diff --git a/shell/option_parser.py b/shell/option_parser.py
index 954bb76..583813c 100755
--- a/shell/option_parser.py
+++ b/shell/option_parser.py
@@ -230,9 +230,9 @@
parser.add_option("--history_file", dest="history_file",
help=("The file in which to store shell history. This may also be "
"configured using the IMPALA_HISTFILE environment variable."))
- parser.add_option("--live_summary", dest="print_summary", action="store_true",
+ parser.add_option("--live_summary", dest="live_summary", action="store_true",
help="Print a query summary every 1s while the query is running.")
- parser.add_option("--live_progress", dest="print_progress", action="store_true",
+ parser.add_option("--live_progress", dest="live_progress", action="store_true",
help="Print a query progress every 1s while the query is running.")
parser.add_option("--auth_creds_ok_in_clear", dest="creds_ok_in_clear",
action="store_true", help="If set, LDAP authentication " +
diff --git a/tests/shell/good_impalarc3 b/tests/shell/good_impalarc3
new file mode 100644
index 0000000..2e09876
--- /dev/null
+++ b/tests/shell/good_impalarc3
@@ -0,0 +1,10 @@
+[impala]
+keyval=msg3=test
+verbose=true
+Q=DEFAULT_FILE_FORMAT=avro
+Q=MEM_LIMIT=1GB
+live_progress=true
+live_summary=1
+
+[impala.query_options]
+DEFAULT_FILE_FORMAT=text
diff --git a/tests/shell/good_impalarc4 b/tests/shell/good_impalarc4
new file mode 100644
index 0000000..24056f9
--- /dev/null
+++ b/tests/shell/good_impalarc4
@@ -0,0 +1,9 @@
+[impala]
+keyval=msg3=test
+verbose=true
+Q=DEFAULT_FILE_FORMAT=avro
+live_progress=False
+live_summary=0
+
+[impala.query_options]
+DEFAULT_FILE_FORMAT=text
diff --git a/tests/shell/impalarc_with_error2 b/tests/shell/impalarc_with_error2
new file mode 100644
index 0000000..b828c1e
--- /dev/null
+++ b/tests/shell/impalarc_with_error2
@@ -0,0 +1,9 @@
+[impala]
+keyval=msg3=test
+verbose=true
+Q=DEFAULT_FILE_FORMAT=avro
+Live_Progress=true
+live_summary=maybe
+
+[impala.query_options]
+DEFAULT_FILE_FORMAT=text
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index 9505bb4..f9cb5fd 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -575,6 +575,26 @@
"'maybe' is not a valid value for a boolean option.")
assert err_msg in result.stderr
+ # Test the optional configuration file with live_progress and live_summary
+ # Positive test
+ args = ['--config_file=%s/good_impalarc3' % QUERY_FILE_PATH]
+ result = run_impala_shell_cmd(vector, args)
+ assert 'WARNING:' not in result.stderr, \
+ "A valid config file should not trigger any warning: {0}".format(result.stderr)
+ # Negative Tests
+ # specified config file with live_progress enabled for non interactive mode
+ args = ['--config_file=%s/good_impalarc3' % QUERY_FILE_PATH, '--query=select 3']
+ result = run_impala_shell_cmd(vector, args, expect_success=False)
+ assert 'Live reporting is available for interactive mode only' in result.stderr
+ # bad formatting of config file
+ args = ['--config_file={0}/impalarc_with_error2'.format(QUERY_FILE_PATH)]
+ result = run_impala_shell_cmd(vector, args, expect_success=False)
+ err_msg = "Ignoring unrecognized config option"
+ assert err_msg in result.stderr
+ err_msg = ("Unexpected value in configuration file. "
+ "'maybe' is not a valid value for a boolean option.")
+ assert err_msg in result.stderr
+
def test_execute_queries_from_stdin(self, vector):
"""Test that queries get executed correctly when STDIN is given as the sql file."""
args = ['-f', '-', '--quiet', '-B']
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index a68b2ef..46447e2 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -497,6 +497,33 @@
# Verify that query options under [impala] override those under [impala.query_options]
assert "\tDEFAULT_FILE_FORMAT: avro" in result.stdout
+ def test_live_option_configuration(self, vector):
+ """Test the optional configuration file with live_progress and live_summary."""
+ # Positive tests
+ # set live_summary and live_progress as True with config file
+ rcfile_path = os.path.join(QUERY_FILE_PATH, 'good_impalarc3')
+ args = ['--config_file=%s' % rcfile_path]
+ cmds = "set all;"
+ result = run_impala_shell_interactive(vector, cmds, shell_args=args)
+ assert 'WARNING:' not in result.stderr, \
+ "A valid config file should not trigger any warning: {0}".format(result.stderr)
+ assert "\tLIVE_SUMMARY: True" in result.stdout
+ assert "\tLIVE_PROGRESS: True" in result.stdout
+
+ # set live_summary and live_progress as False with config file
+ rcfile_path = os.path.join(QUERY_FILE_PATH, 'good_impalarc4')
+ args = ['--config_file=%s' % rcfile_path]
+ result = run_impala_shell_interactive(vector, cmds, shell_args=args)
+ assert 'WARNING:' not in result.stderr, \
+ "A valid config file should not trigger any warning: {0}".format(result.stderr)
+ assert "\tLIVE_SUMMARY: False" in result.stdout
+ assert "\tLIVE_PROGRESS: False" in result.stdout
+ # override options in config file through command line arguments
+ args = ['--live_progress', '--live_summary', '--config_file=%s' % rcfile_path]
+ result = run_impala_shell_interactive(vector, cmds, shell_args=args)
+ assert "\tLIVE_SUMMARY: True" in result.stdout
+ assert "\tLIVE_PROGRESS: True" in result.stdout
+
def test_source_file(self, vector):
cwd = os.getcwd()
try:
