IMPALA-9351: Fix tests depending on hard-coded file paths of managed tables
Some tests (e.g. AnalyzeDDLTest.TestCreateTableLikeFileOrc) depend on
hard-coded file paths of managed tables, assuming that there is always a
file named 'base_0000001/bucket_00000_0' under the table dir. However,
the file name is in the form of bucket_${bucket-id}_${attempt-id}. The
last part of the file name is not guaranteed to be 0. If the first
attempt fails and the second attempt succeeds, the file name will be
bucket_00000_1.
This patch replaces these hard-coded file paths to corresponding files
that are uploaded to HDFS by commands. For tests that do need to use the
file paths of managed table files, we do a listing on the table dir to
get the file names, instead of hard-coding the file paths.
Updated chars-formats.orc to contain column names in the file so can be
used in more tests. The original one only has names like col0, col1,
col2.
Tests:
- Run CORE tests
Change-Id: Ie3136ee90e2444c4a12f0f2e1470fca1d5deaba0
Reviewed-on: http://gerrit.cloudera.org:8080/16441
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index 964162b..47ffe59 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -2022,10 +2022,9 @@
// Inferring primitive and complex types
AnalyzesOk("create table if not exists newtbl_DNE like orc " +
- "'/test-warehouse/managed/alltypestiny_orc_def/year=2009/month=1/" +
- "base_0000001/bucket_00000_0'");
+ "'/test-warehouse/schemas/alltypes_non_acid.orc'");
AnalyzesOk("create table if not exists newtbl_DNE like orc " +
- "'/test-warehouse/managed/complextypestbl_orc_def/base_0000001/bucket_00000_0'");
+ "'/test-warehouse/complextypestbl_non_transactional_orc_def/nullable.orc'");
// check invalid paths
AnalysisError("create table if not exists functional.zipcode_incomes like ORC " +
diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh
index 5d2e0e3..312bd3a 100755
--- a/testdata/bin/create-load-data.sh
+++ b/testdata/bin/create-load-data.sh
@@ -213,6 +213,9 @@
ln -s ${IMPALA_HOME}/testdata/data/chars-formats.orc ${TMP_DIR}/chars_formats_orc_def
ln -s ${IMPALA_HOME}/testdata/data/chars-formats.txt ${TMP_DIR}/chars_formats_text
+ # File used by CreateTableLikeOrc tests
+ ln -s ${IMPALA_HOME}/testdata/data/alltypes_non_acid.orc ${SCHEMA_TMP_DIR}
+
hadoop fs -put -f ${TMP_DIR}/* /test-warehouse
rm -r ${TMP_DIR}
diff --git a/testdata/data/chars-formats.orc b/testdata/data/chars-formats.orc
index 625c2c8..7703435 100644
--- a/testdata/data/chars-formats.orc
+++ b/testdata/data/chars-formats.orc
Binary files differ
diff --git a/testdata/workloads/functional-query/queries/QueryTest/create-table-like-file-orc.test b/testdata/workloads/functional-query/queries/QueryTest/create-table-like-file-orc.test
index 3d36299..85d3f1a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/create-table-like-file-orc.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/create-table-like-file-orc.test
@@ -23,19 +23,16 @@
====
---- QUERY
create table $DATABASE.temp_chars_table like ORC
-'$NAMENODE/$MANAGED_WAREHOUSE_DIR/chars_tiny_orc_def/base_0000001/bucket_00000_0'
+'$NAMENODE/test-warehouse/chars_formats_orc_def/chars-formats.orc'
---- RESULTS
'Table has been created.'
====
---- QUERY
describe $DATABASE.temp_chars_table
---- RESULTS
-'operation','int','Inferred from ORC file.'
-'originaltransaction','bigint','Inferred from ORC file.'
-'rowid','bigint','Inferred from ORC file.'
-'bucket','int','Inferred from ORC file.'
-'currenttransaction','bigint','Inferred from ORC file.'
-'row','struct<\n cs:char(5),\n cl:char(140),\n vc:varchar(32)\n>','Inferred from ORC file.'
+'cs','char(5)','Inferred from ORC file.'
+'cl','char(140)','Inferred from ORC file.'
+'vc','varchar(32)','Inferred from ORC file.'
---- TYPES
STRING, STRING, STRING
====
@@ -115,7 +112,7 @@
====
---- QUERY
create external table transactional_complextypes_clone like ORC
-'$NAMENODE/$MANAGED_WAREHOUSE_DIR/complextypestbl_orc_def/base_0000001/bucket_00000_0'
+'$TRANSACTIONAL_COMPLEXTYPESTBL_FILE'
stored as orc;
---- RESULTS
'Table has been created.'
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index f6282cb..455a1da 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -36,6 +36,7 @@
IS_ADLS,
FILESYSTEM_NAME)
from tests.common.impala_cluster import ImpalaCluster
+from tests.util.filesystem_utils import FILESYSTEM_PREFIX
# Validates DDL statements (create, drop)
class TestDdlStatements(TestDdlBase):
@@ -298,9 +299,17 @@
@SkipIfS3.hive
@UniqueDatabase.parametrize(sync_ddl=True)
def test_create_table_like_file_orc(self, vector, unique_database):
+ COMPLEXTYPETBL_PATH = 'test-warehouse/managed/complextypestbl_orc_def/'
+ base_dir = filter(lambda s: s.startswith('base'),
+ self.filesystem_client.ls(COMPLEXTYPETBL_PATH))[0]
+ bucket_file = filter(lambda s: s.startswith('bucket'),
+ self.filesystem_client.ls(COMPLEXTYPETBL_PATH + base_dir))[0]
vector.get_value('exec_option')['abort_on_error'] = False
self.run_test_case('QueryTest/create-table-like-file-orc', vector,
- use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
+ use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector),
+ test_file_vars={
+ '$TRANSACTIONAL_COMPLEXTYPESTBL_FILE':
+ FILESYSTEM_PREFIX + '/' + COMPLEXTYPETBL_PATH + base_dir + '/' + bucket_file})
@UniqueDatabase.parametrize(sync_ddl=True)
def test_create_table_as_select(self, vector, unique_database):
