IMPALA-9652: CTAS doesn't respect transactional properties
Because of a bug the INSERT part of a CTAS statement didn't
run in a transaction and it just put the new files under the
root directory of the table. This didn't cause too much problems
because there couldn't be any concurrent operations as the table was
under construction. However, this behavior is not working particularly
well in the context of replication, as the notification event needs
a transaction id.
With this fix the INSERT operation runs in a transaction and the new
files are created under a delta directory.
Testing:
* Added CTAS statements and SHOW FILES <tbl> queries to acid-insert.test
Check if the files are created in a delta directory, if so, then
the INSERT must have been running in a transaction.
Change-Id: I6ed96aeadbcead9fdc548da5922a066460ff9f77
Reviewed-on: http://gerrit.cloudera.org:8080/16472
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 8568618..69301f9 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -1579,7 +1579,8 @@
if (!analysisResult.isCreateTableAsSelectStmt()) {
return result;
}
- } else if (analysisResult.isInsertStmt() ||
+ }
+ if (analysisResult.isInsertStmt() ||
analysisResult.isCreateTableAsSelectStmt()) {
InsertStmt insertStmt = analysisResult.getInsertStmt();
FeTable targetTable = insertStmt.getTargetTable();
@@ -2020,7 +2021,9 @@
Preconditions.checkState(
AcidUtils.isTransactionalTable(targetTable.getMetaStoreTable().getParameters()));
List<LockComponent> lockComponents = new ArrayList<>(tables.size());
- for (FeTable table : tables) {
+ List<FeTable> lockTables = new ArrayList<>(tables);
+ if (!lockTables.contains(targetTable)) lockTables.add(targetTable);
+ for (FeTable table : lockTables) {
if (!AcidUtils.isTransactionalTable(table.getMetaStoreTable().getParameters())) {
continue;
}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test b/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test
index 094b463..8665c56 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test
@@ -45,6 +45,28 @@
INT
====
---- QUERY
+create table insertonly_nopart_ctas
+tblproperties('transactional'='true', 'transactional_properties'='insert_only')
+as select * from insertonly_nopart;
+select * from insertonly_nopart_ctas;
+---- RESULTS
+100
+---- TYPES
+INT
+====
+---- QUERY
+show files in insertonly_nopart_ctas;
+---- LABELS
+Path,Size,Partition
+---- RESULTS
+row_regex:'$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE.db/insertonly_nopart_ctas/delta_1_1/.*','\d+B',''
+---- TYPES
+STRING,STRING,STRING
+====
+---- QUERY
+drop table insertonly_nopart_ctas;
+====
+---- QUERY
insert overwrite insertonly_nopart
select * from insertonly_nopart limit 0;
====
@@ -128,3 +150,32 @@
---- TYPES
INT,INT
====
+---- QUERY
+create table insertonly_part_ctas
+partitioned by (p)
+tblproperties('transactional'='true', 'transactional_properties'='insert_only')
+as select * from insertonly_part;
+select p, i from insertonly_part_ctas;
+---- RESULTS
+1,1000
+2,2000
+3,31
+4,4000
+5,5000
+5,5001
+---- TYPES
+INT,INT
+====
+---- QUERY
+show files in insertonly_part_ctas;
+---- LABELS
+Path,Size,Partition
+---- RESULTS
+row_regex:'$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE.db/insertonly_part_ctas/p=1/delta_1_1/.*','\d+B','p=1'
+row_regex:'$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE.db/insertonly_part_ctas/p=2/delta_1_1/.*','\d+B','p=2'
+row_regex:'$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE.db/insertonly_part_ctas/p=3/delta_1_1/.*','\d+B','p=3'
+row_regex:'$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE.db/insertonly_part_ctas/p=4/delta_1_1/.*','\d+B','p=4'
+row_regex:'$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE.db/insertonly_part_ctas/p=5/delta_1_1/.*','\d+B','p=5'
+---- TYPES
+STRING,STRING,STRING
+====
