blob: 5ca188b9483779671e6649ee9354b78982167d91 [file] [log] [blame]
PREHOOK: query: CREATE TABLE orc_create_staging (
str STRING,
mp MAP<STRING,STRING>,
lst ARRAY<STRING>,
strct STRUCT<A:STRING,B:STRING>
) ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
COLLECTION ITEMS TERMINATED BY ','
MAP KEYS TERMINATED BY ':'
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@orc_create_staging
POSTHOOK: query: CREATE TABLE orc_create_staging (
str STRING,
mp MAP<STRING,STRING>,
lst ARRAY<STRING>,
strct STRUCT<A:STRING,B:STRING>
) ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
COLLECTION ITEMS TERMINATED BY ','
MAP KEYS TERMINATED BY ':'
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@orc_create_staging
PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@orc_create_staging
POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@orc_create_staging
PREHOOK: query: CREATE TABLE orc_create_complex (
str STRING,
mp MAP<STRING,STRING>,
lst ARRAY<STRING>,
strct STRUCT<A:STRING,B:STRING>,
val INT
) STORED AS ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="1000", "orc.compress.size"="10000")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@orc_create_complex
POSTHOOK: query: CREATE TABLE orc_create_complex (
str STRING,
mp MAP<STRING,STRING>,
lst ARRAY<STRING>,
strct STRUCT<A:STRING,B:STRING>,
val INT
) STORED AS ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="1000", "orc.compress.size"="10000")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@orc_create_complex
PREHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT str,mp,lst,strct,0 FROM orc_create_staging
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_create_staging
PREHOOK: Output: default@orc_create_complex
POSTHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT str,mp,lst,strct,0 FROM orc_create_staging
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_create_staging
POSTHOOK: Output: default@orc_create_complex
POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.val SIMPLE []
PREHOOK: query: INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,strct,0 FROM orc_create_staging
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_create_staging
PREHOOK: Output: default@orc_create_complex
POSTHOOK: query: INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,strct,0 FROM orc_create_staging
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_create_staging
POSTHOOK: Output: default@orc_create_complex
POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.val SIMPLE []
Found 2 items
#### A masked pattern was here ####
PREHOOK: query: select sum(hash(*)) from orc_create_complex
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
POSTHOOK: query: select sum(hash(*)) from orc_create_complex
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
953053114
PREHOOK: query: ALTER TABLE orc_create_complex CONCATENATE
PREHOOK: type: ALTER_TABLE_MERGE
PREHOOK: Input: default@orc_create_complex
PREHOOK: Output: default@orc_create_complex
POSTHOOK: query: ALTER TABLE orc_create_complex CONCATENATE
POSTHOOK: type: ALTER_TABLE_MERGE
POSTHOOK: Input: default@orc_create_complex
POSTHOOK: Output: default@orc_create_complex
Found 1 items
#### A masked pattern was here ####
PREHOOK: query: select sum(hash(*)) from orc_create_complex
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
POSTHOOK: query: select sum(hash(*)) from orc_create_complex
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
953053114
PREHOOK: query: ALTER TABLE orc_create_complex
CHANGE COLUMN strct strct STRUCT<A:STRING,B:STRING,C:STRING>
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@orc_create_complex
PREHOOK: Output: default@orc_create_complex
POSTHOOK: query: ALTER TABLE orc_create_complex
CHANGE COLUMN strct strct STRUCT<A:STRING,B:STRING,C:STRING>
POSTHOOK: type: ALTERTABLE_RENAMECOL
POSTHOOK: Input: default@orc_create_complex
POSTHOOK: Output: default@orc_create_complex
PREHOOK: query: EXPLAIN VECTORIZATION
INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,NAMED_STRUCT('A',strct.A,'B',strct.B,'C','c'),0 FROM orc_create_staging
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_create_staging
PREHOOK: Output: default@orc_create_complex
POSTHOOK: query: EXPLAIN VECTORIZATION
INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,NAMED_STRUCT('A',strct.A,'B',strct.B,'C','c'),0 FROM orc_create_staging
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_create_staging
POSTHOOK: Output: default@orc_create_complex
PLAN VECTORIZATION:
enabled: true
enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: orc_create_staging
Pruned Column Paths: strct.a, strct.b
Statistics: Num rows: 1 Data size: 3440 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: str (type: string), mp (type: map<string,string>), lst (type: array<string>), named_struct('A',strct.a,'B',strct.b,'C','c') (type: struct<a:string,b:string,c:string>), 0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1 Data size: 3440 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 1 Data size: 3440 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
name: default.orc_create_complex
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
inputFormatFeatureSupport: [DECIMAL_64]
featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
allNative: false
usesVectorUDFAdaptor: true
vectorized: true
Stage: Stage-2
Dependency Collection
Stage: Stage-0
Move Operator
tables:
replace: false
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
name: default.orc_create_complex
Stage: Stage-3
Stats Work
Basic Stats Work:
PREHOOK: query: INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,NAMED_STRUCT('A',strct.A,'B',strct.B,'C','c'),0 FROM orc_create_staging
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_create_staging
PREHOOK: Output: default@orc_create_complex
POSTHOOK: query: INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,NAMED_STRUCT('A',strct.A,'B',strct.B,'C','c'),0 FROM orc_create_staging
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_create_staging
POSTHOOK: Output: default@orc_create_complex
POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.strct EXPRESSION [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.val SIMPLE []
Found 2 items
#### A masked pattern was here ####
PREHOOK: query: EXPLAIN VECTORIZATION
select sum(hash(*)) from orc_create_complex
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN VECTORIZATION
select sum(hash(*)) from orc_create_complex
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
PLAN VECTORIZATION:
enabled: true
enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: orc_create_complex
Statistics: Num rows: 6 Data size: 21816 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: hash(str,mp,lst,strct,val) (type: int)
outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 21816 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col0)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 3644 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
Statistics: Num rows: 1 Data size: 3644 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFormatFeatureSupport: [DECIMAL_64]
featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: true
vectorized: true
Reducer 2
Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 3644 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 1 Data size: 3644 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
ListSink
PREHOOK: query: select sum(hash(*)) from orc_create_complex
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
POSTHOOK: query: select sum(hash(*)) from orc_create_complex
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
4334574594
PREHOOK: query: ALTER TABLE orc_create_complex CONCATENATE
PREHOOK: type: ALTER_TABLE_MERGE
PREHOOK: Input: default@orc_create_complex
PREHOOK: Output: default@orc_create_complex
POSTHOOK: query: ALTER TABLE orc_create_complex CONCATENATE
POSTHOOK: type: ALTER_TABLE_MERGE
POSTHOOK: Input: default@orc_create_complex
POSTHOOK: Output: default@orc_create_complex
Found 2 items
#### A masked pattern was here ####
PREHOOK: query: select sum(hash(*)) from orc_create_complex
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
POSTHOOK: query: select sum(hash(*)) from orc_create_complex
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_create_complex
#### A masked pattern was here ####
4334574594