| PREHOOK: query: CREATE TABLE orc_create_staging ( |
| str STRING, |
| mp MAP<STRING,STRING>, |
| lst ARRAY<STRING>, |
| strct STRUCT<A:STRING,B:STRING> |
| ) ROW FORMAT DELIMITED |
| FIELDS TERMINATED BY '|' |
| COLLECTION ITEMS TERMINATED BY ',' |
| MAP KEYS TERMINATED BY ':' |
| PREHOOK: type: CREATETABLE |
| PREHOOK: Output: database:default |
| PREHOOK: Output: default@orc_create_staging |
| POSTHOOK: query: CREATE TABLE orc_create_staging ( |
| str STRING, |
| mp MAP<STRING,STRING>, |
| lst ARRAY<STRING>, |
| strct STRUCT<A:STRING,B:STRING> |
| ) ROW FORMAT DELIMITED |
| FIELDS TERMINATED BY '|' |
| COLLECTION ITEMS TERMINATED BY ',' |
| MAP KEYS TERMINATED BY ':' |
| POSTHOOK: type: CREATETABLE |
| POSTHOOK: Output: database:default |
| POSTHOOK: Output: default@orc_create_staging |
| PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging |
| PREHOOK: type: LOAD |
| #### A masked pattern was here #### |
| PREHOOK: Output: default@orc_create_staging |
| POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging |
| POSTHOOK: type: LOAD |
| #### A masked pattern was here #### |
| POSTHOOK: Output: default@orc_create_staging |
| PREHOOK: query: CREATE TABLE orc_create_complex ( |
| str STRING, |
| mp MAP<STRING,STRING>, |
| lst ARRAY<STRING>, |
| strct STRUCT<A:STRING,B:STRING>, |
| val INT |
| ) STORED AS ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="1000", "orc.compress.size"="10000") |
| PREHOOK: type: CREATETABLE |
| PREHOOK: Output: database:default |
| PREHOOK: Output: default@orc_create_complex |
| POSTHOOK: query: CREATE TABLE orc_create_complex ( |
| str STRING, |
| mp MAP<STRING,STRING>, |
| lst ARRAY<STRING>, |
| strct STRUCT<A:STRING,B:STRING>, |
| val INT |
| ) STORED AS ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="1000", "orc.compress.size"="10000") |
| POSTHOOK: type: CREATETABLE |
| POSTHOOK: Output: database:default |
| POSTHOOK: Output: default@orc_create_complex |
| PREHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT str,mp,lst,strct,0 FROM orc_create_staging |
| PREHOOK: type: QUERY |
| PREHOOK: Input: default@orc_create_staging |
| PREHOOK: Output: default@orc_create_complex |
| POSTHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT str,mp,lst,strct,0 FROM orc_create_staging |
| POSTHOOK: type: QUERY |
| POSTHOOK: Input: default@orc_create_staging |
| POSTHOOK: Output: default@orc_create_complex |
| POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.val SIMPLE [] |
| PREHOOK: query: INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,strct,0 FROM orc_create_staging |
| PREHOOK: type: QUERY |
| PREHOOK: Input: default@orc_create_staging |
| PREHOOK: Output: default@orc_create_complex |
| POSTHOOK: query: INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,strct,0 FROM orc_create_staging |
| POSTHOOK: type: QUERY |
| POSTHOOK: Input: default@orc_create_staging |
| POSTHOOK: Output: default@orc_create_complex |
| POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.val SIMPLE [] |
| Found 2 items |
| #### A masked pattern was here #### |
| PREHOOK: query: select sum(hash(*)) from orc_create_complex |
| PREHOOK: type: QUERY |
| PREHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| POSTHOOK: query: select sum(hash(*)) from orc_create_complex |
| POSTHOOK: type: QUERY |
| POSTHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| 953053114 |
| PREHOOK: query: ALTER TABLE orc_create_complex CONCATENATE |
| PREHOOK: type: ALTER_TABLE_MERGE |
| PREHOOK: Input: default@orc_create_complex |
| PREHOOK: Output: default@orc_create_complex |
| POSTHOOK: query: ALTER TABLE orc_create_complex CONCATENATE |
| POSTHOOK: type: ALTER_TABLE_MERGE |
| POSTHOOK: Input: default@orc_create_complex |
| POSTHOOK: Output: default@orc_create_complex |
| Found 1 items |
| #### A masked pattern was here #### |
| PREHOOK: query: select sum(hash(*)) from orc_create_complex |
| PREHOOK: type: QUERY |
| PREHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| POSTHOOK: query: select sum(hash(*)) from orc_create_complex |
| POSTHOOK: type: QUERY |
| POSTHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| 953053114 |
| PREHOOK: query: ALTER TABLE orc_create_complex |
| CHANGE COLUMN strct strct STRUCT<A:STRING,B:STRING,C:STRING> |
| PREHOOK: type: ALTERTABLE_RENAMECOL |
| PREHOOK: Input: default@orc_create_complex |
| PREHOOK: Output: default@orc_create_complex |
| POSTHOOK: query: ALTER TABLE orc_create_complex |
| CHANGE COLUMN strct strct STRUCT<A:STRING,B:STRING,C:STRING> |
| POSTHOOK: type: ALTERTABLE_RENAMECOL |
| POSTHOOK: Input: default@orc_create_complex |
| POSTHOOK: Output: default@orc_create_complex |
| PREHOOK: query: EXPLAIN VECTORIZATION |
| INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,NAMED_STRUCT('A',strct.A,'B',strct.B,'C','c'),0 FROM orc_create_staging |
| PREHOOK: type: QUERY |
| PREHOOK: Input: default@orc_create_staging |
| PREHOOK: Output: default@orc_create_complex |
| POSTHOOK: query: EXPLAIN VECTORIZATION |
| INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,NAMED_STRUCT('A',strct.A,'B',strct.B,'C','c'),0 FROM orc_create_staging |
| POSTHOOK: type: QUERY |
| POSTHOOK: Input: default@orc_create_staging |
| POSTHOOK: Output: default@orc_create_complex |
| PLAN VECTORIZATION: |
| enabled: true |
| enabledConditionsMet: [hive.vectorized.execution.enabled IS true] |
| |
| STAGE DEPENDENCIES: |
| Stage-1 is a root stage |
| Stage-2 depends on stages: Stage-1 |
| Stage-0 depends on stages: Stage-2 |
| Stage-3 depends on stages: Stage-0 |
| |
| STAGE PLANS: |
| Stage: Stage-1 |
| Tez |
| #### A masked pattern was here #### |
| Vertices: |
| Map 1 |
| Map Operator Tree: |
| TableScan |
| alias: orc_create_staging |
| Pruned Column Paths: strct.a, strct.b |
| Statistics: Num rows: 1 Data size: 3440 Basic stats: COMPLETE Column stats: NONE |
| Select Operator |
| expressions: str (type: string), mp (type: map<string,string>), lst (type: array<string>), named_struct('A',strct.a,'B',strct.b,'C','c') (type: struct<a:string,b:string,c:string>), 0 (type: int) |
| outputColumnNames: _col0, _col1, _col2, _col3, _col4 |
| Statistics: Num rows: 1 Data size: 3440 Basic stats: COMPLETE Column stats: NONE |
| File Output Operator |
| compressed: false |
| Statistics: Num rows: 1 Data size: 3440 Basic stats: COMPLETE Column stats: NONE |
| table: |
| input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat |
| output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat |
| serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde |
| name: default.orc_create_complex |
| Execution mode: vectorized, llap |
| LLAP IO: all inputs |
| Map Vectorization: |
| enabled: true |
| enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true |
| inputFormatFeatureSupport: [DECIMAL_64] |
| featureSupportInUse: [DECIMAL_64] |
| inputFileFormats: org.apache.hadoop.mapred.TextInputFormat |
| allNative: false |
| usesVectorUDFAdaptor: true |
| vectorized: true |
| |
| Stage: Stage-2 |
| Dependency Collection |
| |
| Stage: Stage-0 |
| Move Operator |
| tables: |
| replace: false |
| table: |
| input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat |
| output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat |
| serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde |
| name: default.orc_create_complex |
| |
| Stage: Stage-3 |
| Stats Work |
| Basic Stats Work: |
| |
| PREHOOK: query: INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,NAMED_STRUCT('A',strct.A,'B',strct.B,'C','c'),0 FROM orc_create_staging |
| PREHOOK: type: QUERY |
| PREHOOK: Input: default@orc_create_staging |
| PREHOOK: Output: default@orc_create_complex |
| POSTHOOK: query: INSERT INTO TABLE orc_create_complex SELECT str,mp,lst,NAMED_STRUCT('A',strct.A,'B',strct.B,'C','c'),0 FROM orc_create_staging |
| POSTHOOK: type: QUERY |
| POSTHOOK: Input: default@orc_create_staging |
| POSTHOOK: Output: default@orc_create_complex |
| POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.strct EXPRESSION [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] |
| POSTHOOK: Lineage: orc_create_complex.val SIMPLE [] |
| Found 2 items |
| #### A masked pattern was here #### |
| PREHOOK: query: EXPLAIN VECTORIZATION |
| select sum(hash(*)) from orc_create_complex |
| PREHOOK: type: QUERY |
| PREHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| POSTHOOK: query: EXPLAIN VECTORIZATION |
| select sum(hash(*)) from orc_create_complex |
| POSTHOOK: type: QUERY |
| POSTHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| PLAN VECTORIZATION: |
| enabled: true |
| enabledConditionsMet: [hive.vectorized.execution.enabled IS true] |
| |
| STAGE DEPENDENCIES: |
| Stage-1 is a root stage |
| Stage-0 depends on stages: Stage-1 |
| |
| STAGE PLANS: |
| Stage: Stage-1 |
| Tez |
| #### A masked pattern was here #### |
| Edges: |
| Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) |
| #### A masked pattern was here #### |
| Vertices: |
| Map 1 |
| Map Operator Tree: |
| TableScan |
| alias: orc_create_complex |
| Statistics: Num rows: 6 Data size: 21816 Basic stats: COMPLETE Column stats: NONE |
| Select Operator |
| expressions: hash(str,mp,lst,strct,val) (type: int) |
| outputColumnNames: _col0 |
| Statistics: Num rows: 6 Data size: 21816 Basic stats: COMPLETE Column stats: NONE |
| Group By Operator |
| aggregations: sum(_col0) |
| minReductionHashAggr: 0.99 |
| mode: hash |
| outputColumnNames: _col0 |
| Statistics: Num rows: 1 Data size: 3644 Basic stats: COMPLETE Column stats: NONE |
| Reduce Output Operator |
| null sort order: |
| sort order: |
| Statistics: Num rows: 1 Data size: 3644 Basic stats: COMPLETE Column stats: NONE |
| value expressions: _col0 (type: bigint) |
| Execution mode: vectorized, llap |
| LLAP IO: all inputs |
| Map Vectorization: |
| enabled: true |
| enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true |
| inputFormatFeatureSupport: [DECIMAL_64] |
| featureSupportInUse: [DECIMAL_64] |
| inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat |
| allNative: false |
| usesVectorUDFAdaptor: true |
| vectorized: true |
| Reducer 2 |
| Execution mode: vectorized, llap |
| Reduce Vectorization: |
| enabled: true |
| enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true |
| allNative: false |
| usesVectorUDFAdaptor: false |
| vectorized: true |
| Reduce Operator Tree: |
| Group By Operator |
| aggregations: sum(VALUE._col0) |
| mode: mergepartial |
| outputColumnNames: _col0 |
| Statistics: Num rows: 1 Data size: 3644 Basic stats: COMPLETE Column stats: NONE |
| File Output Operator |
| compressed: false |
| Statistics: Num rows: 1 Data size: 3644 Basic stats: COMPLETE Column stats: NONE |
| table: |
| input format: org.apache.hadoop.mapred.SequenceFileInputFormat |
| output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat |
| serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe |
| |
| Stage: Stage-0 |
| Fetch Operator |
| limit: -1 |
| Processor Tree: |
| ListSink |
| |
| PREHOOK: query: select sum(hash(*)) from orc_create_complex |
| PREHOOK: type: QUERY |
| PREHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| POSTHOOK: query: select sum(hash(*)) from orc_create_complex |
| POSTHOOK: type: QUERY |
| POSTHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| 4334574594 |
| PREHOOK: query: ALTER TABLE orc_create_complex CONCATENATE |
| PREHOOK: type: ALTER_TABLE_MERGE |
| PREHOOK: Input: default@orc_create_complex |
| PREHOOK: Output: default@orc_create_complex |
| POSTHOOK: query: ALTER TABLE orc_create_complex CONCATENATE |
| POSTHOOK: type: ALTER_TABLE_MERGE |
| POSTHOOK: Input: default@orc_create_complex |
| POSTHOOK: Output: default@orc_create_complex |
| Found 2 items |
| #### A masked pattern was here #### |
| PREHOOK: query: select sum(hash(*)) from orc_create_complex |
| PREHOOK: type: QUERY |
| PREHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| POSTHOOK: query: select sum(hash(*)) from orc_create_complex |
| POSTHOOK: type: QUERY |
| POSTHOOK: Input: default@orc_create_complex |
| #### A masked pattern was here #### |
| 4334574594 |