blob: 81f61a673fae2bc7bc7432ece2fa088654c18785 [file] [log] [blame]
PREHOOK: query: create table srcpart_merge_dp_n2 like srcpart
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@srcpart_merge_dp_n2
POSTHOOK: query: create table srcpart_merge_dp_n2 like srcpart
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@srcpart_merge_dp_n2
PREHOOK: query: create table merge_dynamic_part_n2 like srcpart
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@merge_dynamic_part_n2
POSTHOOK: query: create table merge_dynamic_part_n2 like srcpart
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@merge_dynamic_part_n2
PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=11)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2
POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=11)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
PREHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=11)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
POSTHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=11)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=11)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=11)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
PREHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=11)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
POSTHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=11)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=12)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2
POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=12)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
PREHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=12)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
POSTHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=12)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=12)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=12)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
PREHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=12)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
POSTHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-08', hr=12)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
PREHOOK: query: load data local inpath '../../data/files/kv1.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-09', hr=11)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2
POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-09', hr=11)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=11
PREHOOK: query: load data local inpath '../../data/files/kv2.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-09', hr=11)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=11
POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-09', hr=11)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=11
PREHOOK: query: load data local inpath '../../data/files/kv1.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-09', hr=12)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2
POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-09', hr=12)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=12
PREHOOK: query: load data local inpath '../../data/files/kv2.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-09', hr=12)
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=12
POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' INTO TABLE srcpart_merge_dp_n2 partition(ds='2008-04-09', hr=12)
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=12
PREHOOK: query: show partitions srcpart_merge_dp_n2
PREHOOK: type: SHOWPARTITIONS
PREHOOK: Input: default@srcpart_merge_dp_n2
POSTHOOK: query: show partitions srcpart_merge_dp_n2
POSTHOOK: type: SHOWPARTITIONS
POSTHOOK: Input: default@srcpart_merge_dp_n2
ds=2008-04-08/hr=11
ds=2008-04-08/hr=12
ds=2008-04-09/hr=11
ds=2008-04-09/hr=12
PREHOOK: query: explain
insert overwrite table merge_dynamic_part_n2 partition (ds, hr) select key, value, ds, hr from srcpart_merge_dp_n2 where ds>='2008-04-08'
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_merge_dp_n2
PREHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
PREHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
PREHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=11
PREHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=12
PREHOOK: Output: default@merge_dynamic_part_n2
POSTHOOK: query: explain
insert overwrite table merge_dynamic_part_n2 partition (ds, hr) select key, value, ds, hr from srcpart_merge_dp_n2 where ds>='2008-04-08'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_merge_dp_n2
POSTHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
POSTHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
POSTHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=11
POSTHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=12
POSTHOOK: Output: default@merge_dynamic_part_n2
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
Reducer 3 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: srcpart_merge_dp_n2
filterExpr: (ds >= '2008-04-08') (type: boolean)
Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL
Select Operator
expressions: key (type: string), value (type: string), ds (type: string), hr (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
outputColumnNames: key, value, ds, hr
Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL
Group By Operator
aggregations: max(length(key)), avg(COALESCE(length(key),0)), count(1), count(key), compute_bit_vector(key, 'hll'), max(length(value)), avg(COALESCE(length(value),0)), count(value), compute_bit_vector(value, 'hll')
keys: ds (type: string), hr (type: string)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 594 Data size: 685904 Basic stats: PARTIAL Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
null sort order: zz
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 594 Data size: 685904 Basic stats: PARTIAL Column stats: PARTIAL
value expressions: _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: struct<count:bigint,sum:double,input:int>), _col9 (type: bigint), _col10 (type: binary)
Reduce Output Operator
key expressions: _col2 (type: string), _col3 (type: string)
null sort order: aa
sort order: ++
Map-reduce partition columns: _col2 (type: string), _col3 (type: string)
Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL
value expressions: _col0 (type: string), _col1 (type: string)
Execution mode: llap
LLAP IO: all inputs
Reducer 2
Execution mode: llap
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector(VALUE._col4), max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7), compute_bit_vector(VALUE._col8)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 297 Data size: 327128 Basic stats: PARTIAL Column stats: PARTIAL
Select Operator
expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col7,0)) (type: bigint), COALESCE(_col8,0) (type: double), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Statistics: Num rows: 297 Data size: 327128 Basic stats: PARTIAL Column stats: PARTIAL
File Output Operator
compressed: false
Statistics: Num rows: 297 Data size: 327128 Basic stats: PARTIAL Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Reducer 3
Execution mode: llap
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
File Output Operator
compressed: false
Dp Sort State: PARTITION_SORTED
Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.merge_dynamic_part_n2
Stage: Stage-2
Dependency Collection
Stage: Stage-0
Move Operator
tables:
partition:
ds
hr
replace: true
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.merge_dynamic_part_n2
Stage: Stage-3
Stats Work
Basic Stats Work:
Column Stats Desc:
Columns: key, value
Column Types: string, string
Table: default.merge_dynamic_part_n2
PREHOOK: query: insert overwrite table merge_dynamic_part_n2 partition (ds, hr) select key, value, ds, hr from srcpart_merge_dp_n2 where ds>='2008-04-08'
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_merge_dp_n2
PREHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
PREHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
PREHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=11
PREHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=12
PREHOOK: Output: default@merge_dynamic_part_n2
POSTHOOK: query: insert overwrite table merge_dynamic_part_n2 partition (ds, hr) select key, value, ds, hr from srcpart_merge_dp_n2 where ds>='2008-04-08'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_merge_dp_n2
POSTHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=11
POSTHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-08/hr=12
POSTHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=11
POSTHOOK: Input: default@srcpart_merge_dp_n2@ds=2008-04-09/hr=12
POSTHOOK: Output: default@merge_dynamic_part_n2
POSTHOOK: Output: default@merge_dynamic_part_n2@ds=2008-04-08/hr=11
POSTHOOK: Output: default@merge_dynamic_part_n2@ds=2008-04-08/hr=12
POSTHOOK: Output: default@merge_dynamic_part_n2@ds=2008-04-09/hr=11
POSTHOOK: Output: default@merge_dynamic_part_n2@ds=2008-04-09/hr=12
POSTHOOK: Lineage: merge_dynamic_part_n2 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp_n2)srcpart_merge_dp_n2.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: merge_dynamic_part_n2 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp_n2)srcpart_merge_dp_n2.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: merge_dynamic_part_n2 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp_n2)srcpart_merge_dp_n2.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: merge_dynamic_part_n2 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp_n2)srcpart_merge_dp_n2.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: merge_dynamic_part_n2 PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart_merge_dp_n2)srcpart_merge_dp_n2.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: merge_dynamic_part_n2 PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart_merge_dp_n2)srcpart_merge_dp_n2.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: merge_dynamic_part_n2 PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart_merge_dp_n2)srcpart_merge_dp_n2.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: merge_dynamic_part_n2 PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart_merge_dp_n2)srcpart_merge_dp_n2.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: select ds, hr, count(1) from merge_dynamic_part_n2 where ds>='2008-04-08' group by ds, hr
PREHOOK: type: QUERY
PREHOOK: Input: default@merge_dynamic_part_n2
PREHOOK: Input: default@merge_dynamic_part_n2@ds=2008-04-08/hr=11
PREHOOK: Input: default@merge_dynamic_part_n2@ds=2008-04-08/hr=12
PREHOOK: Input: default@merge_dynamic_part_n2@ds=2008-04-09/hr=11
PREHOOK: Input: default@merge_dynamic_part_n2@ds=2008-04-09/hr=12
#### A masked pattern was here ####
POSTHOOK: query: select ds, hr, count(1) from merge_dynamic_part_n2 where ds>='2008-04-08' group by ds, hr
POSTHOOK: type: QUERY
POSTHOOK: Input: default@merge_dynamic_part_n2
POSTHOOK: Input: default@merge_dynamic_part_n2@ds=2008-04-08/hr=11
POSTHOOK: Input: default@merge_dynamic_part_n2@ds=2008-04-08/hr=12
POSTHOOK: Input: default@merge_dynamic_part_n2@ds=2008-04-09/hr=11
POSTHOOK: Input: default@merge_dynamic_part_n2@ds=2008-04-09/hr=12
#### A masked pattern was here ####
2008-04-08 11 500
2008-04-08 12 500
2008-04-09 11 1000
2008-04-09 12 1000
PREHOOK: query: show table extended like `merge_dynamic_part_n2`
PREHOOK: type: SHOW_TABLESTATUS
POSTHOOK: query: show table extended like `merge_dynamic_part_n2`
POSTHOOK: type: SHOW_TABLESTATUS
tableName:merge_dynamic_part_n2
#### A masked pattern was here ####
inputformat:org.apache.hadoop.mapred.TextInputFormat
outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
columns:struct columns { string key, string value}
partitioned:true
partitionColumns:struct partition_columns { string ds, string hr}
totalNumberFiles:4
totalFileSize:34830
maxFileSize:11603
minFileSize:5812
#### A masked pattern was here ####