blob: b0168dc87fa7b6aed43ff0f2c2de1b8f876aa45e [file] [log] [blame]
PREHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as
select cast(key as int) as key, value from src
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@masking_test_n_mv
POSTHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as
select cast(key as int) as key, value from src
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
POSTHOOK: Output: default@masking_test_n_mv
POSTHOOK: Lineage: masking_test_n_mv.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: masking_test_n_mv.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: explain
create materialized view `masking_test_view_n_mv` as
select key from `masking_test_n_mv`
PREHOOK: type: CREATE_MATERIALIZED_VIEW
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Output: database:default
PREHOOK: Output: default@masking_test_view_n_mv
POSTHOOK: query: explain
create materialized view `masking_test_view_n_mv` as
select key from `masking_test_n_mv`
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Output: database:default
POSTHOOK: Output: default@masking_test_view_n_mv
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-2 depends on stages: Stage-1
Stage-4 depends on stages: Stage-0, Stage-2
Stage-3 depends on stages: Stage-4
Stage-5 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: masking_test_n_mv
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
name: default.masking_test_view_n_mv
Select Operator
expressions: _col0 (type: int)
outputColumnNames: col1
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(col1), max(col1), count(1), count(col1), compute_bit_vector(col1, 'hll')
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
null sort order:
sort order:
Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary)
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
Group By Operator
aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector(VALUE._col4)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: 'LONG' (type: string), UDFToLong(_col0) (type: bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 1 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-2
Dependency Collection
Stage: Stage-4
Create Materialized View
columns: key int
expanded text: select `masking_test_n_mv`.`key` from `default`.`masking_test_n_mv`
name: default.masking_test_view_n_mv
original text: select key from masking_test_n_mv
rewrite enabled: true
Stage: Stage-3
Stats Work
Basic Stats Work:
Column Stats Desc:
Columns: key
Column Types: int
Table: default.masking_test_view_n_mv
Stage: Stage-5
Materialized View Update
name: default.masking_test_view_n_mv
retrieve and include: true
Stage: Stage-0
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####
PREHOOK: query: create materialized view `masking_test_view_n_mv` as
select key from `masking_test_n_mv`
PREHOOK: type: CREATE_MATERIALIZED_VIEW
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Output: database:default
PREHOOK: Output: default@masking_test_view_n_mv
POSTHOOK: query: create materialized view `masking_test_view_n_mv` as
select key from `masking_test_n_mv`
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Output: database:default
POSTHOOK: Output: default@masking_test_view_n_mv
PREHOOK: query: describe formatted `masking_test_view_n_mv`
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@masking_test_view_n_mv
POSTHOOK: query: describe formatted `masking_test_view_n_mv`
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@masking_test_view_n_mv
# col_name data_type comment
key int
# Detailed Table Information
Database: default
#### A masked pattern was here ####
Retention: 0
#### A masked pattern was here ####
Table Type: MATERIALIZED_VIEW
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
bucketing_version 2
numFiles 1
numRows 500
rawDataSize 2000
totalSize 976
#### A masked pattern was here ####
# Storage Information
SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
Compressed: No
Num Buckets: -1
Bucket Columns: []
Sort Columns: []
# Materialized View Information
Original Query: select key from masking_test_n_mv
Expanded Query: select `masking_test_n_mv`.`key` from `default`.`masking_test_n_mv`
Rewrite Enabled: Yes
Outdated for Rewriting: No
PREHOOK: query: explain
select key from `masking_test_n_mv`
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv
#### A masked pattern was here ####
POSTHOOK: query: explain
select key from `masking_test_n_mv`
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
TableScan
alias: default.masking_test_view_n_mv
filterExpr: ((key < 10) and (0 = (key % 2))) (type: boolean)
Filter Operator
predicate: ((key < 10) and (0 = (key % 2))) (type: boolean)
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
ListSink
PREHOOK: query: select key from `masking_test_n_mv`
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv
#### A masked pattern was here ####
POSTHOOK: query: select key from `masking_test_n_mv`
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv
#### A masked pattern was here ####
0
4
8
0
0
2
PREHOOK: query: create materialized view `masking_test_view_n_mv_3` as
select value, sum(key) from `masking_test_n_mv` group by value
PREHOOK: type: CREATE_MATERIALIZED_VIEW
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Output: database:default
PREHOOK: Output: default@masking_test_view_n_mv_3
POSTHOOK: query: create materialized view `masking_test_view_n_mv_3` as
select value, sum(key) from `masking_test_n_mv` group by value
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Output: database:default
POSTHOOK: Output: default@masking_test_view_n_mv_3
PREHOOK: query: explain
select key from `masking_test_n_mv` group by key
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv
#### A masked pattern was here ####
POSTHOOK: query: explain
select key from `masking_test_n_mv` group by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: default.masking_test_view_n_mv
filterExpr: ((key < 10) and (0 = (key % 2))) (type: boolean)
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((key < 10) and (0 = (key % 2))) (type: boolean)
Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: key (type: int)
minReductionHashAggr: 0.6
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: z
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
ListSink
PREHOOK: query: select key from `masking_test_n_mv` group by key
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv
#### A masked pattern was here ####
POSTHOOK: query: select key from `masking_test_n_mv` group by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv
#### A masked pattern was here ####
0
2
4
8
PREHOOK: query: explain
select value from `masking_test_n_mv` group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv_3
#### A masked pattern was here ####
POSTHOOK: query: explain
select value from `masking_test_n_mv` group by value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv_3
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: masking_test_n_mv
filterExpr: ((key < 10) and ((key % 2) = 0)) (type: boolean)
Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((key < 10) and ((key % 2) = 0)) (type: boolean)
Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: reverse(value) (type: string)
outputColumnNames: _col0
Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: string)
minReductionHashAggr: 0.6
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: z
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
ListSink
PREHOOK: query: select value from `masking_test_n_mv` group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv_3
#### A masked pattern was here ####
POSTHOOK: query: select value from `masking_test_n_mv` group by value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv_3
#### A masked pattern was here ####
2_lav
4_lav
0_lav
8_lav
PREHOOK: query: explain
select value, sum(key) from `masking_test_n_mv` group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv_3
#### A masked pattern was here ####
POSTHOOK: query: explain
select value, sum(key) from `masking_test_n_mv` group by value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv_3
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: masking_test_n_mv
filterExpr: ((key < 10) and ((key % 2) = 0)) (type: boolean)
Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((key < 10) and ((key % 2) = 0)) (type: boolean)
Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: reverse(value) (type: string), key (type: int)
outputColumnNames: _col0, _col1
Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(_col1)
keys: _col0 (type: string)
minReductionHashAggr: 0.6
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: z
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
ListSink
PREHOOK: query: select value, sum(key) from `masking_test_n_mv` group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv_3
#### A masked pattern was here ####
POSTHOOK: query: select value, sum(key) from `masking_test_n_mv` group by value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv_3
#### A masked pattern was here ####
2_lav 2
4_lav 4
0_lav 0
8_lav 8
PREHOOK: query: create materialized view `masking_test_view_n_mv_4` as
select key, value from `masking_test_n_mv`
PREHOOK: type: CREATE_MATERIALIZED_VIEW
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Output: database:default
PREHOOK: Output: default@masking_test_view_n_mv_4
POSTHOOK: query: create materialized view `masking_test_view_n_mv_4` as
select key, value from `masking_test_n_mv`
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Output: database:default
POSTHOOK: Output: default@masking_test_view_n_mv_4
PREHOOK: query: explain
select value from `masking_test_n_mv` group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv_3
PREHOOK: Input: default@masking_test_view_n_mv_4
#### A masked pattern was here ####
POSTHOOK: query: explain
select value from `masking_test_n_mv` group by value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv_3
POSTHOOK: Input: default@masking_test_view_n_mv_4
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: default.masking_test_view_n_mv_4
filterExpr: ((key < 10) and (0 = (key % 2))) (type: boolean)
Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((key < 10) and (0 = (key % 2))) (type: boolean)
Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: reverse(value) (type: string)
outputColumnNames: _col0
Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: string)
minReductionHashAggr: 0.6
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: z
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
ListSink
PREHOOK: query: select value from `masking_test_n_mv` group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv_3
PREHOOK: Input: default@masking_test_view_n_mv_4
#### A masked pattern was here ####
POSTHOOK: query: select value from `masking_test_n_mv` group by value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv_3
POSTHOOK: Input: default@masking_test_view_n_mv_4
#### A masked pattern was here ####
2_lav
4_lav
0_lav
8_lav
PREHOOK: query: explain
select value, sum(key) from `masking_test_n_mv` group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv_3
PREHOOK: Input: default@masking_test_view_n_mv_4
#### A masked pattern was here ####
POSTHOOK: query: explain
select value, sum(key) from `masking_test_n_mv` group by value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv_3
POSTHOOK: Input: default@masking_test_view_n_mv_4
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: default.masking_test_view_n_mv_4
filterExpr: ((key < 10) and (0 = (key % 2))) (type: boolean)
Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((key < 10) and (0 = (key % 2))) (type: boolean)
Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: reverse(value) (type: string), key (type: int)
outputColumnNames: _col0, _col1
Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(_col1)
keys: _col0 (type: string)
minReductionHashAggr: 0.6
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: z
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
ListSink
PREHOOK: query: select value, sum(key) from `masking_test_n_mv` group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Input: default@masking_test_view_n_mv_3
PREHOOK: Input: default@masking_test_view_n_mv_4
#### A masked pattern was here ####
POSTHOOK: query: select value, sum(key) from `masking_test_n_mv` group by value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Input: default@masking_test_view_n_mv_3
POSTHOOK: Input: default@masking_test_view_n_mv_4
#### A masked pattern was here ####
2_lav 2
4_lav 4
0_lav 0
8_lav 8
PREHOOK: query: create table `srcTnx` stored as orc TBLPROPERTIES ('transactional'='true') as
select cast(key as int) as key, value from src
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@srcTnx
POSTHOOK: query: create table `srcTnx` stored as orc TBLPROPERTIES ('transactional'='true') as
select cast(key as int) as key, value from src
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
POSTHOOK: Output: default@srcTnx
POSTHOOK: Lineage: srctnx.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: srctnx.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: explain
create materialized view `masking_test_view_n_mv_2` as
select key from `srcTnx`
PREHOOK: type: CREATE_MATERIALIZED_VIEW
PREHOOK: Input: default@srctnx
PREHOOK: Output: database:default
PREHOOK: Output: default@masking_test_view_n_mv_2
POSTHOOK: query: explain
create materialized view `masking_test_view_n_mv_2` as
select key from `srcTnx`
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
POSTHOOK: Input: default@srctnx
POSTHOOK: Output: database:default
POSTHOOK: Output: default@masking_test_view_n_mv_2
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-2 depends on stages: Stage-1
Stage-4 depends on stages: Stage-0, Stage-2
Stage-3 depends on stages: Stage-4
Stage-5 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: srctnx
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
name: default.masking_test_view_n_mv_2
Select Operator
expressions: _col0 (type: int)
outputColumnNames: col1
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(col1), max(col1), count(1), count(col1), compute_bit_vector(col1, 'hll')
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
null sort order:
sort order:
Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary)
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
Group By Operator
aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector(VALUE._col4)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: 'LONG' (type: string), UDFToLong(_col0) (type: bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 1 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-2
Dependency Collection
Stage: Stage-4
Create Materialized View
columns: key int
expanded text: select `srctnx`.`key` from `default`.`srcTnx`
name: default.masking_test_view_n_mv_2
original text: select key from srcTnx
rewrite enabled: true
Stage: Stage-3
Stats Work
Basic Stats Work:
Column Stats Desc:
Columns: key
Column Types: int
Table: default.masking_test_view_n_mv_2
Stage: Stage-5
Materialized View Update
name: default.masking_test_view_n_mv_2
retrieve and include: true
Stage: Stage-0
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####
PREHOOK: query: create materialized view `masking_test_view_n_mv_2` as
select key from `srcTnx`
PREHOOK: type: CREATE_MATERIALIZED_VIEW
PREHOOK: Input: default@srctnx
PREHOOK: Output: database:default
PREHOOK: Output: default@masking_test_view_n_mv_2
POSTHOOK: query: create materialized view `masking_test_view_n_mv_2` as
select key from `srcTnx`
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
POSTHOOK: Input: default@srctnx
POSTHOOK: Output: database:default
POSTHOOK: Output: default@masking_test_view_n_mv_2
PREHOOK: query: describe formatted `masking_test_view_n_mv_2`
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@masking_test_view_n_mv_2
POSTHOOK: query: describe formatted `masking_test_view_n_mv_2`
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@masking_test_view_n_mv_2
# col_name data_type comment
key int
# Detailed Table Information
Database: default
#### A masked pattern was here ####
Retention: 0
#### A masked pattern was here ####
Table Type: MATERIALIZED_VIEW
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
bucketing_version 2
numFiles 1
numRows 500
rawDataSize 2000
totalSize 976
#### A masked pattern was here ####
# Storage Information
SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
Compressed: No
Num Buckets: -1
Bucket Columns: []
Sort Columns: []
# Materialized View Information
Original Query: select key from srcTnx
Expanded Query: select `srctnx`.`key` from `default`.`srcTnx`
Rewrite Enabled: Yes
Outdated for Rewriting: No
PREHOOK: query: explain
select key from `masking_test_view_n_mv_2` order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_view_n_mv_2
#### A masked pattern was here ####
POSTHOOK: query: explain
select key from `masking_test_view_n_mv_2` order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_view_n_mv_2
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: masking_test_view_n_mv_2
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: z
sort order: +
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
ListSink
PREHOOK: query: select key from `masking_test_view_n_mv_2` order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@masking_test_view_n_mv_2
#### A masked pattern was here ####
POSTHOOK: query: select key from `masking_test_view_n_mv_2` order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@masking_test_view_n_mv_2
#### A masked pattern was here ####
0
0
0
2
4
5
5
5
8
9
10
11
12
12
15
15
17
18
18
19
20
24
24
26
26
27
28
30
33
34
35
35
35
37
37
41
42
42
43
44
47
51
51
53
54
57
58
58
64
65
66
67
67
69
70
70
70
72
72
74
76
76
77
78
80
82
83
83
84
84
85
86
87
90
90
90
92
95
95
96
97
97
98
98
100
100
103
103
104
104
105
111
113
113
114
116
118
118
119
119
119
120
120
125
125
126
128
128
128
129
129
131
133
134
134
136
137
137
138
138
138
138
143
145
146
146
149
149
150
152
152
153
155
156
157
158
160
162
163
164
164
165
165
166
167
167
167
168
169
169
169
169
170
172
172
174
174
175
175
176
176
177
178
179
179
180
181
183
186
187
187
187
189
190
191
191
192
193
193
193
194
195
195
196
197
197
199
199
199
200
200
201
202
203
203
205
205
207
207
208
208
208
209
209
213
213
214
216
216
217
217
218
219
219
221
221
222
223
223
224
224
226
228
229
229
230
230
230
230
230
233
233
235
237
237
238
238
239
239
241
242
242
244
247
248
249
252
255
255
256
256
257
258
260
262
263
265
265
266
272
272
273
273
273
274
275
277
277
277
277
278
278
280
280
281
281
282
282
283
284
285
286
287
288
288
289
291
292
296
298
298
298
302
305
306
307
307
308
309
309
310
311
311
311
315
316
316
316
317
317
318
318
318
321
321
322
322
323
325
325
327
327
327
331
331
332
333
333
335
336
338
339
341
342
342
344
344
345
348
348
348
348
348
351
353
353
356
360
362
364
365
366
367
367
368
369
369
369
373
374
375
377
378
379
382
382
384
384
384
386
389
392
393
394
395
395
396
396
396
397
397
399
399
400
401
401
401
401
401
402
403
403
403
404
404
406
406
406
406
407
409
409
409
411
413
413
414
414
417
417
417
418
419
421
424
424
427
429
429
430
430
430
431
431
431
432
435
436
437
438
438
438
439
439
443
444
446
448
449
452
453
454
454
454
455
457
458
458
459
459
460
462
462
463
463
466
466
466
467
468
468
468
468
469
469
469
469
469
470
472
475
477
478
478
479
480
480
480
481
482
483
484
485
487
489
489
489
489
490
491
492
492
493
494
495
496
497
498
498
498
PREHOOK: query: drop materialized view `masking_test_view_n_mv`
PREHOOK: type: DROP_MATERIALIZED_VIEW
PREHOOK: Input: default@masking_test_view_n_mv
PREHOOK: Output: default@masking_test_view_n_mv
POSTHOOK: query: drop materialized view `masking_test_view_n_mv`
POSTHOOK: type: DROP_MATERIALIZED_VIEW
POSTHOOK: Input: default@masking_test_view_n_mv
POSTHOOK: Output: default@masking_test_view_n_mv
PREHOOK: query: drop materialized view `masking_test_view_n_mv_2`
PREHOOK: type: DROP_MATERIALIZED_VIEW
PREHOOK: Input: default@masking_test_view_n_mv_2
PREHOOK: Output: default@masking_test_view_n_mv_2
POSTHOOK: query: drop materialized view `masking_test_view_n_mv_2`
POSTHOOK: type: DROP_MATERIALIZED_VIEW
POSTHOOK: Input: default@masking_test_view_n_mv_2
POSTHOOK: Output: default@masking_test_view_n_mv_2
PREHOOK: query: drop materialized view `masking_test_view_n_mv_3`
PREHOOK: type: DROP_MATERIALIZED_VIEW
PREHOOK: Input: default@masking_test_view_n_mv_3
PREHOOK: Output: default@masking_test_view_n_mv_3
POSTHOOK: query: drop materialized view `masking_test_view_n_mv_3`
POSTHOOK: type: DROP_MATERIALIZED_VIEW
POSTHOOK: Input: default@masking_test_view_n_mv_3
POSTHOOK: Output: default@masking_test_view_n_mv_3
PREHOOK: query: drop materialized view `masking_test_view_n_mv_4`
PREHOOK: type: DROP_MATERIALIZED_VIEW
PREHOOK: Input: default@masking_test_view_n_mv_4
PREHOOK: Output: default@masking_test_view_n_mv_4
POSTHOOK: query: drop materialized view `masking_test_view_n_mv_4`
POSTHOOK: type: DROP_MATERIALIZED_VIEW
POSTHOOK: Input: default@masking_test_view_n_mv_4
POSTHOOK: Output: default@masking_test_view_n_mv_4
PREHOOK: query: drop table `masking_test_n_mv`
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@masking_test_n_mv
PREHOOK: Output: default@masking_test_n_mv
POSTHOOK: query: drop table `masking_test_n_mv`
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@masking_test_n_mv
POSTHOOK: Output: default@masking_test_n_mv
PREHOOK: query: drop table `srcTnx`
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@srctnx
PREHOOK: Output: default@srctnx
POSTHOOK: query: drop table `srcTnx`
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@srctnx
POSTHOOK: Output: default@srctnx