set hive.mapred.mode=nonstrict;
set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;

set hive.vectorized.execution.enabled=true;
-- enable ppd
set hive.optimize.index.filter=true;

set hive.explain.user=false;

-- needed for TestCliDriver but not TestMiniTezCliDriver
-- set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;

-- attempts to get compaction to run - see below
-- set yarn.scheduler.maximum-allocation-mb=2024;
-- set hive.tez.container.size=500;
-- set mapreduce.map.memory.mb=500;
-- set mapreduce.reduce.memory.mb=500;
-- set mapred.job.map.memory.mb=500;
-- set mapred.job.reduce.memory.mb=500;


CREATE TEMPORARY FUNCTION runWorker AS 'org.apache.hadoop.hive.ql.udf.UDFRunWorker';
create table mydual(a int);
insert into mydual values(1);

CREATE TABLE over10k_n2(t tinyint,
           si smallint,
           i int,
           b bigint,
           f float,
           d double,
           bo boolean,
           s string,
           ts timestamp,
           `dec` decimal(4,2),
           bin binary)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
STORED AS TEXTFILE;

--oddly this has 9999 rows not > 10K
LOAD DATA LOCAL INPATH '../../data/files/over1k' OVERWRITE INTO TABLE over10k_n2;

CREATE TABLE over10k_orc_bucketed(t tinyint,
           si smallint,
           i int,
           b bigint,
           f float,
           d double,
           bo boolean,
           s string,
           ts timestamp,
           `dec` decimal(4,2),
           bin binary) CLUSTERED BY(si) INTO 4 BUCKETS STORED AS ORC;

-- this produces about 250 distinct values across all 4 equivalence classes
select distinct si, si%4 from over10k_n2 order by si;

-- explain insert into over10k_orc_bucketed select * from over10k_n2;
insert into over10k_orc_bucketed select * from over10k_n2;

dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/over10k_orc_bucketed;
-- create copy_N files
insert into over10k_orc_bucketed select * from over10k_n2;

-- this output of this is masked in .out - it is visible in .orig
dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/over10k_orc_bucketed;

--this actually shows the data files in the .out on Tez but not LLAP
select distinct 7 as seven, INPUT__FILE__NAME from over10k_orc_bucketed;

-- convert table to acid
alter table over10k_orc_bucketed set TBLPROPERTIES ('transactional'='true');

-- this should vectorize (and push predicate to storage: filterExpr in TableScan )
--             Execution mode: vectorized, llap
--             LLAP IO: may be used (ACID table)
explain select t, si, i from over10k_orc_bucketed where b = 4294967363 and t < 100 order by t, si, i;
select t, si, i from over10k_orc_bucketed where b = 4294967363 and t < 100 order by  t, si, i;

-- this should vectorize (and push predicate to storage: filterExpr in TableScan )
--             Execution mode: vectorized, llap
--             LLAP IO: may be used (ACID table)
explain select ROW__ID, t, si, i from over10k_orc_bucketed where b = 4294967363 and t < 100 order by ROW__ID;
-- HIVE-17943
select ROW__ID, t, si, i from over10k_orc_bucketed where b = 4294967363 and t < 100 order by ROW__ID;

-- this should vectorize (and push predicate to storage: filterExpr in TableScan )
--            Execution mode: vectorized, llap
--            LLAP IO: may be used (ACID table)
explain update over10k_orc_bucketed set i = 0 where b = 4294967363 and t < 100;
update over10k_orc_bucketed set i = 0 where b = 4294967363 and t < 100;

-- this should produce the same result (data) as previous time this exact query ran
-- ROW__ID will be different (same bucketProperty)
select ROW__ID, t, si, i from over10k_orc_bucketed where b = 4294967363 and t < 100 order by ROW__ID;

-- The idea below was to do check sum queries to ensure that ROW__IDs are unique
-- to run Compaction and to check that ROW__IDs are the same before and after compaction (for rows
-- w/o applicable delete events)
-- Everything below is commented out because it doesn't work
-- group by ROW__ID produces wrong results
-- compactor doesn't run - see error below...

-- this doesn't vectorize but use llap which is perhaps the problem if it's using cache
-- use explain VECTORIZATION DETAIL to see
-- notVectorizedReason: Key expression for GROUPBY operator: Vectorizing complex type STRUCT not supported
explain select ROW__ID, count(*) from over10k_orc_bucketed group by ROW__ID having count(*) > 1;

-- this test that there are no duplicate ROW__IDs so should produce no output
-- on LLAP this produces "NULL, 6"; on tez it produces nothing: HIVE-17921
-- this makes sure that the same code is running on the Ptest and on localhost. The target is:
-- Original split count is 11 grouped split count is 1, for bucket: 1
set tez.grouping.split-count=1;

select ROW__ID, count(*) from over10k_orc_bucketed group by ROW__ID having count(*) > 1;
-- this produces nothing (as it should)
select ROW__ID, * from over10k_orc_bucketed where ROW__ID is null;

-- schedule compactor
-- alter table over10k_orc_bucketed compact 'major' WITH OVERWRITE TBLPROPERTIES ("compactor.mapreduce.map.memory.mb"="500","compactor.hive.tez.container.size"="500");;


-- run compactor - this currently fails with
-- Invalid resource request, requested memory < 0, or requested memory > max configured, requestedMemory=1536, maxMemory=512
-- HIVE-17922
-- select runWorker() from mydual;

-- show compactions;

-- this should produce the same result (data+ ROW__ID) as previous time this exact query ran
-- select ROW__ID, t, si, i from over10k_orc_bucketed where b = 4294967363 and t < 100 order by ROW__ID;

-- this test that there are no duplicate ROW__IDs so should produce no output
-- select ROW__ID, count(*) from over10k_orc_bucketed group by ROW__ID having count(*) > 1;

-- select ROW__ID, * from over10k_orc_bucketed where ROW__ID is null;

-- TODO: Remove this line after fixing HIVE-24351
set hive.vectorized.execution.enabled=false;

CREATE TABLE over10k_orc STORED AS ORC as select * from over10k_n2 where t between 3 and 4;

-- Make sure there are multiple original files
INSERT INTO over10k_orc select * from over10k_n2 where t between 3 and 4;

-- TODO: Remove this line after fixing HIVE-24351
set hive.vectorized.execution.enabled=true;

alter table over10k_orc set TBLPROPERTIES ('transactional'='true');

-- row id is projected but there are no delete deltas
set hive.exec.orc.split.strategy=ETL;
select o1.ROW__ID r1, o1.* from over10k_orc o1 join over10k_orc o2
on o1.ROW__ID.rowid == o2.ROW__ID.rowid and o1.ROW__ID.writeid == o2.ROW__ID.writeid and o1.ROW__ID.bucketid == o2.ROW__ID.bucketid;

set hive.exec.orc.split.strategy=BI;
select o1.ROW__ID r1, o1.* from over10k_orc o1 join over10k_orc o2
on o1.ROW__ID.rowid == o2.ROW__ID.rowid
and o1.ROW__ID.writeid == o2.ROW__ID.writeid
and o1.ROW__ID.bucketid == o2.ROW__ID.bucketid;

delete from over10k_orc where t = 3;

-- row id not projected but has delete deltas
set hive.exec.orc.split.strategy=ETL;
select t, count(*) from over10k_orc
group by t;

set hive.exec.orc.split.strategy=BI;
select t, count(*) from over10k_orc
group by t;

select oo.ROW__ID.writeId, oo.ROW__IS__DELETED, oo.* from over10k_orc('acid.fetch.deleted.rows'='true') oo order by si;