blob: b8807a9cdd31f046443d400650f91fa3327c0946 [file] [log] [blame]
#!/usr/bin/env perl
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
###############################################################################
#
# Do
# egrep '^#|name.*=>' hcat.conf | egrep -v '^#!|egrep' | less
# to get an outline of this test conf file
#
# Has a couple of Hive set directives:
# set hive.exec.dynamic.partition.mode=nonstrict;
# set hive.exec.dynamic.partition=true;
$cfg = {
'driver' => 'HCat',
'groups' => [
# This first group should be moved to deployer ?
{
'name' => 'HCat_CreateTable',
'tests' => [
{
'num' => 1
,'hcat' => q\
drop table if exists hcat_createtable_1;
create table hcat_createtable_1(name string,
age int,
gpa double)
stored as textfile;
describe hcat_createtable_1;\
,'rc' => 0
,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double'
,'expected_err_regex' => 'OK(.*)OK(.*)OK'
},
{
'num' => 2
,'hcat' => q\
drop table if exists hcat_createtable_2;
create table hcat_createtable_2(name string,
age int,
gpa double) partitioned by (b string) stored as TEXTFILE;
describe extended hcat_createtable_2;
\,
,'rc' => 0
,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double'
},
{
'num' => 3
,'hcat' => q\
drop table if exists hcat_createtable_3;
create table if not exists hcat_createtable_3(name string, age int, gpa double) stored as textfile;
create table if not exists hcat_createtable_3(name string, age int, gpa double) stored as textfile;
describe hcat_createtable_3;
\,
,'rc' => 0
,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double'
,'expected_err_regex' => 'OK(.*)OK(.*)OK(.*)OK'
},
{
'num' => 4
,'hcat' => q\
drop table if exists hcat_createtable_4;
create table hcat_createtable_4(name string, age int, gpa double) partitioned by (grade string, section string) stored as textfile;
alter table hcat_createtable_4 add partition (grade='7',section='a');
alter table hcat_createtable_4 add partition (grade='8',section='b');
show partitions hcat_createtable_4;
\,
,'rc' => 0
,'expected_out_regex' => 'grade=7/section=a(\s)*grade=8/section=b'
,'expected_err_regex' => 'OK(.*)OK(.*)OK'
},
{
'num' => 5
,'hcat' => q\
drop table if exists hcat_createtable_5;
create table hcat_createtable_5(name string, age int, gpa double) stored as
inputformat 'org.apache.hadoop.hive.ql.io.RCFileInputFormat'
outputformat 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat';
describe extended hcat_createtable_5;
\,
,'rc' => 0
,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double(.*(\s))*inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat'
,'expected_err_regex' => 'OK(.*)OK(.*)OK'
},
{
'num' => 6
,'hcat' => q#
drop table if exists hcat_createtable_6;
create table hcat_createtable_6(a int, b array<int>, c map<string, string>)
row format delimited
fields terminated by '^'
collection items terminated by ','
map keys terminated by ':'
lines terminated by '\\n'
stored as textfile;
describe extended hcat_createtable_6;
#,
,'rc' => 0
,'expected_out_regex' => 'a int(\s)*b array<int>(\s)*c map<string,string>(.*(\s))*.*parameters:.*colelction.delim=,, mapkey.delim=:, serialization.format=|, line.delim=\n, field.delim=^'
,'expected_err_regex' => 'OK(.*)OK(.*)OK'
},
],
}, # end g
{
'name' => 'HCat_DropTable',
'tests' => [
{
'num' => 1
,'hcat' => q\
create table if not exists hcat_droptable_1(name string,
age int,
gpa double)
stored as textfile;
drop table hcat_droptable_1;
describe hcat_droptable_1;\
,'expected_err_regex' => 'Table not found'
},
{
'num' => 2
,'hcat' => q\
create table if not exists hcat_droptable_2(name string,
age int,
gpa double)
stored as textfile;
drop table if exists hcat_droptable_2;
describe hcat_droptable_2;\,
,'rc' => 17
,'expected_err_regex' => 'Table not found'
},
{
'num' => 3
,'hcat' => q\
drop table if exists hcat_drop_table_4;
dfs -mkdir :OUTPATHPARENT:/drop_table_ext;
dfs -cp :INPATH:/studentnull10k/ :OUTPATHPARENT:drop_table_ext;
\,
,'rc' => 0
},
{
'num' => 4
,'depends_on' => 'HCat_DropTable_3'
,'hcat' => q\
create external table hcat_drop_table_4(name string, age int, gpa double) stored as textfile location 'hdfs://:OUTPATHPARENT:drop_table_ext';
describe extended hcat_drop_table_4;
\,
,'rc' => 0
,'expected_out_regex' => '(.*)location:hdfs://(.*)drop_table_ext(.*)tableType:EXTERNAL_TABLE'
},
{
'num' => 5
,'depends_on' => 'HCat_DropTable_4'
,'hcat' => q\
drop table hcat_drop_table_4;
\,
,'rc' => 0
},
{
'num' => 6
,'depends_on' => 'HCat_DropTable_5'
,'hcat' => q\
dfs -ls :OUTPATHPARENT:drop_table_ext
\,
,'rc' => 0
,'expected_out_regex' => '(.*(\s))*.*drop_table_ext/studentnull10k'
},
],
}, # end g
{
'name' => 'HCat_AlterTable',
'tests' => [
{
'num' => 1
,'hcat' => q\
drop table if exists hcat_altertable_1;
create table hcat_altertable_1(name string, age int, gpa double) partitioned by (b string) stored as textfile;
alter table hcat_altertable_1 add partition (b='2010-10-10');
show partitions hcat_altertable_1;\
,'rc' => 0
,'expected_out_regex' => 'b=2010-10-10'
},
{
'num' => 2
,'hcat' => q\
alter table hcat_altertable_1 add partition (b='2010-10-11');
alter table hcat_altertable_1 drop partition (b='2010-10-10');
show partitions hcat_altertable_1;\,
,'rc' => 0
,'expected_out_regex' => 'b=2010-10-11'
,'not_expected_out_regex' => 'b=2010-10-10'
},
{
'num' => 3
,'hcat' => q\
alter table studenttab10k touch;
\
,'rc' => 0
},
{
'num' => 4
,'hcat' => q\
drop table if exists hcat_altertable_4;
create table hcat_altertable_4(name string, age int, gpa double) stored as textfile;
alter table hcat_altertable_4 set serdeproperties('xyz'='0');
\
,'rc' => 0
},
{
'num' => 5
,'hcat' => q\
drop table if exists hcat_altertable_5;
create table hcat_altertable_5(name string, age int, gpa double) stored as textfile;
alter table hcat_altertable_5 clustered by (age) into 1 buckets;
\
,'rc' => 0
},
{
'num' => 6
,'hcat' => q\
drop table if exists hcat_altertable_6;
create table hcat_altertable_6(name string, age int, gpa double) stored as textfile;
alter table hcat_altertable_6 set serde 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe';
\
,'rc' => 0
},
{
'num' => 7
,'hcat' => q\
drop table if exists hcat_altertable_7;
create table hcat_altertable_7(name string, age int, gpa double) partitioned by (grade string, section string) stored as textfile;
alter table hcat_altertable_7 add partition (grade='7',section='a');
show partitions hcat_altertable_7;\,
,'rc' => 0
,'expected_out_regex' => 'grade=7/section=a'
},
{
'num' => 8
,'depends_on' => 'HCat_AlterTable_7'
,'hcat' => q\
alter table hcat_altertable_7 add partition (grade='7',section='b');
alter table hcat_altertable_7 add partition (grade='7',section='c');
show partitions hcat_altertable_7;\,
,'rc' => 0
,'expected_out_regex' => 'grade=7/section=a(\s)*grade=7/section=b(\s)*grade=7/section=c'
},
{
'num' => 9
,'depends_on' => 'HCat_AlterTable_8'
,'hcat' => q\
alter table hcat_altertable_7 add if not exists partition (grade='7',section='a');\
,'rc' => 0
},
{
'num' => 10
,'depends_on' => 'HCat_AlterTable_9'
,'hcat' => q\
alter table hcat_altertable_7 add partition (grade='8',section='a') partition (grade='8',section='b');
show partitions hcat_altertable_7;\,
,'rc' => 0
,'expected_out_regex' => 'grade=7/section=a(\s)*grade=7/section=b(\s)*grade=7/section=c(\s)*grade=8/section=a(\s)*grade=8/section=b'
},
{
'num' => 11
,'depends_on' => 'HCat_AlterTable_10'
,'hcat' => q\
alter table hcat_altertable_7 drop partition (grade='8',section='a'), partition (grade='8',section='b');
show partitions hcat_altertable_7;\,
,'rc' => 0
,'expected_out_regex' => 'grade=7/section=a(\s)*grade=7/section=b(\s)*grade=7/section=c'
,'not_expected_out_regex' => 'grade=8/section=a(\s)*grade=8/section=b'
},
{
'num' => 12
,'depends_on' => 'HCat_AlterTable_11'
,'hcat' => q\
set hive.exec.drop.ignorenonexistent = false;
alter table hcat_altertable_7 drop if exists partition (grade='8',section='a'), partition(grade='7',section='d'), partition(grade='7',section='c');
set hive.exec.drop.ignorenonexistent = true;
show partitions hcat_altertable_7;\,
,'expected_out_regex' => 'grade=7/section=a(\s)*grade=7/section=b'
,'rc' => 0
},
{
'num' => 13
,'hcat' => q\
drop table if exists hcat_altertable_13;
create table hcat_altertable_13(a int, b string) partitioned by (c string, d string) stored as textfile;
alter table hcat_altertable_13 add partition (c='1',d='1');
alter table hcat_altertable_13 add columns (x string comment 'extra');
describe hcat_altertable_13;\,
,'rc' => 0
,'expected_out_regex' => 'a int(\s)*b string(\s)*x string extra(\s)*c string(\s)*d string'
},
{
'num' => 14
,'depends_on' => 'HCat_AlterTable_13'
,'hcat' => q\
alter table hcat_altertable_13 replace columns (k int, l string, m string);
describe hcat_altertable_13;\,
,'rc' => 0
,'expected_out_regex' => 'k int(\s)*l string(\s)*m string(\s)*c string(\s)*d string'
},
{
'num' => 15
,'hcat' => q\
drop table if exists hcat_altertable_15b;
drop table if exists hcat_altertable_15a;
create table hcat_altertable_15a(a int, b string) stored as textfile;
alter table hcat_altertable_15a rename to hcat_altertable_15b;
show tables like 'hcat_altertable_15.*';\,
,'rc' => 0
,'expected_out_regex' => 'hcat_altertable_15b'
,'not_expected_out_regex' => 'hcat_altertable_15a'
},
{
'num' => 16
,'hcat' => q\
drop table if exists hcat_altertable_16;
create table hcat_altertable_16(a int, b string) stored as textfile;
show table extended like hcat_altertable_16;
;\,
,'rc' => 0
,'expected_out_regex' => '((.*)\s)*location(.*)hcat_altertable_16'
},
{
'num' => 17
,'depends_on' => 'HCat_AlterTable_16'
,'hcat' => q\
alter table hcat_altertable_16 set location 'hdfs:///tmp/table_has_moved/';
show table extended like hcat_altertable_16;
;\,
,'rc' => 0
,'expected_out_regex' => 'location:hdfs:///tmp/table_has_moved'
,'not_expected_out_regex' => 'location(.*)hcat_altertable_16'
},
],
}, # end g
{
'name' => 'HCat_Database',
'tests' => [
{
'num' => 1
,'hcat' => q\
drop database if exists hcat_database_1;
create database hcat_database_1;
alter database hcat_database_1 set dbproperties ('new.property'='some props');
describe database hcat_database_1;
show databases;
use hcat_database_1;
use default;
drop database hcat_database_1;\
,'rc' => 0
},
],
}, # end g
{
'name' => 'HCat_View',
'tests' => [
{
'num' => 1
,'hcat' => q"
drop table if exists hcat_view_1;
drop view if exists hcat_view_1_1;
drop view if exists hcat_view_1_2;
create external table hcat_view_1 (name string, age int, gpa double) row format delimited fields terminated by '\t' stored as TEXTFILE location ':INPATH:/studenttab10k';
create view hcat_view_1_1 as select name, gpa, age from studenttab10k;
create view hcat_view_1_2 partitioned on (age) as select name, gpa, age from studenttab10k;
alter view hcat_view_1_1 set tblproperties('key'='value');
show tables;
describe hcat_view_1_1;
describe hcat_view_1_2;
drop view hcat_view_1_1;
drop view hcat_view_1_3;"
,'rc' => 0
},
{
'num' => 2
,'hcat' => q"
drop view if exists hcat_view_2_1;
create view hcat_view_2_1(name,age) as select name, age from studenttab10k;
describe extended hcat_view_2_1;
"
,'expected_out_regex' => 'name string.*(\s)age int(.*(\s))*viewOriginalText:select name, age from studenttab10k.*tableType:VIRTUAL_VIEW'
,'rc' => 0
},
],
}, # end g
{
'name' => 'HCat_Authorize',
'tests' => [
{
'num' => 1
,'hcat' => q\
create role role1;
grant drop, select on table studenttab10k to role role1 with grant option;
show grant role role1 on table studenttab10k;
revoke drop on table studenttab10k from role role1;
drop role role1;\
,'rc' => 0
},
{
'num' => 2
,'hcat' => q\
grant drop, select on table studenttab10k to user root;
show grant user root on table studenttab10k;
revoke drop, select on table studenttab10k from user root;\
,'rc' => 0
},
],
}, # end g
{
'name' => 'HCat_Index',
'tests' => [
{
'num' => 1
,'hcat' => q\
drop table if exists hcat_index_1;
create table hcat_index_1 (a string) partitioned by (b string) stored as TEXTFILE;
create index hcat_index_1_1 on table hcat_index_1(a) as 'compact' with deferred rebuild comment 'hcat test';
alter index hcat_index_1_1 on hcat_index_1 set idxproperties ('prop1'='val1');
show indexes on hcat_index_1;
create index hcat_index_1_2 on table hcat_index_1(a) as 'compact' with deferred rebuild comment 'hcat test second';
alter index hcat_index_1_2 on hcat_index_1 set idxproperties ('prop1'='val2');
show indexes on hcat_index_1;
drop index hcat_index_1_1 on hcat_index_1;
drop index hcat_index_1_2 on hcat_index_1;
;\
,'rc' => 0
},
{
'num' => 2
,'hcat' => q\
drop index if exists hcat_index_2_1 on hcat_index_2;
drop table if exists hcat_index_2;
create table hcat_index_2 (a string) partitioned by (b string) stored as TEXTFILE;
create index hcat_index_2_1 on table hcat_index_2(a) as 'compact' with deferred rebuild comment 'hcat compact idx';
alter index hcat_index_2_1 on hcat_index_2 set idxproperties ('prop1'='val1');
show indexes on hcat_index_2;
;\
,'rc' => 0
,'expected_out_regex' => 'hcat_index_2_1(\s)*hcat_index_2(\s)*a(\s)*default__hcat_index_2_hcat_index_2_1__(\s)*compact(\s)*hcat compact idx'
},
{
'num' => 3
,'hcat' => q\
drop index if exists hcat_index_3_1 on hcat_index_3;
drop table if exists hcat_index_3;
create table hcat_index_3 (a string) partitioned by (b string) stored as TEXTFILE;
create index hcat_index_3_1 on table hcat_index_3(a) as 'bitmap' with deferred rebuild comment 'hcat bitmap idx';
alter index hcat_index_3_1 on hcat_index_3 set idxproperties ('prop1'='val1');
show indexes on hcat_index_3;
;\
,'rc' => 0
,'expected_out_regex' => 'hcat_index_3_1(\s)*hcat_index_3(\s)*a(\s)*default__hcat_index_3_hcat_index_3_1__(\s)*bitmap(\s)*hcat bitmap idx'
},
],
}, # end g
{
'name' => 'HCat_Lock',
'tests' => [
{
'num' => 1,
'ignore' => 1, # this test need zookeeper setup, to ease the tests, ignore it by default, you may enable it if you have the right zookeeper setup
,'hcat' => q\
set hive.support.concurrency=true;
set hive.zookeeper.quorum=localhost;
lock table studenttab10k shared;
show locks;
unlock table studenttab10k;
;\
,'rc' => 0
},
],
}, # end g
{
'name' => 'HCat_ShowDes',
'tests' => [
{
'num' => 1
,'hcat' => q\
drop table if exists hcat_showdes.hcat_showdestable_1;
drop database if exists hcat_showdes;
drop database if exists t_showdes_abc;
drop table if exists hcat_showdestable_2;
create database hcat_showdes;
create database t_showdes_abc;
create table hcat_showdes.hcat_showdestable_1 (a int, b string);
create table hcat_showdestable_2 (a int comment 'first', b struct< foo : int, bar : struct< bar1:int, bar2:int > > comment 'second') partitioned by (c string);
alter table hcat_showdestable_2 add partition (c = '41');
alter table hcat_showdestable_2 add partition (c = '42');
;\
,'rc' => 0
},
{
'num' => 2
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
show databases;\
,'rc' => 0
,'expected_out_regex' => 'hcat_showdes'
},
{
'num' => 3
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
show databases like '.*abc';\
,'rc' => 0
,'expected_out_regex' => 't_showdes_abc'
,'not_expected_out_regex' => 'hcat_showdes'
},
{
'num' => 4
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
show tables;\,
,'rc' => 0
,'expected_out_regex' => 'hcat_showdestable_2'
},
{
'num' => 5
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
show tables in hcat_showdes;\,
,'rc' => 0
,'expected_out_regex' => 'hcat_showdestable_1'
,'not_expected_out_regex' => 'hcat_createtable_1'
},
{
'num' => 6
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
show tables in hcat_showdes like '.*table_1';\,
,'rc' => 0
,'expected_out_regex' => 'hcat_showdestable_1'
},
{
'num' => 7
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
show table extended like hcat_showdestable_2;\,
,'rc' => 0
,'expected_out_regex' => 'tableName(.*(\s))*owner(.*(\s))*location(.*(\s))*columns(.*(\s))*totalNumberFiles(.*(\s))*totalFileSize(.*(\s))*maxFileSize(.*(\s))*minFileSize(.*(\s))*lastAccessTime(.*(\s))*lastUpdateTime(.*)'
},
{
'num' => 8
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
show table extended like hcat_showdestable_2 partition(c='42');\,
,'rc' => 0
,'expected_out_regex' => 'tableName(.*(\s))*owner(.*(\s))*location(.*(\s))*columns(.*(\s))*totalNumberFiles(.*(\s))*totalFileSize(.*(\s))*maxFileSize(.*(\s))*minFileSize(.*(\s))*lastAccessTime(.*(\s))*lastUpdateTime(.*)'
},
{
'num' => 9
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
describe extended hcat_showdestable_2 partition(c='42');\
,'rc' => 0
,'expected_out_regex' => 'a int first(\s)*b struct<foo:int,bar:struct<bar1:int,bar2:int>> second(\s)*c string(.*(\s))*Detailed Partition Information(\s)*Partition(.*)values:\[42\]'
},
{
'num' => 10
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
describe extended hcat_showdestable_2.b;\
,'rc' => 0
,'expected_out_regex' => 'foo int from deserializer(\s)*bar struct<bar1:int,bar2:int> from deserializer'
},
{
'num' => 11
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
describe extended hcat_showdestable_2.b.foo;\
,'rc' => 0
,'expected_out_regex' => 'foo int from deserializer'
,'not_expected_out_regex' => 'bar'
},
{
'num' => 12
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
describe extended hcat_showdestable_2.b.bar;\
,'rc' => 0
,'expected_out_regex' => 'bar1 int from deserializer(\s)*bar2 int from deserializer'
},
{
'num' => 13
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
describe extended hcat_showdestable_2.b.bar.bar1;\
,'rc' => 0
,'expected_out_regex' => 'bar1 int from deserializer'
,'not_expected_out_regex' => 'bar2'
},
{
'num' => 14
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
show partitions hcat_showdestable_2;\
,'rc' => 0
,'expected_out_regex' => 'c=41(\s)*c=42'
},
{
'num' => 15
,'depends_on' => 'HCat_ShowDes_1'
,'hcat' => q\
show partitions hcat_showdestable_2 partition(c='41');\
,'rc' => 0
,'expected_out_regex' => 'c=41'
,'not_expected_out_regex' => 'c=42'
},
],
}, # end g
{
'name' => 'HCat_Misc',
'tests' => [
{
'num' => 1
,'hcat' => q\
show databases;\
,'rc' => 0
,'expected_out_regex' => 'default'
},
{
'num' => 2
,'hcat' => q\
show tables;\,
,'rc' => 0
,'expected_out_regex' => 'hcat_createtable_1'
},
{
'num' => 3
,'hcat' => q\
show tables in default;\,
,'rc' => 0
},
{
'num' => 4
,'hcat' => q\
explain select * from studenttab10k;\,
,'rc' => 0
},
{
'num' => 5
,'hcat' => q\
show functions;\,
,'rc' => 0
},
{
'num' => 6
,'hcat' => q\
describe function xpath_int;\,
,'rc' => 0
},
{
'num' => 7
,'hcat' => q\
dfs -ls;\,
,'rc' => 0
},
],
}, # end g
{
'name' => 'HCat_Negative',
'tests' => [
{
'num' => 1
,'hcat' => q\
create table hcat_negative_1 as select * from studenttab10k;
;\
,'expected_err_regex' => 'Operation not supported'
},
{
'num' => 2
,'hcat' => q\
alter index test111 on hcat_test2 rebuild;
;\
,'expected_err_regex' => 'Operation not supported'
},
{
'num' => 3
,'hcat' => q\
alter table studentparttab30k PARTITION (ds='1') CONCATENATE;
;\
,'expected_err_regex' => 'Operation not supported'
},
{
'num' => 4
,'hcat' => q\
alter table studentparttab30k archive PARTITION (ds='20110924');
;\
,'expected_err_regex' => 'Operation not supported'
},
{
'num' => 5
,'hcat' => q\
analyze table studenttab10k compute statistics;
;\
,'expected_err_regex' => 'Operation not supported'
},
{
'num' => 6
,'hcat' => q\
export table studenttab10k to '111';
;\
,'expected_err_regex' => 'Operation not supported'
},
{
'num' => 7
,'hcat' => q\
import from '111';
;\
,'expected_err_regex' => 'Operation not supported'
},
],
}, # end g
]
}