[SDV] Enhancements in SDV Test framework
Problem: Migrated sdv framework to run with Spark-2.3. Some testcases were failing because hive
deletes the csv file after loading, hence next load was failing.
Solution: So, fixed the same by adding CSV files for hive tables and fixed compile issues for different spark
version.
This closes #3397
diff --git a/integration/spark-common-cluster-test/pom.xml b/integration/spark-common-cluster-test/pom.xml
index 9fe89cc..e8cece2 100644
--- a/integration/spark-common-cluster-test/pom.xml
+++ b/integration/spark-common-cluster-test/pom.xml
@@ -69,6 +69,18 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.9.9.Final</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.17.Final</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark2</artifactId>
<version>${project.version}</version>
@@ -180,7 +192,7 @@
<spark.master.url>${spark.master.url}</spark.master.url>
<hdfs.url>${hdfs.url}</hdfs.url>
<presto.jdbc.url>${presto.jdbc.url}</presto.jdbc.url>
- <spark.hadoop.hive.metastore.uris>${spark.hadoop.hive.metastore.uris}</spark.hadoop.hive.metastore.uris>
+ <!--<spark.hadoop.hive.metastore.uris>${spark.hadoop.hive.metastore.uris}</spark.hadoop.hive.metastore.uris>-->
<spark.carbon.hive.schema.store>${carbon.hive.based.metastore}</spark.carbon.hive.schema.store>
</systemProperties>
</configuration>
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BadRecordTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BadRecordTestCase.scala
index 3b57ff0..a4cd333 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BadRecordTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BadRecordTestCase.scala
@@ -117,7 +117,7 @@
sql(s"""drop table if exists hivetable7""").collect
sql(s"""CREATE TABLE badrecordtest7 (ID int,CUST_ID int,cust_name string) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""CREATE TABLE hivetable7 (ID int,CUST_ID int,cust_name string) row format delimited fields terminated by ','""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/badrecord/test2.csv' into table hivetable7""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/badrecord/test2_hive1.csv' into table hivetable7""").collect
sql(s"""insert into table badrecordtest7 select * from hivetable7""").collect
checkAnswer(s"""select count(*) from badrecordtest7""",
Seq(Row(3)), "BadRecordTestCase-BadRecords-001_PTS008_TC001")
@@ -132,7 +132,7 @@
sql(s"""drop table if exists hivetable9""").collect
sql(s"""CREATE TABLE badrecordTest9 (ID int,CUST_ID int,cust_name string) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""CREATE TABLE hivetable9 (ID int,CUST_ID int,cust_name string) row format delimited fields terminated by ','""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/badrecord/test2.csv' into table hivetable9""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/badrecord/test2_hive2.csv' into table hivetable9""").collect
sql(s"""insert into table badrecordTest9 select * from hivetable9""").collect
checkAnswer(s"""select count(*) from badrecordTest9""",
Seq(Row(3)), "BadRecordTestCase-BadRecords-001_PTS015_TC001")
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad3TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad3TestCase.scala
index 5aaeffe..ed803ca 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad3TestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad3TestCase.scala
@@ -170,7 +170,7 @@
sql(s"""drop TABLE if exists uniqdata_h""").collect
sql(s"""drop TABLE if exists uniqdata_c""").collect
sql(s"""CREATE TABLE uniqdata_h (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','""").collect
- sql(s"""load data inpath '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata_h""").collect
+ sql(s"""load data inpath '$resourcesPath/Data/uniqdata/7000_UniqData_hive2.csv' into table uniqdata_h""").collect
sql(s"""CREATE TABLE uniqdata_c (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect
sql(s"""insert into uniqdata_c select * from uniqdata_h""")
sql(s"""drop table if exists uniqdata_h""").collect
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BucketingTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BucketingTestCase.scala
index 501b089..a6fcf62 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BucketingTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BucketingTestCase.scala
@@ -22,7 +22,7 @@
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.common.util._
-import org.apache.spark.sql.execution.exchange.ShuffleExchange
+import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
import org.scalatest.BeforeAndAfterAll
class BucketingTestCase extends QueryTest with BeforeAndAfterAll {
@@ -100,7 +100,7 @@
""".stripMargin).queryExecution.executedPlan
var shuffleExists = false
plan.collect {
- case s: ShuffleExchange => shuffleExists = true
+ case s: ShuffleExchangeExec => shuffleExists = true
}
assert(!shuffleExists, "shuffle should not exist on bucket column join")
}
@@ -121,7 +121,7 @@
var shuffleExists = false
plan.collect {
- case s: ShuffleExchange => shuffleExists = true
+ case s: ShuffleExchangeExec => shuffleExists = true
}
assert(shuffleExists, "shuffle should exist on non-bucket column join")
}
@@ -145,7 +145,7 @@
""".stripMargin).queryExecution.executedPlan
var shuffleExists = false
plan.collect {
- case s: ShuffleExchange => shuffleExists = true
+ case s: ShuffleExchangeExec => shuffleExists = true
}
assert(!shuffleExists, "shuffle should not exist on bucket tables")
}
@@ -167,7 +167,7 @@
""".stripMargin).queryExecution.executedPlan
var shuffleExists = false
plan.collect {
- case s: ShuffleExchange => shuffleExists = true
+ case s: ShuffleExchangeExec => shuffleExists = true
}
assert(!shuffleExists, "shuffle should not exist on bucket tables")
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
index f4d7034..c344f81 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
@@ -81,9 +81,9 @@
//NA
test("IUD-01-01-01_001-001", Include) {
sql("create table T_Hive1(Active_status BOOLEAN, Item_type_cd TINYINT, Qty_day_avg SMALLINT, Qty_total INT, Sell_price BIGINT, Sell_pricep FLOAT, Discount_price DOUBLE , Profit DECIMAL(3,2), Item_code STRING, Item_name VARCHAR(50), Outlet_name CHAR(100), Update_time TIMESTAMP, Create_date DATE) row format delimited fields terminated by ',' collection items terminated by '$'")
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' overwrite into table T_Hive1""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1_hive10.csv' overwrite into table T_Hive1""").collect
sql("create table T_Hive6(Item_code STRING, Sub_item_cd ARRAY<string>)row format delimited fields terminated by ',' collection items terminated by '$'")
- sql(s"""load data inpath '$resourcesPath/Data/InsertData/T_Hive1.csv' overwrite into table T_Hive6""").collect
+ sql(s"""load data inpath '$resourcesPath/Data/InsertData/T_Hive1_hive11.csv' overwrite into table T_Hive6""").collect
sql(s"""create table t_carbn02(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""insert into t_carbn02 select * from default.t_carbn01b limit 4""").collect
checkAnswer(s"""select count(*) from t_carbn01b""",
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
index 172cb64..ff8e193 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
@@ -295,7 +295,7 @@
sql(s"""drop table IF EXISTS T_Carbn01""").collect
sql(s"""create table T_Carbn01(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""create table T_Hive1(Active_status String, Item_type_cd INT, Qty_day_avg INT, Qty_total INT, Sell_price BIGINT, Sell_pricep DOUBLE, Discount_price DOUBLE , Profit DECIMAL(3,2), Item_code STRING, Item_name VARCHAR(50), Outlet_name CHAR(100), Update_time TIMESTAMP, Create_date String)row format delimited fields terminated by ',' collection items terminated by '$DOLLAR'""").collect
- sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' overwrite into table T_Hive1""").collect
+ sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive1_hive1.csv' overwrite into table T_Hive1""").collect
sql(s"""insert into T_Carbn01 select * from T_Hive1""").collect
checkAnswer(s"""select active_status,item_type_cd,qty_day_avg,qty_total,sell_price,sell_pricep,discount_price, profit,item_code,item_name from T_Carbn01 order by update_time""",
s"""select active_status,item_type_cd,qty_day_avg,qty_total,sell_price,sell_pricep,discount_price, profit,item_code,item_name from T_Hive1 order by update_time""", "DataLoadingTestCase-Insert_Func_005")
@@ -347,7 +347,7 @@
test("Insert_Func_008", Include) {
sql(s"""drop table IF EXISTS t_hive2""").collect
sql(s"""create table T_Hive2(Active_status String, Item_type_cd INT, Qty_day_avg SMALLINT, Qty_total INT, Sell_price BIGINT, Sell_pricep DOUBLE, Discount_price DOUBLE , Profit DECIMAL(3,2), Item_code STRING, Item_name VARCHAR(50), Outlet_name CHAR(100), Update_time TIMESTAMP, Create_date String,Profit_perc DECIMAL(4,3),name string)row format delimited fields terminated by ',' collection items terminated by '$DOLLAR'""").collect
- sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive2.csv' overwrite into table T_Hive2""").collect
+ sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive2_hive2.csv' overwrite into table T_Hive2""").collect
sql(s"""create table T_Carbn01(Active_status String,Item_type_cd INT,Qty_day_avg SMALLINT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""insert into T_Carbn01 select * from T_Hive2""").collect
checkAnswer(s"""select active_status,item_type_cd,qty_day_avg,qty_total,sell_price,sell_pricep,discount_price, profit,item_code,item_name from T_Carbn01 order by update_time""",
@@ -427,8 +427,8 @@
sql(s"""drop table IF EXISTS T_Carbn01""").collect
sql(s"""create table T_Hive4(Item_code STRING, Item_name VARCHAR(50))row format delimited fields terminated by ',' collection items terminated by '$DOLLAR'""").collect
sql(s"""create table T_Hive5(Item_code STRING, Profit DECIMAL(3,2))row format delimited fields terminated by ',' collection items terminated by '$DOLLAR'""").collect
- sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive4.csv' overwrite into table T_Hive4""").collect
- sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive5.csv' overwrite into table T_Hive5""").collect
+ sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive4_hive4.csv' overwrite into table T_Hive4""").collect
+ sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive5_hive5.csv' overwrite into table T_Hive5""").collect
sql(s"""create table T_Carbn01(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""insert into T_carbn01 select x.Active_status,x.Item_type_cd,x.Qty_day_avg,x.Qty_total,x.Sell_price,x.Sell_pricep,x.Discount_price,z.Profit,x.Item_code,y.Item_name,x.Outlet_name,x.Update_time,x.Create_date from T_Hive1 x,T_Hive4 y, T_Hive5 z where x.Item_code = y.Item_code and x.Item_code = z.Item_code""").collect
checkAnswer(s"""select active_status,item_type_cd,qty_day_avg,qty_total,sell_price,sell_pricep,discount_price, profit,item_code,item_name from T_Carbn01 order by update_time""",
@@ -442,8 +442,8 @@
sql(s"""drop table IF EXISTS t_hive7""").collect
sql(s"""drop table IF EXISTS T_Carbn01""").collect
sql(s"""create table T_Hive7(Active_status1 BOOLEAN, Item_type_cd1 TINYINT, Qty_day_avg1 SMALLINT, Qty_total1 INT, Sell_price1 BIGINT, Sell_pricep1 FLOAT, Discount_price1 DOUBLE , Profit1 DECIMAL(3,2), Item_code1 STRING, Item_name1 VARCHAR(50), Outlet_name1 CHAR(100), Update_time TIMESTAMP, Create_date DATE)row format delimited fields terminated by ',' collection items terminated by '$DOLLAR'""").collect
- sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' overwrite into table T_Hive7""").collect
- sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' into table T_Hive7""").collect
+ sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive1_hive0.csv' overwrite into table T_Hive7""").collect
+ sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive1_hive3.csv' into table T_Hive7""").collect
sql(s"""create table T_Carbn01(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""insert into T_Carbn01 select * from T_Hive7""").collect
checkAnswer(s"""select active_status,item_type_cd,qty_day_avg,qty_total,sell_price,sell_pricep,discount_price, profit,item_code,item_name from T_Carbn01 order by update_time""",
@@ -484,7 +484,7 @@
sql(s"""create table T_Carbn01(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' INTO table T_Carbn01 options ('DELIMITER'=',', 'QUOTECHAR'='\','FILEHEADER'='Active_status,Item_type_cd,Qty_day_avg,Qty_total,Sell_price,Sell_pricep,Discount_price,Profit,Item_code,Item_name,Outlet_name,Update_time,Create_date')""").collect
sql(s"""create table T_Hive_1(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String) row format delimited fields terminated by ',' collection items terminated by '\n'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' INTO table T_Hive_1""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1_hive6.csv' INTO table T_Hive_1""").collect
sql(s"""insert into T_Carbn01 select * from T_Hive_1""").collect
checkAnswer(s"""select count(*) from T_Carbn01""",
Seq(Row(20)), "DataLoadingTestCase-Insert_Func_027")
@@ -520,7 +520,7 @@
sql(s"""drop table IF EXISTS t_hive5""").collect
sql(s"""drop table IF EXISTS T_Carbn01""").collect
sql(s"""create table T_Hive5(Item_code STRING, Profit DECIMAL(3,2))row format delimited fields terminated by ',' collection items terminated by '$DOLLAR'""").collect
- sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive5.csv' overwrite into table T_Hive5""").collect
+ sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive5_hive7.csv' overwrite into table T_Hive5""").collect
sql(s"""create table T_Carbn01(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""insert into T_Carbn01 select * from T_Hive1 x where exists (select * from T_Hive5 y where x.Item_code= y.Item_code) """).collect
checkAnswer(s"""select active_status,item_type_cd,qty_day_avg,qty_total,sell_price,sell_pricep,discount_price, profit,item_code,item_name from T_Carbn01 order by update_time""",
@@ -534,7 +534,7 @@
sql(s"""drop table if exists t_hive4""").collect
sql(s"""drop table IF EXISTS T_Carbn01""").collect
sql(s"""create table T_Hive4(Item_code STRING, Item_name VARCHAR(50))row format delimited fields terminated by ','""").collect
- sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive4.csv' overwrite into table T_Hive4""").collect
+ sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive4_hive8.csv' overwrite into table T_Hive4""").collect
sql(s"""create table T_Carbn01(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""insert into T_Carbn01 select * from T_Hive1 a where a.Item_code in (select b.item_code from T_Hive4 b)""").collect
checkAnswer(s"""select active_status,item_type_cd,qty_day_avg,qty_total,sell_price,sell_pricep,discount_price, profit,item_code,item_name from T_Carbn01 order by update_time""",
@@ -657,7 +657,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' INTO table T_Carbn01 options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='Active_status,Item_type_cd,Qty_day_avg,Qty_total,Sell_price,Sell_pricep,Discount_price,Profit,Item_code,Item_name,Outlet_name,Update_time,Create_date')""").collect
sql(s"""drop table if exists T_Hive1""").collect
sql(s"""create table T_Hive1(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String) row format delimited fields terminated by ',' collection items terminated by '\n'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' INTO table T_Hive1""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1_hive9.csv' INTO table T_Hive1""").collect
sql(s"""Insert into T_Carbn01 select * from T_Hive1""").collect
sql(s"""delete from table T_Carbn01 where segment.id in (0)""").collect
sql(s"""select count(*) from T_Carbn01""").collect
@@ -696,7 +696,7 @@
sql(s"""create table T_Hive14(Item_code STRING, Profit DECIMAL(3,2)) partitioned by (Qty_total INT, Item_type_cd TINYINT) row format delimited fields terminated by ',' collection items terminated by '$DOLLAR'""").collect
sql(s"""drop table IF EXISTS T_Carbn014""").collect
sql(s"""create table T_Carbn014(Item_code STRING, Profit DECIMAL(3,2), Qty_total INT, Item_type_cd INT) STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive14.csv' overwrite into table T_Hive14 partition(Qty_total=100, Item_type_cd=2)""").collect
+ sql(s"""load data INPATH '$resourcesPath/Data/InsertData/T_Hive14_hive1.csv' overwrite into table T_Hive14 partition(Qty_total=100, Item_type_cd=2)""").collect
sql(s"""insert into T_carbn014 select * from T_Hive14 where Qty_total =100""").collect
checkAnswer(s"""select item_code, profit from T_Carbn014 order by item_code, profit""",
Seq(Row("BE3423ee",4.99),Row("BE3423ee",4.99),Row("SE3423ee",4.99),Row("SE3423ee",4.99),Row("SE3423ee",4.99),Row("SE3423ee",4.99)), "DataLoadingTestCase-Insert_Func_109")
@@ -790,7 +790,7 @@
//Check for insert into carbon table with select from Hive table where only Measures columns are present.
test("Insert_Func_066", Include) {
sql(s"""create table Measures_Dataload_H (Item_code STRING, Qty int)row format delimited fields terminated by ',' LINES TERMINATED BY '\n'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/vardhandaterestruct.csv' INTO TABLE Measures_Dataload_H""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/vardhandaterestruct_hive1.csv' INTO TABLE Measures_Dataload_H""").collect
sql(s"""create table Measures_Dataload_C (Item_code STRING, Qty int)stored by 'org.apache.carbondata.format'""").collect
sql(s"""insert into Measures_Dataload_C select * from Measures_Dataload_H""").collect
checkAnswer(s"""select count(*) from Measures_Dataload_C""",
@@ -819,8 +819,8 @@
//Check for insert into carbon table with select statement having logical operators
test("Insert_Func_043", Include) {
sql(s"""create table Logical_Dataload_H (Item_code STRING, Qty int)row format delimited fields terminated by ',' LINES TERMINATED BY '\n'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures.csv' INTO TABLE Logical_Dataload_H""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures.csv' INTO TABLE Logical_Dataload_H""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures_hive2.csv' INTO TABLE Logical_Dataload_H""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures_hive3.csv' INTO TABLE Logical_Dataload_H""").collect
sql(s"""create table Logical_Dataload_C (Item_code STRING, Qty int)stored by 'org.apache.carbondata.format'""").collect
sql(s"""insert into Logical_Dataload_C select * from Logical_Dataload_H where Item_Code != 'D' and Qty < 40""").collect
checkAnswer(s"""select count(*) from Logical_Dataload_C""",
@@ -833,7 +833,7 @@
//Check that select query fetches the correct data after doing insert and load .
test("Insert_Func_073", Include) {
sql(s"""create table Dataload_H (Item_code STRING, Qty int)row format delimited fields terminated by ',' LINES TERMINATED BY '\n'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures.csv' INTO TABLE Dataload_H""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures_hive4.csv' INTO TABLE Dataload_H""").collect
sql(s"""create table Dataload_C (Item_code STRING, Qty int)stored by 'org.apache.carbondata.format'""").collect
@@ -849,7 +849,7 @@
test("Insert_Func_111", Include) {
sql(s"""create database insert1""").collect
sql(s"""create table insert1.DiffDB_Dataload_H(Item_code STRING, Qty int)row format delimited fields terminated by ',' LINES TERMINATED BY '\n'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures.csv' INTO TABLE insert1.DiffDB_Dataload_H""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures_hive5.csv' INTO TABLE insert1.DiffDB_Dataload_H""").collect
sql(s"""create database insert2""").collect
sql(s"""create table insert2.DiffDB_Dataload_C(Item_code STRING, Qty int)stored by 'org.apache.carbondata.format'""").collect
sql(s"""insert into insert2.DiffDB_Dataload_C select * from insert1.DiffDB_Dataload_H""").collect
@@ -863,7 +863,7 @@
//Check for insert into carbon table with select from Hive table where only Dimension columns are present.
ignore("Insert_Func_065", Include) {
sql(s"""create table Dimension_Dataload_H (Item_code STRING)row format delimited fields terminated by ',' LINES TERMINATED BY '\n'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures.csv' INTO TABLE Dimension_Dataload_H""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures_hive1.csv' INTO TABLE Dimension_Dataload_H""").collect
sql(s"""create table Dimension_Dataload_C (Item_code STRING)stored by 'org.apache.carbondata.format'""").collect
sql(s"""insert into Dimension_Dataload_C select * from Dimension_Dataload_H""").collect
checkAnswer(s"""select count(*) from Dimension_Dataload_C""",
@@ -1418,18 +1418,16 @@
test("Insert_Func_023_01", Include) {
dropTable("Norecords_Dataload_C")
dropTable("Norecords_Dataload_H")
- intercept[Exception] {
- sql(s"""create table Norecords_Dataload_H (Item_code STRING, Qty int)stored by 'org.apache.carbondata.format'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures.csv' INTO TABLE Norecords_Dataload_H OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='Item_code,Qty')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures.csv' INTO TABLE Norecords_Dataload_H OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='Item_code,Qty')""").collect
- sql(s"""create table Norecords_Dataload_C (Item_code STRING, Qty int)stored by 'org.apache.carbondata.format'""").collect
- sql(s"""delete from table Norecords_Dataload_H where segment.id in (0,1)""").collect
- sql(s"""insert into Norecords_Dataload_C select * from Norecords_Dataload_H""").collect
- checkAnswer(s"""select count(*) from Norecords_Dataload_C""",
- Seq(Row(0)), "DataLoadingTestCase-Insert_Func_023_01")
- }
- dropTable("Norecords_Dataload_C")
- dropTable("Norecords_Dataload_H")
+ sql(s"""create table Norecords_Dataload_H (Item_code STRING, Qty int)stored by 'org.apache.carbondata.format'""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures.csv' INTO TABLE Norecords_Dataload_H OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='Item_code,Qty')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/Measures.csv' INTO TABLE Norecords_Dataload_H OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='Item_code,Qty')""").collect
+ sql(s"""create table Norecords_Dataload_C (Item_code STRING, Qty int)stored by 'org.apache.carbondata.format'""").collect
+ sql(s"""delete from table Norecords_Dataload_H where segment.id in (0,1)""").collect
+ sql(s"""insert into Norecords_Dataload_C select * from Norecords_Dataload_H""").collect
+ checkAnswer(s"""select count(*) from Norecords_Dataload_C""",
+ Seq(Row(0)), "DataLoadingTestCase-Insert_Func_023_01")
+ dropTable("Norecords_Dataload_C")
+ dropTable("Norecords_Dataload_H")
}
@@ -1458,7 +1456,7 @@
sql(s"""drop database if exists insertInto CASCADE""").collect
sql(s"""create database insertInto""").collect
sql(s"""create table insertInto.Norecords_Dataload_H (Item_code STRING, Qty int)row format delimited fields terminated by ',' LINES TERMINATED BY '\n'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/vardhandaterestruct.csv' INTO TABLE insertInto.Norecords_Dataload_H""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/vardhandaterestruct_hive2.csv' INTO TABLE insertInto.Norecords_Dataload_H""").collect
sql(s"""use insertInto""").collect
sql(s"""drop table if exists Norecords_Dataload_Carbon""").collect
sql(s"""create table Norecords_Dataload_Carbon (Item_code STRING, Qty int)stored by 'org.apache.carbondata.format'""").collect
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
index 36b7b6e..2c3ebca 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
@@ -19,19 +19,23 @@
package org.apache.carbondata.cluster.sdv.generated
import org.apache.spark.sql.common.util._
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
/**
* Test Class for globalsort1TestCase to verify all scenerios
*/
-class GlobalSortTestCase extends QueryTest with BeforeAndAfterAll {
+class GlobalSortTestCase extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach{
override def beforeAll {
sql(s"""drop table if exists uniqdata11""").collect
sql(s"""drop table if exists uniqdataquery1""").collect
}
+ override def beforeEach(): Unit = {
+ sql(s"""drop table if exists uniqdata11""").collect
+ sql(s"""drop table if exists uniqdataquery1""").collect
+ }
//Carbon-Loading-Optimizations-Global-Sort-01-01-01
test("Carbon-Loading-Optimizations-Global-Sort-01-01-01", Include) {
@@ -234,7 +238,7 @@
sql(s"""drop table if exists uniqdata_h""").collect
sql(s"""drop table if exists uniqdata_c""").collect
sql(s"""CREATE TABLE uniqdata_h (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','""").collect
- sql(s"""load data inpath '$resourcesPath/Data/uniqdata/2000_UniqData.csv' into table uniqdata_h""").collect
+ sql(s"""load data inpath '$resourcesPath/Data/uniqdata/2000_UniqData_hive2.csv' into table uniqdata_h""").collect
sql(s"""CREATE TABLE uniqdata_c (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect
sql(s"""insert into uniqdata_c select * from uniqdata_h""").collect
@@ -618,4 +622,9 @@
sql(s"""drop table if exists uniqdata11""").collect
sql(s"""drop table if exists uniqdataquery1""").collect
}
+
+ override def afterEach: Unit = {
+ sql(s"""drop table if exists uniqdata11""").collect
+ sql(s"""drop table if exists uniqdataquery1""").collect
+ }
}
\ No newline at end of file
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesBVATestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesBVATestCase.scala
index 11c705d..f0c5599 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesBVATestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesBVATestCase.scala
@@ -54,7 +54,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/Test_Data1.csv' INTO table Test_Boundary OPTIONS('DELIMITER'=',','QUOTECHAR'='', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='c1_int,c2_Bigint,c3_Decimal,c4_double,c5_string,c6_Timestamp,c7_Datatype_Desc')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/Test_Data1.csv' INTO table Test_Boundary_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/Test_Data1_hive1.csv' INTO table Test_Boundary_hive """).collect
}
@@ -78,7 +78,7 @@
sql(s""" LOAD DATA INPATH '$resourcesPath/Data/Test_Data1.csv' INTO table Test_Boundary1 OPTIONS('DELIMITER'=',','QUOTECHAR'='', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='c1_int,c2_Bigint,c3_Decimal,c4_double,c5_string,c6_Timestamp,c7_Datatype_Desc')""").collect
- sql(s""" LOAD DATA INPATH '$resourcesPath/Data/Test_Data1.csv' INTO table Test_Boundary1_hive """).collect
+ sql(s""" LOAD DATA INPATH '$resourcesPath/Data/Test_Data1_hive2.csv' INTO table Test_Boundary1_hive """).collect
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesBasicTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesBasicTestCase.scala
index 5b3df0a..ed58e70 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesBasicTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesBasicTestCase.scala
@@ -52,7 +52,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' into table uniqdata_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData_hive1.csv' into table uniqdata_hive """).collect
}
@@ -71,7 +71,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/4000_UniqData.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/4000_UniqData.csv' into table uniqdata_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/4000_UniqData_hive1.csv' into table uniqdata_hive """).collect
}
@@ -89,7 +89,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/6000_UniqData.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/6000_UniqData.csv' into table uniqdata_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/6000_UniqData_hive1.csv' into table uniqdata_hive """).collect
}
@@ -100,7 +100,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData_hive1.csv' into table uniqdata_hive """).collect
}
@@ -111,7 +111,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/3000_1_UniqData.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/3000_1_UniqData.csv' into table uniqdata_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/3000_1_UniqData_hive1.csv' into table uniqdata_hive """).collect
}
test("UNIQDATA_DeleteLoad1_3", Include) {
@@ -5304,7 +5304,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/HiveData/100_hive_test.csv' INTO TABLE Carbon_automation OPTIONS('DELIMITER'=',','QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription,imei_count')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/HiveData/100_hive_test.csv' INTO TABLE Carbon_automation_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/HiveData/100_hive_test_hive1.csv' INTO TABLE Carbon_automation_hive """).collect
}
@@ -13093,7 +13093,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_VMALL_1_Day_DATA_2015-09-15.csv' INTO table pushupfilter options('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,uuid,MAC,device_color,device_shell_color,device_name,product_name,ram,rom,cpu_clock,series,check_date,check_year,check_month,check_day,check_hour,bom,inside_name,packing_date,packing_year,packing_month,packing_day,packing_hour,customer_name,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,packing_list_no,order_no,Active_check_time,Active_check_year,Active_check_month,Active_check_day,Active_check_hour,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,Active_network,Active_firmware_version,Active_emui_version,Active_os_version,Latest_check_time,Latest_check_year,Latest_check_month,Latest_check_day,Latest_check_hour,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_firmware_version,Latest_emui_version,Latest_os_version,Latest_network,site,site_desc,product,product_desc')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_VMALL_1_Day_DATA_2015-09-15.csv' INTO table pushupfilter_hive """).collect
+ sql(s"""insert overwrite table pushupfilter_hive select * from pushupfilter""").collect
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesCompactionTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesCompactionTestCase.scala
index 5fdc098..cb275d6 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesCompactionTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesCompactionTestCase.scala
@@ -47,7 +47,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive1.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -58,7 +58,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive2.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -69,7 +69,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive3.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -80,7 +80,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive4.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -91,7 +91,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive5.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -102,7 +102,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive6.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -113,7 +113,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive7.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -124,7 +124,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive8.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -135,7 +135,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive9.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -146,7 +146,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive10.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -157,7 +157,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive11.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -168,7 +168,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive12.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -179,7 +179,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive13.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -190,7 +190,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive14.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -201,7 +201,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive15.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -212,7 +212,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive16.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -223,7 +223,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive17.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -234,7 +234,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive18.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -245,7 +245,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive19.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -256,7 +256,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive20.csv' INTO table Comp_DICTIONARY_INCLUDE_hive """).collect
}
@@ -287,7 +287,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive21.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -298,7 +298,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive22.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -309,7 +309,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive23.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -320,7 +320,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive24.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -331,7 +331,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive25.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -342,7 +342,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive26.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -353,7 +353,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive27.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -364,7 +364,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive28.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -375,7 +375,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive29.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -386,7 +386,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive30.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -397,7 +397,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive31.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -408,7 +408,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive32.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -419,7 +419,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive33.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -430,7 +430,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive34.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -441,7 +441,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive35.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -452,7 +452,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive36.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -463,7 +463,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive37.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -474,7 +474,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive38.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -485,7 +485,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive39.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -496,7 +496,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive40.csv' INTO table Comp_DICTIONARY_EXCLUDE_hive """).collect
}
@@ -3265,8 +3265,8 @@
//Comp_DICTIONARY_INCLUDE_344
test("Comp_DICTIONARY_INCLUDE_344", Include) {
- checkAnswer(s"""select b.contractNumber,b.Latest_DAY,b.gamePointId,b.productionDate,b.deviceInformationId,b.IMEI from Comp_DICTIONARY_INCLUDE a join Comp_DICTIONARY_INCLUDE b on a.Latest_Day=b.Latest_Day""",
- s"""select b.contractNumber,b.Latest_DAY,b.gamePointId,b.productionDate,b.deviceInformationId,b.IMEI from Comp_DICTIONARY_INCLUDE_hive a join Comp_DICTIONARY_INCLUDE_hive b on a.Latest_Day=b.Latest_Day""", "QueriesCompactionTestCase_DICTIONARY_INCLUDE_344")
+ checkAnswer(s"""select b.contractNumber,b.Latest_DAY,b.gamePointId,b.productionDate,b.deviceInformationId,b.IMEI from Comp_DICTIONARY_INCLUDE a join Comp_DICTIONARY_INCLUDE b on a.Latest_Day=b.Latest_Day limit 5""",
+ s"""select b.contractNumber,b.Latest_DAY,b.gamePointId,b.productionDate,b.deviceInformationId,b.IMEI from Comp_DICTIONARY_INCLUDE_hive a join Comp_DICTIONARY_INCLUDE_hive b on a.Latest_Day=b.Latest_Day limit 5""", "QueriesCompactionTestCase_DICTIONARY_INCLUDE_344")
}
@@ -3274,8 +3274,8 @@
//Comp_DICTIONARY_INCLUDE_345
test("Comp_DICTIONARY_INCLUDE_345", Include) {
- checkAnswer(s"""select b.contractNumber,b.Latest_DAY,b.gamePointId,b.productionDate,b.deviceInformationId,b.IMEI from Comp_DICTIONARY_INCLUDE a join Comp_DICTIONARY_INCLUDE b on a.contractnumber=b.contractnumber""",
- s"""select b.contractNumber,b.Latest_DAY,b.gamePointId,b.productionDate,b.deviceInformationId,b.IMEI from Comp_DICTIONARY_INCLUDE_hive a join Comp_DICTIONARY_INCLUDE_hive b on a.contractnumber=b.contractnumber""", "QueriesCompactionTestCase_DICTIONARY_INCLUDE_345")
+ checkAnswer(s"""select b.contractNumber,b.Latest_DAY,b.gamePointId,b.productionDate,b.deviceInformationId,b.IMEI from Comp_DICTIONARY_INCLUDE a join Comp_DICTIONARY_INCLUDE b on a.contractnumber=b.contractnumber limit 5""",
+ s"""select b.contractNumber,b.Latest_DAY,b.gamePointId,b.productionDate,b.deviceInformationId,b.IMEI from Comp_DICTIONARY_INCLUDE_hive a join Comp_DICTIONARY_INCLUDE_hive b on a.contractnumber=b.contractnumber limit 5""", "QueriesCompactionTestCase_DICTIONARY_INCLUDE_345")
}
@@ -3283,8 +3283,8 @@
//Comp_DICTIONARY_INCLUDE_346
test("Comp_DICTIONARY_INCLUDE_346", Include) {
- checkAnswer(s"""select count( contractNumber ),sum( contractNumber ),count(distinct contractNumber ),avg( contractNumber ),max( contractNumber ),min( contractNumber ),1 from Comp_DICTIONARY_INCLUDE""",
- s"""select count( contractNumber ),sum( contractNumber ),count(distinct contractNumber ),avg( contractNumber ),max( contractNumber ),min( contractNumber ),1 from Comp_DICTIONARY_INCLUDE_hive""", "QueriesCompactionTestCase_DICTIONARY_INCLUDE_346")
+ checkAnswer(s"""select count( contractNumber ),sum( contractNumber ),count(distinct contractNumber ),avg( contractNumber ),max( contractNumber ),min( contractNumber ),1 from Comp_DICTIONARY_INCLUDE limit 5""",
+ s"""select count( contractNumber ),sum( contractNumber ),count(distinct contractNumber ),avg( contractNumber ),max( contractNumber ),min( contractNumber ),1 from Comp_DICTIONARY_INCLUDE_hive limit 5""", "QueriesCompactionTestCase_DICTIONARY_INCLUDE_346")
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesExcludeDictionaryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesExcludeDictionaryTestCase.scala
index fcd20fd..49d5b3e 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesExcludeDictionaryTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesExcludeDictionaryTestCase.scala
@@ -54,7 +54,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table TABLE_DICTIONARY_EXCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table TABLE_DICTIONARY_EXCLUDE1_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive41.csv' INTO table TABLE_DICTIONARY_EXCLUDE1_hive """).collect
}
@@ -3892,7 +3892,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/SEQ500/seq_500Records.csv' into table smart_500_DE options('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='SID,PROBEID,INTERFACEID,GROUPID,GGSN_ID,SGSN_ID,dummy,SESSION_INDICATOR,BEGIN_TIME,BEGIN_TIME_MSEL,END_TIME,END_TIME_MSEL,PROT_CATEGORY,PROT_TYPE,L7_CARRIER_PROT,SUB_PROT_TYPE,MSISDN,IMSI,IMEI,ENCRYPT_VERSION,ROAMING_TYPE,ROAM_DIRECTION,MS_IP,SERVER_IP,MS_PORT,APN,SGSN_SIG_IP,GGSN_USER_IP,SGSN_USER_IP,MCC,MNC,RAT,LAC,RAC,SAC,CI,SERVER_DECIMAL,BROWSER_TIMESTAMP,TCP_CONN_STATES,GGSN_SIG_IP_BigInt_NEGATIVE,TCP_STATES_BIGINTPOSITIVE,dummy,TCP_WIN_SIZE,dummy,TCP_MSS,dummy,TCP_CONN_TIMES,dummy,TCP_CONN_2_FAILED_TIMES,dummy,TCP_CONN_3_FAILED_TIMES,HOST,STREAMING_URL,dummy,GET_STREAMING_FAILED_CODE,dummy,GET_STREAMING_FLAG,dummy,GET_NUM,dummy,GET_SUCCEED_NUM,dummy,GET_RETRANS_NUM,dummy,GET_TIMEOUT_NUM,INTBUFFER_FST_FLAG,INTBUFFER_FULL_FLAG,STALL_NUM,dummy,VIDEO_FRAME_RATE,dummy,VIDEO_CODEC_ID,dummy,VIDEO_WIDTH,dummy,VIDEO_HEIGHT,dummy,AUDIO_CODEC_ID,dummy,MEDIA_FILE_TYPE,dummy,PLAY_STATE,dummy,PLAY_STATE,dummy,STREAMING_FLAG,dummy,TCP_STATUS_INDICATOR,dummy,DISCONNECTION_FLAG,dummy,FAILURE_CODE,FLAG,TAC,ECI,dummy,TCP_SYN_TIME_MSEL,dummy,TCP_FST_SYN_DIRECTION,RAN_NE_USER_IP,HOMEMCC,HOMEMNC,dummy,CHARGE_FLAG,dummy,PREPAID_FLAG,dummy,USER_AGENT,dummy,MS_WIN_STAT_TOTAL_NUM,dummy,MS_WIN_STAT_SMALL_NUM,dummy,MS_ACK_TO_1STGET_DELAY,dummy,SERVER_ACK_TO_1STDATA_DELAY,dummy,STREAMING_TYPE,dummy,SOURCE_VIDEO_QUALITY,TETHERING_FLAG,CARRIER_ID,LAYER1ID,LAYER2ID,dummy,LAYER3ID,dummy,LAYER4ID,dummy,LAYER5ID,dummy,LAYER6ID,CHARGING_RULE_BASE_NAME,SP,dummy,EXTENDED_URL,SV,FIRST_SAI_CGI_ECGI,dummy,EXTENDED_URL_OTHER,SIGNALING_USE_FLAG,dummy,DNS_RETRANS_NUM,dummy,DNS_FAIL_CODE,FIRST_RAT,FIRST_RAT,MS_INDICATOR,LAST_SAI_CGI_ECGI,LAST_RAT,dummy,FIRST_LONGITUDE,dummy,FIRST_LATITUDE,dummy,FIRST_ALTITUDE,dummy,FIRST_RASTERLONGITUDE,dummy,FIRST_RASTERLATITUDE,dummy,FIRST_RASTERALTITUDE,dummy,FIRST_FREQUENCYSPOT,dummy,FIRST_CLUTTER,dummy,FIRST_USERBEHAVIOR,dummy,FIRST_SPEED,dummy,FIRST_CREDIBILITY,dummy,LAST_LONGITUDE,dummy,LAST_LATITUDE,dummy,LAST_ALTITUDE,dummy,LAST_RASTERLONGITUDE,dummy,LAST_RASTERLATITUDE,dummy,LAST_RASTERALTITUDE,dummy,LAST_FREQUENCYSPOT,dummy,LAST_CLUTTER,dummy,LAST_USERBEHAVIOR,dummy,LAST_SPEED,dummy,LAST_CREDIBILITY,dummy,IMEI_CIPHERTEXT,APP_ID,dummy,DOMAIN_NAME,dummy,STREAMING_CACHE_IP,dummy,STOP_LONGER_THAN_MIN_THRESHOLD,dummy,STOP_LONGER_THAN_MAX_THRESHOLD,dummy,PLAY_END_STAT,dummy,STOP_START_TIME1,dummy,STOP_END_TIME1,dummy,STOP_START_TIME2,dummy,STOP_END_TIME2,dummy,STOP_START_TIME3,dummy,STOP_END_TIME3,dummy,STOP_START_TIME4,dummy,STOP_END_TIME4,dummy,STOP_START_TIME5,dummy,STOP_END_TIME5,dummy,STOP_START_TIME6,dummy,STOP_END_TIME6,dummy,STOP_START_TIME7,dummy,STOP_END_TIME7,dummy,STOP_START_TIME8,dummy,STOP_END_TIME8,dummy,STOP_START_TIME9,dummy,STOP_END_TIME9,dummy,STOP_START_TIME10,dummy,STOP_END_TIME10,dummy,FAIL_CLASS,RECORD_TYPE,dummy,NODATA_COUNT,dummy,VIDEO_NODATA_DURATION,dummy,VIDEO_SMOOTH_DURATION,dummy,VIDEO_SD_DURATION,dummy,VIDEO_HD_DURATION,dummy,VIDEO_UHD_DURATION,dummy,VIDEO_FHD_DURATION,dummy,FLUCTUATION,dummy,START_DOWNLOAD_THROUGHPUT,dummy,L7_UL_GOODPUT_FULL_MSS,dummy,SESSIONKEY,dummy,FIRST_UCELLID,dummy,LAST_UCELLID,dummy,UCELLID1,dummy,LONGITUDE1,dummy,LATITUDE1,dummy,UCELLID2,dummy,LONGITUDE2,dummy,LATITUDE2,dummy,UCELLID3,dummy,LONGITUDE3,dummy,LATITUDE3,dummy,UCELLID4,dummy,LONGITUDE4,dummy,LATITUDE4,dummy,UCELLID5,dummy,LONGITUDE5,dummy,LATITUDE5,dummy,UCELLID6,dummy,LONGITUDE6,dummy,LATITUDE6,dummy,UCELLID7,dummy,LONGITUDE7,dummy,LATITUDE7,dummy,UCELLID8,dummy,LONGITUDE8,dummy,LATITUDE8,dummy,UCELLID9,dummy,LONGITUDE9,dummy,LATITUDE9,dummy,UCELLID10,dummy,LONGITUDE10,dummy,LATITUDE10,dummy,INTBUFFER_FULL_DELAY,dummy,STALL_DURATION,dummy,STREAMING_DW_PACKETS,dummy,STREAMING_DOWNLOAD_DELAY,dummy,PLAY_DURATION,dummy,STREAMING_QUALITY,dummy,VIDEO_DATA_RATE,dummy,AUDIO_DATA_RATE,dummy,STREAMING_FILESIZE,dummy,STREAMING_DURATIOIN,dummy,TCP_SYN_TIME,dummy,TCP_RTT_STEP1,CHARGE_ID,dummy,UL_REVERSE_TO_DL_DELAY,dummy,DL_REVERSE_TO_UL_DELAY,dummy,DATATRANS_DW_GOODPUT,dummy,DATATRANS_DW_TOTAL_DURATION,dummy,SUM_FRAGMENT_INTERVAL,dummy,TCP_FIN_TIMES,dummy,TCP_RESET_TIMES,dummy,URL_CLASSIFICATION,dummy,STREAMING_LQ_DURATIOIN,dummy,MAX_DNS_DELAY,dummy,MAX_DNS2SYN,dummy,MAX_LATANCY_OF_LINK_SETUP,dummy,MAX_SYNACK2FIRSTACK,dummy,MAX_SYNACK2LASTACK,dummy,MAX_ACK2GET_DELAY,dummy,MAX_FRAG_INTERVAL_PREDELAY,dummy,SUM_FRAG_INTERVAL_PREDELAY,dummy,SERVICE_DELAY_MSEC,dummy,HOMEPROVINCE,dummy,HOMECITY,dummy,SERVICE_ID,dummy,CHARGING_CLASS,dummy,DATATRANS_UL_DURATION,dummy,ASSOCIATED_ID,dummy,PACKET_LOSS_NUM,dummy,JITTER,dummy,MS_DNS_DELAY_MSEL,dummy,GET_STREAMING_DELAY,dummy,TCP_UL_RETRANS_WITHOUTPL,dummy,TCP_DW_RETRANS_WITHOUTPL,dummy,GET_MAX_UL_SIZE,dummy,GET_MIN_UL_SIZE,dummy,GET_MAX_DL_SIZE,dummy,GET_MIN_DL_SIZE,dummy,FLOW_SAMPLE_RATIO,dummy,UL_RTT_LONG_NUM,dummy,DW_RTT_LONG_NUM,dummy,UL_RTT_STAT_NUM,dummy,DW_RTT_STAT_NUM,dummy,USER_PROBE_UL_LOST_PKT,dummy,SERVER_PROBE_UL_LOST_PKT,dummy,SERVER_PROBE_DW_LOST_PKT,dummy,USER_PROBE_DW_LOST_PKT,dummy,AVG_DW_RTT_MICRO_SEC,dummy,AVG_UL_RTT_MICRO_SEC,dummy,RAN_NE_ID,dummy,AVG_UL_RTT,dummy,AVG_DW_RTT,dummy,CHARGING_CHARACTERISTICS,dummy,DL_SERIOUS_OUT_OF_ORDER_NUM,dummy,DL_SLIGHT_OUT_OF_ORDER_NUM,dummy,DL_FLIGHT_TOTAL_SIZE,dummy,DL_FLIGHT_TOTAL_NUM,dummy,DL_MAX_FLIGHT_SIZE,dummy,VALID_TRANS_DURATION,dummy,AIR_PORT_DURATION,dummy,RADIO_CONN_TIMES,dummy,UL_SERIOUS_OUT_OF_ORDER_NUM,dummy,UL_SLIGHT_OUT_OF_ORDER_NUM,dummy,UL_FLIGHT_TOTAL_SIZE,dummy,UL_FLIGHT_TOTAL_NUM,dummy,UL_MAX_FLIGHT_SIZE,dummy,USER_DL_SLIGHT_OUT_OF_ORDER_PACKETS,dummy,SERVER_UL_SLIGHT_OUT_OF_ORDER_PACKETS,dummy,DL_CONTINUOUS_RETRANSMISSION_DELAY,dummy,USER_HUNGRY_DELAY,dummy,SERVER_HUNGRY_DELAY,dummy,UPPERLAYER_IP_UL_FRAGMENTS,dummy,UPPERLAYER_IP_DL_FRAGMENTS,dummy,DOWNLAYER_IP_UL_FRAGMENTS,dummy,DOWNLAYER_IP_DL_FRAGMENTS,dummy,UPPERLAYER_IP_UL_PACKETS,dummy,UPPERLAYER_IP_DL_PACKETS,dummy,DOWNLAYER_IP_UL_PACKETS,dummy,DOWNLAYER_IP_DL_PACKETS,dummy,TCP_UL_PACKAGES_WITHPL,dummy,TCP_DW_PACKAGES_WITHPL,dummy,TCP_UL_PACKAGES_WITHOUTPL,dummy,TCP_DW_PACKAGES_WITHOUTPL,dummy,TCP_UL_RETRANS_WITHPL,dummy,TCP_DW_RETRANS_WITHPL,L4_UL_THROUGHPUT,L4_DW_THROUGHPUT,L4_UL_GOODPUT,L4_DW_GOODPUT,NETWORK_UL_TRAFFIC,NETWORK_DL_TRAFFIC,L4_UL_PACKETS,L4_DW_PACKETS,TCP_RTT,TCP_UL_OUTOFSEQU,TCP_DW_OUTOFSEQU,TCP_UL_RETRANS,TCP_DW_RETRANS')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/SEQ500/seq_500Records.csv' into table smart_500_DE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/SEQ500/seq_500Records_hive1.csv' into table smart_500_DE_hive """).collect
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesIncludeDictionaryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesIncludeDictionaryTestCase.scala
index 769911c..dad731f 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesIncludeDictionaryTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesIncludeDictionaryTestCase.scala
@@ -55,7 +55,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table TABLE_DICTIONARY_INCLUDE options ('DELIMITER'=',', 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20.csv' INTO table TABLE_DICTIONARY_INCLUDE_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/100_olap_C20_hive42.csv' INTO table TABLE_DICTIONARY_INCLUDE_hive """).collect
}
@@ -3883,7 +3883,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/SEQ500/seq_500Records.csv' into table smart_500_DINC options('DELIMITER'=',', 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='SID,PROBEID,INTERFACEID,GROUPID,GGSN_ID,SGSN_ID,dummy,SESSION_INDICATOR,BEGIN_TIME,BEGIN_TIME_MSEL,END_TIME,END_TIME_MSEL,PROT_CATEGORY,PROT_TYPE,L7_CARRIER_PROT,SUB_PROT_TYPE,MSISDN,IMSI,IMEI,ENCRYPT_VERSION,ROAMING_TYPE,ROAM_DIRECTION,MS_IP,SERVER_IP,MS_PORT,APN,SGSN_SIG_IP,GGSN_USER_IP,SGSN_USER_IP,MCC,MNC,RAT,LAC,RAC,SAC,CI,SERVER_DECIMAL,BROWSER_TIMESTAMP,TCP_CONN_STATES,GGSN_SIG_IP_BigInt_NEGATIVE,TCP_STATES_BIGINTPOSITIVE,dummy,TCP_WIN_SIZE,dummy,TCP_MSS,dummy,TCP_CONN_TIMES,dummy,TCP_CONN_2_FAILED_TIMES,dummy,TCP_CONN_3_FAILED_TIMES,HOST,STREAMING_URL,dummy,GET_STREAMING_FAILED_CODE,dummy,GET_STREAMING_FLAG,dummy,GET_NUM,dummy,GET_SUCCEED_NUM,dummy,GET_RETRANS_NUM,dummy,GET_TIMEOUT_NUM,INTBUFFER_FST_FLAG,INTBUFFER_FULL_FLAG,STALL_NUM,dummy,VIDEO_FRAME_RATE,dummy,VIDEO_CODEC_ID,dummy,VIDEO_WIDTH,dummy,VIDEO_HEIGHT,dummy,AUDIO_CODEC_ID,dummy,MEDIA_FILE_TYPE,dummy,PLAY_STATE,dummy,PLAY_STATE,dummy,STREAMING_FLAG,dummy,TCP_STATUS_INDICATOR,dummy,DISCONNECTION_FLAG,dummy,FAILURE_CODE,FLAG,TAC,ECI,dummy,TCP_SYN_TIME_MSEL,dummy,TCP_FST_SYN_DIRECTION,RAN_NE_USER_IP,HOMEMCC,HOMEMNC,dummy,CHARGE_FLAG,dummy,PREPAID_FLAG,dummy,USER_AGENT,dummy,MS_WIN_STAT_TOTAL_NUM,dummy,MS_WIN_STAT_SMALL_NUM,dummy,MS_ACK_TO_1STGET_DELAY,dummy,SERVER_ACK_TO_1STDATA_DELAY,dummy,STREAMING_TYPE,dummy,SOURCE_VIDEO_QUALITY,TETHERING_FLAG,CARRIER_ID,LAYER1ID,LAYER2ID,dummy,LAYER3ID,dummy,LAYER4ID,dummy,LAYER5ID,dummy,LAYER6ID,CHARGING_RULE_BASE_NAME,SP,dummy,EXTENDED_URL,SV,FIRST_SAI_CGI_ECGI,dummy,EXTENDED_URL_OTHER,SIGNALING_USE_FLAG,dummy,DNS_RETRANS_NUM,dummy,DNS_FAIL_CODE,FIRST_RAT,FIRST_RAT,MS_INDICATOR,LAST_SAI_CGI_ECGI,LAST_RAT,dummy,FIRST_LONGITUDE,dummy,FIRST_LATITUDE,dummy,FIRST_ALTITUDE,dummy,FIRST_RASTERLONGITUDE,dummy,FIRST_RASTERLATITUDE,dummy,FIRST_RASTERALTITUDE,dummy,FIRST_FREQUENCYSPOT,dummy,FIRST_CLUTTER,dummy,FIRST_USERBEHAVIOR,dummy,FIRST_SPEED,dummy,FIRST_CREDIBILITY,dummy,LAST_LONGITUDE,dummy,LAST_LATITUDE,dummy,LAST_ALTITUDE,dummy,LAST_RASTERLONGITUDE,dummy,LAST_RASTERLATITUDE,dummy,LAST_RASTERALTITUDE,dummy,LAST_FREQUENCYSPOT,dummy,LAST_CLUTTER,dummy,LAST_USERBEHAVIOR,dummy,LAST_SPEED,dummy,LAST_CREDIBILITY,dummy,IMEI_CIPHERTEXT,APP_ID,dummy,DOMAIN_NAME,dummy,STREAMING_CACHE_IP,dummy,STOP_LONGER_THAN_MIN_THRESHOLD,dummy,STOP_LONGER_THAN_MAX_THRESHOLD,dummy,PLAY_END_STAT,dummy,STOP_START_TIME1,dummy,STOP_END_TIME1,dummy,STOP_START_TIME2,dummy,STOP_END_TIME2,dummy,STOP_START_TIME3,dummy,STOP_END_TIME3,dummy,STOP_START_TIME4,dummy,STOP_END_TIME4,dummy,STOP_START_TIME5,dummy,STOP_END_TIME5,dummy,STOP_START_TIME6,dummy,STOP_END_TIME6,dummy,STOP_START_TIME7,dummy,STOP_END_TIME7,dummy,STOP_START_TIME8,dummy,STOP_END_TIME8,dummy,STOP_START_TIME9,dummy,STOP_END_TIME9,dummy,STOP_START_TIME10,dummy,STOP_END_TIME10,dummy,FAIL_CLASS,RECORD_TYPE,dummy,NODATA_COUNT,dummy,VIDEO_NODATA_DURATION,dummy,VIDEO_SMOOTH_DURATION,dummy,VIDEO_SD_DURATION,dummy,VIDEO_HD_DURATION,dummy,VIDEO_UHD_DURATION,dummy,VIDEO_FHD_DURATION,dummy,FLUCTUATION,dummy,START_DOWNLOAD_THROUGHPUT,dummy,L7_UL_GOODPUT_FULL_MSS,dummy,SESSIONKEY,dummy,FIRST_UCELLID,dummy,LAST_UCELLID,dummy,UCELLID1,dummy,LONGITUDE1,dummy,LATITUDE1,dummy,UCELLID2,dummy,LONGITUDE2,dummy,LATITUDE2,dummy,UCELLID3,dummy,LONGITUDE3,dummy,LATITUDE3,dummy,UCELLID4,dummy,LONGITUDE4,dummy,LATITUDE4,dummy,UCELLID5,dummy,LONGITUDE5,dummy,LATITUDE5,dummy,UCELLID6,dummy,LONGITUDE6,dummy,LATITUDE6,dummy,UCELLID7,dummy,LONGITUDE7,dummy,LATITUDE7,dummy,UCELLID8,dummy,LONGITUDE8,dummy,LATITUDE8,dummy,UCELLID9,dummy,LONGITUDE9,dummy,LATITUDE9,dummy,UCELLID10,dummy,LONGITUDE10,dummy,LATITUDE10,dummy,INTBUFFER_FULL_DELAY,dummy,STALL_DURATION,dummy,STREAMING_DW_PACKETS,dummy,STREAMING_DOWNLOAD_DELAY,dummy,PLAY_DURATION,dummy,STREAMING_QUALITY,dummy,VIDEO_DATA_RATE,dummy,AUDIO_DATA_RATE,dummy,STREAMING_FILESIZE,dummy,STREAMING_DURATIOIN,dummy,TCP_SYN_TIME,dummy,TCP_RTT_STEP1,CHARGE_ID,dummy,UL_REVERSE_TO_DL_DELAY,dummy,DL_REVERSE_TO_UL_DELAY,dummy,DATATRANS_DW_GOODPUT,dummy,DATATRANS_DW_TOTAL_DURATION,dummy,SUM_FRAGMENT_INTERVAL,dummy,TCP_FIN_TIMES,dummy,TCP_RESET_TIMES,dummy,URL_CLASSIFICATION,dummy,STREAMING_LQ_DURATIOIN,dummy,MAX_DNS_DELAY,dummy,MAX_DNS2SYN,dummy,MAX_LATANCY_OF_LINK_SETUP,dummy,MAX_SYNACK2FIRSTACK,dummy,MAX_SYNACK2LASTACK,dummy,MAX_ACK2GET_DELAY,dummy,MAX_FRAG_INTERVAL_PREDELAY,dummy,SUM_FRAG_INTERVAL_PREDELAY,dummy,SERVICE_DELAY_MSEC,dummy,HOMEPROVINCE,dummy,HOMECITY,dummy,SERVICE_ID,dummy,CHARGING_CLASS,dummy,DATATRANS_UL_DURATION,dummy,ASSOCIATED_ID,dummy,PACKET_LOSS_NUM,dummy,JITTER,dummy,MS_DNS_DELAY_MSEL,dummy,GET_STREAMING_DELAY,dummy,TCP_UL_RETRANS_WITHOUTPL,dummy,TCP_DW_RETRANS_WITHOUTPL,dummy,GET_MAX_UL_SIZE,dummy,GET_MIN_UL_SIZE,dummy,GET_MAX_DL_SIZE,dummy,GET_MIN_DL_SIZE,dummy,FLOW_SAMPLE_RATIO,dummy,UL_RTT_LONG_NUM,dummy,DW_RTT_LONG_NUM,dummy,UL_RTT_STAT_NUM,dummy,DW_RTT_STAT_NUM,dummy,USER_PROBE_UL_LOST_PKT,dummy,SERVER_PROBE_UL_LOST_PKT,dummy,SERVER_PROBE_DW_LOST_PKT,dummy,USER_PROBE_DW_LOST_PKT,dummy,AVG_DW_RTT_MICRO_SEC,dummy,AVG_UL_RTT_MICRO_SEC,dummy,RAN_NE_ID,dummy,AVG_UL_RTT,dummy,AVG_DW_RTT,dummy,CHARGING_CHARACTERISTICS,dummy,DL_SERIOUS_OUT_OF_ORDER_NUM,dummy,DL_SLIGHT_OUT_OF_ORDER_NUM,dummy,DL_FLIGHT_TOTAL_SIZE,dummy,DL_FLIGHT_TOTAL_NUM,dummy,DL_MAX_FLIGHT_SIZE,dummy,VALID_TRANS_DURATION,dummy,AIR_PORT_DURATION,dummy,RADIO_CONN_TIMES,dummy,UL_SERIOUS_OUT_OF_ORDER_NUM,dummy,UL_SLIGHT_OUT_OF_ORDER_NUM,dummy,UL_FLIGHT_TOTAL_SIZE,dummy,UL_FLIGHT_TOTAL_NUM,dummy,UL_MAX_FLIGHT_SIZE,dummy,USER_DL_SLIGHT_OUT_OF_ORDER_PACKETS,dummy,SERVER_UL_SLIGHT_OUT_OF_ORDER_PACKETS,dummy,DL_CONTINUOUS_RETRANSMISSION_DELAY,dummy,USER_HUNGRY_DELAY,dummy,SERVER_HUNGRY_DELAY,dummy,UPPERLAYER_IP_UL_FRAGMENTS,dummy,UPPERLAYER_IP_DL_FRAGMENTS,dummy,DOWNLAYER_IP_UL_FRAGMENTS,dummy,DOWNLAYER_IP_DL_FRAGMENTS,dummy,UPPERLAYER_IP_UL_PACKETS,dummy,UPPERLAYER_IP_DL_PACKETS,dummy,DOWNLAYER_IP_UL_PACKETS,dummy,DOWNLAYER_IP_DL_PACKETS,dummy,TCP_UL_PACKAGES_WITHPL,dummy,TCP_DW_PACKAGES_WITHPL,dummy,TCP_UL_PACKAGES_WITHOUTPL,dummy,TCP_DW_PACKAGES_WITHOUTPL,dummy,TCP_UL_RETRANS_WITHPL,dummy,TCP_DW_RETRANS_WITHPL,L4_UL_THROUGHPUT,L4_DW_THROUGHPUT,L4_UL_GOODPUT,L4_DW_GOODPUT,NETWORK_UL_TRAFFIC,NETWORK_DL_TRAFFIC,L4_UL_PACKETS,L4_DW_PACKETS,TCP_RTT,TCP_UL_OUTOFSEQU,TCP_DW_OUTOFSEQU,TCP_UL_RETRANS,TCP_DW_RETRANS')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/SEQ500/seq_500Records.csv' into table smart_500_DINC_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/SEQ500/seq_500Records_hive2.csv' into table smart_500_DINC_hive """).collect
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesNormalTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesNormalTestCase.scala
index 138dc56..0411dcb 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesNormalTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesNormalTestCase.scala
@@ -287,7 +287,7 @@
sql(s"""CREATE table cmb (Cust_UID String,year String, month String, companyNumber String, familyadNumber String, companyAddress String, company String, occupation String, certicardValidTime String, race String, CerticardCity String, birthday String, VIPLevel String, ageRange String, familyaddress String, dimension16 String, SubsidaryBank String, AccountCreationTime String, dimension19 String, dimension20 String, DemandDeposits double, TimeDeposits double, financial double, TreasuryBonds double, fund double, incomeOneyear double, outcomeOneyear double, insurance double, Goldaccount double, dollarDeposits int, euroDeposits int, euroDeposits1 double, euroDeposits2 double, yenDeposits int, wonDeposits int, rupeeDeposits double, HongKongDeposits double, numberoftransactions int, measure19 double, measure20 double, measure21 int, measure22 double, measure23 double, measure24 int, measure25 double, measure26 double, measure27 int, measure28 double, measure29 int, measure30 double, measure31 double, measure32 double, measure33 double, measure34 int, measure35 double, measure36 double, measure37 int, measure38 double, measure39 double, measure40 int, measure41 double, measure42 double, measure43 int, measure44 double, measure45 int, measure46 double, measure47 int, measure48 double, measure49 int, measure50 double, measure51 int, measure52 double, measure53 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_INCLUDE'='dollarDeposits,euroDeposits,yenDeposits,wonDeposits,numberoftransactions,measure21,measure24,measure27,measure29,measure34,measure37,measure40,measure43,measure45,measure47,measure49,measure51,measure53')""").collect
- sql(s"""CREATE table cmb_hive (Cust_UID String,year String, month String, companyAddress String,companyNumber String,company String,occupation String, certicardValidTime String,race String, CerticardCity String,birthday String, VIPLevel String, ageRange String, familyaddress String,familyadNumber String, dimension16 String, SubsidaryBank String, AccountCreationTime String, dimension19 String, dimension20 String, DemandDeposits double, TimeDeposits double, financial double, TreasuryBonds double, fund double, incomeOneyear double, outcomeOneyear double, insurance double, Goldaccount double, dollarDeposits int, euroDeposits int, euroDeposits1 double, euroDeposits2 double, yenDeposits int, wonDeposits int, rupeeDeposits double, HongKongDeposits double, numberoftransactions int, measure19 double, measure20 double, measure21 int, measure22 double, measure23 double, measure24 int, measure25 double, measure26 double, measure27 int, measure28 double, measure29 int, measure30 double, measure31 double, measure32 double, measure33 double, measure34 int, measure35 double, measure36 double, measure37 int, measure38 double, measure39 double, measure40 int, measure41 double, measure42 double, measure43 int, measure44 double, measure45 int, measure46 double, measure47 int, measure48 double, measure49 int, measure50 double, measure51 int, measure52 double, measure53 int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','""").collect
+ sql(s"""CREATE table cmb_hive (Cust_UID String,year String, month String, companyNumber String, familyadNumber String, companyAddress String, company String, occupation String, certicardValidTime String, race String, CerticardCity String, birthday String, VIPLevel String, ageRange String, familyaddress String, dimension16 String, SubsidaryBank String, AccountCreationTime String, dimension19 String, dimension20 String, DemandDeposits double, TimeDeposits double, financial double, TreasuryBonds double, fund double, incomeOneyear double, outcomeOneyear double, insurance double, Goldaccount double, dollarDeposits int, euroDeposits int, euroDeposits1 double, euroDeposits2 double, yenDeposits int, wonDeposits int, rupeeDeposits double, HongKongDeposits double, numberoftransactions int, measure19 double, measure20 double, measure21 int, measure22 double, measure23 double, measure24 int, measure25 double, measure26 double, measure27 int, measure28 double, measure29 int, measure30 double, measure31 double, measure32 double, measure33 double, measure34 int, measure35 double, measure36 double, measure37 int, measure38 double, measure39 double, measure40 int, measure41 double, measure42 double, measure43 int, measure44 double, measure45 int, measure46 double, measure47 int, measure48 double, measure49 int, measure50 double, measure51 int, measure52 double, measure53 int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','""").collect
}
@@ -297,7 +297,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/cmb/data.csv' INTO table cmb OPTIONS ('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='Cust_UID,year,month,companyAddress,companyNumber,company,occupation,certicardValidTime,race,CerticardCity,birthday,VIPLevel,ageRange,familyaddress,familyadNumber,dimension16,SubsidaryBank,AccountCreationTime,dimension19,dimension20,DemandDeposits,TimeDeposits,financial,TreasuryBonds,fund,incomeOneyear,outcomeOneyear,insurance,Goldaccount,dollarDeposits,euroDeposits,euroDeposits1,euroDeposits2,yenDeposits,wonDeposits,rupeeDeposits,HongKongDeposits,numberoftransactions,measure19,measure20,measure21,measure22,measure23,measure24,measure25,measure26,measure27,measure28,measure29,measure30,measure31,measure32,measure33,measure34,measure35,measure36,measure37,measure38,measure39,measure40,measure41,measure42,measure43,measure44,measure45,measure46,measure47,measure48,measure49,measure50,measure51,measure52,measure53')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/cmb/data.csv' INTO table cmb_hive """).collect
+ sql(s"""insert overwrite table cmb_hive select * from cmb""").collect
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesRangeFilterTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesRangeFilterTestCase.scala
index 4b99dbf..bac4697 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesRangeFilterTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesRangeFilterTestCase.scala
@@ -49,7 +49,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE NO_DICTIONARY_CARBON OPTIONS('DELIMITER'= ',', 'QUOTECHAR'='"', 'FILEHEADER'='empno,empname,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE NO_DICTIONARY_CARBON_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata_hive1.csv' INTO TABLE NO_DICTIONARY_CARBON_hive """).collect
sql(s"""CREATE TABLE NO_DICTIONARY_CARBON_6 (empno string, doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, projectenddate Timestamp, designation String,attendance Int,utilization Int,salary Int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno, empname,designation')""").collect
@@ -57,7 +57,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE NO_DICTIONARY_CARBON_6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'='"', 'FILEHEADER'='empno,empname,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE NO_DICTIONARY_CARBON_6_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata_hive2.csv' INTO TABLE NO_DICTIONARY_CARBON_6_hive """).collect
sql(s"""CREATE TABLE DICTIONARY_CARBON_6 (empno string, doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, deptno Int, deptname String, projectcode Int, projectjoindate timestamp,projectenddate Timestamp, designation String,attendance Int,utilization Int,salary Int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='empname,designation')""").collect
@@ -65,7 +65,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE DICTIONARY_CARBON_6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'='"', 'FILEHEADER'='empno,empname,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE DICTIONARY_CARBON_6_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata_hive3.csv' INTO TABLE DICTIONARY_CARBON_6_hive """).collect
sql(s"""CREATE TABLE NO_DICTIONARY_CARBON_7 (empno string, doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, projectenddate Timestamp, designation String,attendance Int,utilization Int,salary Int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno,empname,designation')""").collect
@@ -73,7 +73,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE NO_DICTIONARY_CARBON_7 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"' , 'FILEHEADER'='empno,empname,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE NO_DICTIONARY_CARBON_7_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata_hive4.csv' INTO TABLE NO_DICTIONARY_CARBON_7_hive """).collect
sql(s"""CREATE TABLE NO_DICTIONARY_CARBON_8 (empno string, doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, projectenddate Timestamp, designation String,attendance Int,utilization Int,salary Int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno,empname,designation')""").collect
@@ -81,7 +81,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE NO_DICTIONARY_CARBON_8 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"' , 'FILEHEADER'='empno,empname,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata.csv' INTO TABLE NO_DICTIONARY_CARBON_8_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangefilterdata_hive5.csv' INTO TABLE NO_DICTIONARY_CARBON_8_hive """).collect
sql(s"""CREATE TABLE if not exists directDictionaryTable (empno int,doj Timestamp, salary int) STORED BY 'org.apache.carbondata.format'""").collect
@@ -89,7 +89,7 @@
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangedatasample.csv' INTO TABLE directDictionaryTable OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '"','FILEHEADER'='empno,doj,salary')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangedatasample.csv' INTO TABLE directDictionaryTable_hive """).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/RangeFilter/rangedatasample_hive1.csv' INTO TABLE directDictionaryTable_hive """).collect
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesSparkBlockDistTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesSparkBlockDistTestCase.scala
index 13c4918..a5004eb 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesSparkBlockDistTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/QueriesSparkBlockDistTestCase.scala
@@ -34,10 +34,10 @@
sql("drop table if exists flow_carbon_256b_hive")
sql(s"""CREATE TABLE IF NOT EXISTS flow_carbon_256b ( txn_dte String, dt String, txn_bk String, txn_br String, own_bk String, own_br String, opp_bk String, bus_opr_cde String, opt_prd_cde String, cus_no String, cus_ac String, opp_ac_nme String, opp_ac String, bv_no String, aco_ac String, ac_dte String, txn_cnt int, jrn_par int, mfm_jrn_no String, cbn_jrn_no String, ibs_jrn_no String, vch_no String, vch_seq String, srv_cde String, bus_cd_no String, id_flg String, bv_cde String, txn_time String, txn_tlr String, ety_tlr String, ety_bk String, ety_br String, bus_pss_no String, chk_flg String, chk_tlr String, chk_jrn_no String, bus_sys_no String, txn_sub_cde String, fin_bus_cde String, fin_bus_sub_cde String, chl String, tml_id String, sus_no String, sus_seq String, cho_seq String, itm_itm String, itm_sub String, itm_sss String, dc_flg String, amt decimal(15,2), bal decimal(15,2), ccy String, spv_flg String, vch_vld_dte String, pst_bk String, pst_br String, ec_flg String, aco_tlr String, gen_flg String, his_rec_sum_flg String, his_flg String, vch_typ String, val_dte String, opp_ac_flg String, cmb_flg String, ass_vch_flg String, cus_pps_flg String, bus_rmk_cde String, vch_bus_rmk String, tec_rmk_cde String, vch_tec_rmk String, rsv_ara String, gems_last_upd_d String, gems_last_upd_d_bat String, maps_date String, maps_job String ) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='dt,jrn_par,txn_bk,txn_br,ety_bk,ety_br,chk_flg,bus_sys_no,bus_opr_cde,chl,sus_no,itm_itm,itm_sub,itm_sss,dc_flg,ccy,spv_flg,pst_bk,pst_br,ec_flg,gen_flg,his_rec_sum_flg,his_flg,vch_typ,opp_ac_flg,cmb_flg,ass_vch_flg,cus_pps_flg,bus_rmk_cde,vch_bus_rmk,tec_rmk_cde,vch_tec_rmk,rsv_ara,own_br,own_bk','DICTIONARY_EXCLUDE'='aco_ac,ac_dte,mfm_jrn_no,cbn_jrn_no,ibs_jrn_no,vch_no,vch_seq,srv_cde,cus_no,bus_cd_no,id_flg,cus_ac,bv_cde,bv_no,txn_dte,txn_time,txn_tlr,ety_tlr,bus_pss_no,chk_tlr,chk_jrn_no,txn_sub_cde,fin_bus_cde,fin_bus_sub_cde,opt_prd_cde,tml_id,sus_seq,cho_seq,vch_vld_dte,aco_tlr,opp_ac,opp_ac_nme,opp_bk,val_dte,gems_last_upd_d,gems_last_upd_d_bat,maps_date,maps_job')""").collect
- sql(s"""CREATE TABLE IF NOT EXISTS flow_carbon_256b_hive ( txn_dte String, dt String, txn_bk String, txn_br String, own_bk String, own_br String, opp_bk String, bus_opr_cde String, opt_prd_cde String, cus_no String, cus_ac String, opp_ac_nme String, opp_ac String, bv_no String, aco_ac String, ac_dte String, txn_cnt int, jrn_par int, mfm_jrn_no String, cbn_jrn_no String, ibs_jrn_no String, vch_no String, vch_seq String, srv_cde String, bus_cd_no String, id_flg String, bv_cde String, txn_time String, txn_tlr String, ety_tlr String, ety_bk String, ety_br String, bus_pss_no String, chk_flg String, chk_tlr String, chk_jrn_no String, bus_sys_no String, txn_sub_cde String, fin_bus_cde String, fin_bus_sub_cde String, chl String, tml_id String, sus_no String, sus_seq String, cho_seq String, itm_itm String, itm_sub String, itm_sss String, dc_flg String, amt decimal(15,2), bal decimal(15,2), ccy String, spv_flg String, vch_vld_dte String, pst_bk String, pst_br String, ec_flg String, aco_tlr String, gen_flg String, his_rec_sum_flg String, his_flg String, vch_typ String, val_dte String, opp_ac_flg String, cmb_flg String, ass_vch_flg String, cus_pps_flg String, bus_rmk_cde String, vch_bus_rmk String, tec_rmk_cde String, vch_tec_rmk String, rsv_ara String, gems_last_upd_d String, gems_last_upd_d_bat String, maps_date String, maps_job String ) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','""").collect
+ sql(s"""CREATE TABLE IF NOT EXISTS flow_carbon_256b_hive ( txn_dte String, dt String, txn_bk String, txn_br String, own_bk String, own_br String, opp_bk String, bus_opr_cde String, opt_prd_cde String, cus_no String, cus_ac String, opp_ac_nme String, opp_ac String, bv_no String, aco_ac String, ac_dte String, txn_cnt int, jrn_par int, mfm_jrn_no String, cbn_jrn_no String, ibs_jrn_no String, vch_no String, vch_seq String, srv_cde String, bus_cd_no String, id_flg String, bv_cde String, txn_time String, txn_tlr String, ety_tlr String, ety_bk String, ety_br String, bus_pss_no String, chk_flg String, chk_tlr String, chk_jrn_no String, bus_sys_no String, txn_sub_cde String, fin_bus_cde String, fin_bus_sub_cde String, chl String, tml_id String, sus_no String, sus_seq String, cho_seq String, itm_itm String, itm_sub String, itm_sss String, dc_flg String, amt decimal(15,2), bal decimal(15,2), ccy String, spv_flg String, vch_vld_dte String, pst_bk String, pst_br String, ec_flg String, aco_tlr String, gen_flg String, his_rec_sum_flg String, his_flg String, vch_typ String, val_dte String, opp_ac_flg String, cmb_flg String, ass_vch_flg String, cus_pps_flg String, bus_rmk_cde String, vch_bus_rmk String, tec_rmk_cde String, vch_tec_rmk String, rsv_ara String, gems_last_upd_d String, gems_last_upd_d_bat String, maps_date String, maps_job String ) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n'""").collect
sql(s"""LOAD DATA inpath '$resourcesPath/Data/cmb/data.csv' into table flow_carbon_256b options('BAD_RECORDS_ACTION'='FORCE','DELIMITER'=',', 'QUOTECHAR'='"','FILEHEADER'='txn_dte,dt,txn_bk,txn_br,own_bk,own_br,opp_bk,bus_opr_cde,opt_prd_cde,cus_no,cus_ac,opp_ac_nme,opp_ac,bv_no,aco_ac,ac_dte,txn_cnt,jrn_par,mfm_jrn_no,cbn_jrn_no,ibs_jrn_no,vch_no,vch_seq,srv_cde,bus_cd_no,id_flg,bv_cde,txn_time,txn_tlr,ety_tlr,ety_bk,ety_br,bus_pss_no,chk_flg,chk_tlr,chk_jrn_no,bus_sys_no,txn_sub_cde,fin_bus_cde,fin_bus_sub_cde,chl,tml_id,sus_no,sus_seq,cho_seq,itm_itm,itm_sub,itm_sss,dc_flg,amt,bal,ccy,spv_flg,vch_vld_dte,pst_bk,pst_br,ec_flg,aco_tlr,gen_flg,his_rec_sum_flg,his_flg,vch_typ,val_dte,opp_ac_flg,cmb_flg,ass_vch_flg,cus_pps_flg,bus_rmk_cde,vch_bus_rmk,tec_rmk_cde,vch_tec_rmk,rsv_ara,gems_last_upd_d,gems_last_upd_d_bat,maps_date,maps_job')""").collect
- sql(s"""LOAD DATA inpath '$resourcesPath/Data/cmb/data.csv' into table flow_carbon_256b_hive""").collect
+ sql(s"""insert overwrite table flow_carbon_256b_hive select * from flow_carbon_256b""").collect
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TableCommentAlterTableTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TableCommentAlterTableTestCase.scala
index 8bd53ae..26a6882 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TableCommentAlterTableTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TableCommentAlterTableTestCase.scala
@@ -74,10 +74,9 @@
}
//Check create table with comment after stored by clause
+ //This behavior is okay in Spark-2.3 but may fail with earlier spark versions.
test("TableCommentAlterTable_001_06", Include) {
- intercept[AnalysisException] {
- sql("create table table_comment_afterstoredby (id int, name string) STORED BY 'carbondata' comment 'This is table comment'")
- }
+ sql("create table table_comment_afterstoredby (id int, name string) STORED BY 'carbondata' comment 'This is table comment'")
}
//Check the comment by "describe formatted"
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
index 7448e95..7e25dcd 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
@@ -29,44 +29,44 @@
*/
class SDVSuites extends Suites with BeforeAndAfterAll {
- val suites = new AlterTableTestCase ::
- new BadRecordTestCase ::
- new BatchSortLoad1TestCase ::
- new BatchSortLoad2TestCase ::
- new BatchSortQueryTestCase ::
- new ColumndictTestCase ::
- new CreateTableAsSelectTestCase ::
- new DataLoadingTestCase ::
- new DataLoadingV3TestCase ::
- new InvertedindexTestCase ::
- new OffheapQuery1TestCase ::
- new OffheapQuery2TestCase ::
- new OffheapSort1TestCase ::
- new OffheapSort2TestCase ::
- new PartitionTestCase ::
- new QueriesBasicTestCase ::
- new QueriesBVATestCase ::
- new QueriesCompactionTestCase ::
- new QueriesExcludeDictionaryTestCase ::
- new QueriesIncludeDictionaryTestCase ::
- new QueriesNormalTestCase ::
- new QueriesRangeFilterTestCase ::
- new QueriesSparkBlockDistTestCase ::
- new ShowLoadsTestCase ::
- new SinglepassTestCase ::
- new SortColumnTestCase ::
- new TableCommentAlterTableTestCase ::
- new TimestamptypesTestCase ::
- new V3offheapvectorTestCase ::
- new StandardPartitionTestCase ::
- new Vector1TestCase ::
- new Vector2TestCase ::
- new PreAggregateTestCase ::
- new TimeSeriesPreAggregateTestCase ::
- new TestPartitionWithGlobalSort ::
- new PartitionWithPreAggregateTestCase ::
- new CreateTableWithLocalDictionaryTestCase ::
- new LoadTableWithLocalDictionaryTestCase :: Nil
+ val suites = new AlterTableTestCase ::
+ new BadRecordTestCase ::
+ new BatchSortLoad1TestCase ::
+ new BatchSortLoad2TestCase ::
+ new BatchSortQueryTestCase ::
+ new ColumndictTestCase ::
+ new CreateTableAsSelectTestCase ::
+ new DataLoadingTestCase ::
+ new DataLoadingV3TestCase ::
+ new InvertedindexTestCase ::
+ new OffheapQuery1TestCase ::
+ new OffheapQuery2TestCase ::
+ new OffheapSort1TestCase ::
+ new OffheapSort2TestCase ::
+ new PartitionTestCase ::
+ new QueriesBasicTestCase ::
+ new QueriesBVATestCase ::
+ new QueriesCompactionTestCase ::
+ new QueriesExcludeDictionaryTestCase ::
+ new QueriesIncludeDictionaryTestCase ::
+ new QueriesNormalTestCase ::
+ new QueriesRangeFilterTestCase ::
+ new QueriesSparkBlockDistTestCase ::
+ new ShowLoadsTestCase ::
+ new SinglepassTestCase ::
+ new SortColumnTestCase ::
+ new TableCommentAlterTableTestCase ::
+ new TimestamptypesTestCase ::
+ new V3offheapvectorTestCase ::
+ new StandardPartitionTestCase ::
+ new Vector1TestCase ::
+ new Vector2TestCase ::
+ new PreAggregateTestCase ::
+ new TimeSeriesPreAggregateTestCase ::
+ new TestPartitionWithGlobalSort ::
+ new PartitionWithPreAggregateTestCase ::
+ new CreateTableWithLocalDictionaryTestCase ::
+ new LoadTableWithLocalDictionaryTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
@@ -82,17 +82,17 @@
*/
class SDVSuites1 extends Suites with BeforeAndAfterAll {
- val suites = new BadRecordTestCase ::
- new ComplexDataTypeTestCase ::
- new BatchSortLoad1TestCase ::
- new BatchSortQueryTestCase ::
- new DataLoadingTestCase ::
- new OffheapSort2TestCase ::
- new PartitionTestCase ::
- new QueriesBasicTestCase ::
- new BatchSortLoad3TestCase ::
- new GlobalSortTestCase ::
- new MergeIndexTestCase :: Nil
+ val suites = new BadRecordTestCase ::
+ new ComplexDataTypeTestCase ::
+ new BatchSortLoad1TestCase ::
+ new BatchSortQueryTestCase ::
+ new DataLoadingTestCase ::
+ new OffheapSort2TestCase ::
+ new PartitionTestCase ::
+ new QueriesBasicTestCase ::
+ new BatchSortLoad3TestCase ::
+ new GlobalSortTestCase ::
+ new MergeIndexTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
@@ -108,10 +108,9 @@
*/
class SDVSuites2 extends Suites with BeforeAndAfterAll {
- val suites = new QueriesBVATestCase ::
- new QueriesCompactionTestCase ::
- new QueriesExcludeDictionaryTestCase ::
- new DataLoadingIUDTestCase :: Nil
+ val suites = new QueriesBVATestCase ::
+ new QueriesExcludeDictionaryTestCase ::
+ new DataLoadingIUDTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
@@ -127,39 +126,55 @@
*/
class SDVSuites3 extends Suites with BeforeAndAfterAll {
- val suites = new AlterTableTestCase ::
- new BatchSortLoad2TestCase ::
- new BucketingTestCase ::
- new CreateTableAsSelectTestCase ::
- new InvertedindexTestCase ::
- new OffheapQuery1TestCase ::
- new OffheapQuery2TestCase ::
- new OffheapSort1TestCase ::
- new ShowLoadsTestCase ::
- new SinglepassTestCase ::
- new SortColumnTestCase ::
- new TimestamptypesTestCase ::
- new V3offheapvectorTestCase ::
- new Vector1TestCase ::
- new Vector2TestCase ::
- new QueriesNormalTestCase ::
- new ColumndictTestCase ::
- new QueriesRangeFilterTestCase ::
- new QueriesSparkBlockDistTestCase ::
- new DataLoadingV3TestCase ::
- new QueriesIncludeDictionaryTestCase ::
- new TestRegisterCarbonTable ::
- new TableCommentAlterTableTestCase ::
- new StandardPartitionTestCase ::
- new PreAggregateTestCase ::
- new LuceneTestCase ::
- new TimeSeriesPreAggregateTestCase ::
- new TestPartitionWithGlobalSort ::
- new SDKwriterTestCase ::
- new SetParameterTestCase ::
- new PartitionWithPreAggregateTestCase ::
- new CreateTableWithLocalDictionaryTestCase ::
- new LoadTableWithLocalDictionaryTestCase :: Nil
+ val suites = new QueriesCompactionTestCase :: Nil
+
+ override val nestedSuites = suites.toIndexedSeq
+
+ override protected def afterAll() = {
+ println("---------------- Stopping spark -----------------")
+ TestQueryExecutor.INSTANCE.stop()
+ println("---------------- Stopped spark -----------------")
+ }
+}
+
+/**
+ * Suite class for all tests.
+ */
+class SDVSuites4 extends Suites with BeforeAndAfterAll {
+
+ val suites = new AlterTableTestCase ::
+ new BatchSortLoad2TestCase ::
+ new BucketingTestCase ::
+ new CreateTableAsSelectTestCase ::
+ new InvertedindexTestCase ::
+ new OffheapQuery1TestCase ::
+ new OffheapQuery2TestCase ::
+ new OffheapSort1TestCase ::
+ new ShowLoadsTestCase ::
+ new SinglepassTestCase ::
+ new SortColumnTestCase ::
+ new TimestamptypesTestCase ::
+ new V3offheapvectorTestCase ::
+ new Vector1TestCase ::
+ new Vector2TestCase ::
+ new QueriesNormalTestCase ::
+ new ColumndictTestCase ::
+ new QueriesRangeFilterTestCase ::
+ new QueriesSparkBlockDistTestCase ::
+ new DataLoadingV3TestCase ::
+ new QueriesIncludeDictionaryTestCase ::
+ new TestRegisterCarbonTable ::
+ new TableCommentAlterTableTestCase ::
+ new StandardPartitionTestCase ::
+ new PreAggregateTestCase ::
+ new LuceneTestCase ::
+ new TimeSeriesPreAggregateTestCase ::
+ new TestPartitionWithGlobalSort ::
+ new SDKwriterTestCase ::
+ new SetParameterTestCase ::
+ new PartitionWithPreAggregateTestCase ::
+ new CreateTableWithLocalDictionaryTestCase ::
+ new LoadTableWithLocalDictionaryTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
@@ -173,11 +188,11 @@
/**
* Suite class for compatabiity tests
*/
-class SDVSuites4 extends Suites with BeforeAndAfterAll {
+class SDVSuites5 extends Suites with BeforeAndAfterAll {
- val suites = new CreateTableUsingSparkCarbonFileFormatTestCase ::
- new SparkCarbonDataSourceTestCase ::
- new CarbonV1toV3CompatabilityTestCase :: Nil
+ val suites = new CreateTableUsingSparkCarbonFileFormatTestCase ::
+ new SparkCarbonDataSourceTestCase ::
+ new CarbonV1toV3CompatabilityTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
@@ -191,9 +206,9 @@
/**
* Suite class for presto tests
*/
-class SDVSuites5 extends Suites with BeforeAndAfterAll {
+class SDVSuites6 extends Suites with BeforeAndAfterAll {
- val suites = new PrestoSampleTestCase :: Nil
+ val suites = new PrestoSampleTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala
index cdd415f..78e157e 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala
@@ -28,7 +28,7 @@
/** Fails the test if the two expressions do not match */
protected def compareExpressions(e1: Expression, e2: Expression): Unit = {
- comparePlans(Filter(e1, OneRowRelation), Filter(e2, OneRowRelation))
+ comparePlans(Filter(e1, OneRowRelation.apply()), Filter(e2, OneRowRelation.apply()))
}
/** Fails the test if the two plans do not match */
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
index 0729713..bfaa0cb 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
@@ -51,8 +51,8 @@
val conf = new SparkConf()
if (!TestQueryExecutor.masterUrl.startsWith("local")) {
conf.setJars(TestQueryExecutor.jars).
- set("spark.driver.memory", "6g").
- set("spark.executor.memory", "4g").
+ set("spark.driver.memory", "14g").
+ set("spark.executor.memory", "8g").
set("spark.executor.cores", "2").
set("spark.executor.instances", "2").
set("spark.cores.max", "4")
diff --git a/pom.xml b/pom.xml
index ea4371e..7a3042c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,7 +133,8 @@
<spark.master.url>local[2]</spark.master.url>
<hdfs.url>local</hdfs.url>
<presto.jdbc.url>localhost:8086</presto.jdbc.url>
- <spark.hadoop.hive.metastore.uris>thrift://localhost:8086</spark.hadoop.hive.metastore.uris>
+ <!--todo:this can be enabled when presto tests need to be run-->
+ <!--<spark.hadoop.hive.metastore.uris>thrift://localhost:8086</spark.hadoop.hive.metastore.uris>-->
<suite.name>org.apache.carbondata.cluster.sdv.suite.SDVSuites</suite.name>
<script.exetension>.sh</script.exetension>
<carbon.hive.based.metastore>false</carbon.hive.based.metastore>
