diff --git a/bin/carbon-spark-sql b/bin/carbon-spark-sql
index 2c799d1..4b927d1 100755
--- a/bin/carbon-spark-sql
+++ b/bin/carbon-spark-sql
@@ -33,10 +33,7 @@
 export FWDIR=$SPARK_HOME
 export CARBON_SOURCE="$(cd "`dirname "$0"`"/..; pwd)"
 
-ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.10"
-if [ -d "$CARBON_SOURCE/assembly/target/scala-2.11" ]; then
-  ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.11"
-fi
+ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.11"
 
 GREP_OPTIONS=
 num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^carbondata.*hadoop.*\.jar$" | wc -l)"
diff --git a/examples/spark/CARBON_EXAMPLESLogResource.properties b/examples/spark/CARBON_EXAMPLESLogResource.properties
deleted file mode 100644
index 78f873e..0000000
--- a/examples/spark/CARBON_EXAMPLESLogResource.properties
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-#
-carbon.examples = {0}
\ No newline at end of file
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
deleted file mode 100644
index cc078ff..0000000
--- a/examples/spark/pom.xml
+++ /dev/null
@@ -1,85 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-  <modelVersion>4.0.0</modelVersion>
-
-  <parent>
-    <groupId>org.apache.carbondata</groupId>
-    <artifactId>carbondata-parent</artifactId>
-    <version>1.3.0-SNAPSHOT</version>
-    <relativePath>../../pom.xml</relativePath>
-  </parent>
-
-  <artifactId>carbondata-examples-spark</artifactId>
-  <name>Apache CarbonData :: Spark Examples</name>
-
-  <properties>
-    <dev.path>${basedir}/../../dev</dev.path>
-  </properties>
-
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.carbondata</groupId>
-      <artifactId>carbondata-spark</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-  </dependencies>
-
-  <build>
-    <sourceDirectory>src/main/scala</sourceDirectory>
-    <resources>
-      <resource>
-        <directory>.</directory>
-        <includes>
-          <include>CARBON_EXAMPLESLogResource.properties</include>
-        </includes>
-      </resource>
-    </resources>
-    <plugins>
-      <plugin>
-        <groupId>org.scala-tools</groupId>
-        <artifactId>maven-scala-plugin</artifactId>
-        <version>2.15.2</version>
-        <executions>
-          <execution>
-            <id>compile</id>
-            <goals>
-              <goal>compile</goal>
-            </goals>
-            <phase>compile</phase>
-          </execution>
-          <execution>
-            <phase>process-resources</phase>
-            <goals>
-              <goal>compile</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <configuration>
-          <source>1.7</source>
-          <target>1.7</target>
-        </configuration>
-      </plugin>
-    </plugins>
-  </build>
-
-</project>
\ No newline at end of file
diff --git a/examples/spark/src/main/resources/complexdata.csv b/examples/spark/src/main/resources/complexdata.csv
deleted file mode 100644
index 23a3949..0000000
--- a/examples/spark/src/main/resources/complexdata.csv
+++ /dev/null
@@ -1,101 +0,0 @@
-deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber
-1,109,4ROM size,29-11-2015,1AA1$2BB1,MAC1$MAC2$MAC3,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,29-11-2015$29-11-2015:29-11-2015,109,2738.562
-10,93,1ROM size,29-11-2015,1AA10$2BB10,MAC4$MAC5$MAC6,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,30-11-2015$30-11-2015:30-11-2015,93,1714.635
-100,2591,2ROM size,29-11-2015,1AA100$2BB100,MAC7$MAC8$MAC9,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,01-12-2015$01-12-2015:01-12-2015,2591,1271
-1000,2531,2ROM size,29-11-2015,1AA1000$2BB1000,MAC10$$MAC12,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,02-12-2015$02-12-2015:02-12-2015,2531,692
-10000,2408,0ROM size,29-11-2015,1AA10000$2BB10000,MAC13$$MAC15,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,03-12-2015$03-12-2015:03-12-2015,2408,2175
-100000,1815,0ROM size,29-11-2015,1AA100000$2BB100000,MAC16$$MAC18,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,04-12-2015$04-12-2015:04-12-2015,1815,136
-1000000,2479,4ROM size,29-11-2015,1AA1000000$2BB1000000,MAC19$$MAC21,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,05-12-2015$05-12-2015:05-12-2015,2479,1600
-100001,1845,7ROM size,29-11-2015,1AA100001$,MAC22$$MAC24,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,06-12-2015$06-12-2015:06-12-2015,1845,505
-100002,2008,1ROM size,29-11-2015,1AA100002$,MAC25$$MAC27,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,07-12-2015$07-12-2015:07-12-2015,2008,1341
-100003,1121,5ROM size,29-11-2015,1AA100003$,MAC28$$MAC30,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,08-12-2015$08-12-2015:08-12-2015,1121,2239
-100004,1511,8ROM size,29-11-2015,1AA100004$,MAC31$$MAC33,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,09-12-2015$09-12-2015:09-12-2015,1511,2970
-100005,2759,0ROM size,29-11-2015,1AA100005$,MAC34$$MAC36,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,10-12-2015$10-12-2015:10-12-2015,2759,2593
-100006,2069,7ROM size,29-11-2015,1AA100006$,MAC37$$MAC39,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,11-12-2015$11-12-2015:11-12-2015,2069,2572
-100007,396,7ROM size,29-11-2015,1AA100007$,MAC40$$MAC42,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,12-12-2015$12-12-2015:12-12-2015,396,1991
-100008,104,2ROM size,29-11-2015,1AA100008$,MAC43$$MAC45,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,13-12-2015$13-12-2015:13-12-2015,104,1442
-100009,477,3ROM size,29-11-2015,1AA100009$,MAC46$$MAC48,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,14-12-2015$14-12-2015:14-12-2015,477,1841
-10001,546,8ROM size,29-11-2015,1AA10001$2,MAC49$$MAC51,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,15-12-2015$15-12-2015:15-12-2015,546,298
-100010,2696,3ROM size,29-11-2015,1AA100010$2BB100010,MAC52$$MAC54,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,16-12-2015$16-12-2015:16-12-2015,2696,79
-100011,466,2ROM size,29-11-2015,1AA100011$2BB100011,MAC55$$MAC57,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,17-12-2015$17-12-2015:17-12-2015,466,202
-100012,2644,2ROM size,29-11-2015,1AA100012$2BB100012,MAC58$$MAC60,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,18-12-2015$18-12-2015:18-12-2015,2644,568
-100013,2167,3ROM size,29-11-2015,1AA100013$2BB100013,MAC61$MAC62,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,19-12-2015$19-12-2015:19-12-2015,2167,355
-100014,1069,7ROM size,29-11-2015,1AA100014$2BB100014,MAC64$MAC65,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,20-12-2015$20-12-2015:20-12-2015,1069,151
-100015,1447,9ROM size,29-11-2015,1AA100015$2BB100015,MAC67$MAC68,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,21-12-2015$21-12-2015:21-12-2015,1447,2863
-100016,2963,3ROM size,29-11-2015,1AA100016$2BB100016,MAC70$MAC71,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,22-12-2015$22-12-2015:22-12-2015,2963,1873
-100017,1580,5ROM size,29-11-2015,1AA100017$2BB100017,MAC73$MAC74,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,23-12-2015$23-12-2015:23-12-2015,1580,2205
-100018,446,2ROM size,29-11-2015,1AA100018$2BB100018,MAC76$MAC77,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,24-12-2015$24-12-2015:24-12-2015,446,441
-100019,2151,7ROM size,29-11-2015,1AA100019$2BB100019,MAC79$MAC80,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,25-12-2015$25-12-2015:25-12-2015,2151,2194
-10002,2201,1ROM size,29-11-2015,2BB10002,MAC82$MAC83,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,26-12-2015$26-12-2015:26-12-2015,2201,2972
-100020,2574,5ROM size,29-11-2015,$2BB100020,MAC85$MAC86,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,27-12-2015$27-12-2015:27-12-2015,2574,256
-100021,1734,4ROM size,29-11-2015,$2BB100021,MAC88$MAC89,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,28-12-2015$28-12-2015:28-12-2015,1734,1778
-100022,155,3ROM size,29-11-2015,$2BB100022,MAC91$MAC92,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,29-12-2015$29-12-2015:29-12-2015,155,1999
-100023,1386,8ROM size,29-11-2015,$2BB100023,MAC94$MAC95,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,30-12-2015$30-12-2015:30-12-2015,1386,2194
-100024,1017,9ROM size,29-11-2015,$2BB100024,MAC97$MAC98,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,,1017,2483
-100025,47,2ROM size,29-11-2015,$2BB100025,$MAC101$MAC102,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,,47,1724
-100026,2930,7ROM size,29-11-2015,$2BB100026,$MAC104$MAC105,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,,2930,1768
-100027,2940,0ROM size,29-11-2015,$2BB100027,$MAC107$MAC108,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,,2940,2436
-100028,297,5ROM size,29-11-2015,$2BB100028,$MAC110$MAC111,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,,297,2849
-100029,1695,2ROM size,29-11-2015,$2BB100029,$MAC113$MAC114,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,,1695,1691
-10003,1326,7ROM size,29-11-2015,2BB10003,$MAC116$MAC117,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,,1326,2071
-100030,513,7ROM size,29-11-2015,$2BB100030,$MAC119$MAC120,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,07-01-2016$07-01-2016:,513,1333
-100031,1741,1ROM size,29-11-2015,$2BB100031,$MAC122$MAC123,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,08-01-2016$08-01-2016:,1741,1080
-100032,1198,0ROM size,29-11-2015,$2BB100032,$MAC125$MAC126,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,09-01-2016$09-01-2016:,1198,1053
-100033,273,9ROM size,29-11-2015,$2BB100033,$MAC128$MAC129,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,10-01-2016$10-01-2016:,273,760
-100034,1234,6ROM size,29-11-2015,$2BB100034,$MAC131$MAC132,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,11-01-2016$11-01-2016:,1234,2061
-100035,1619,1ROM size,29-11-2015,$2BB100035,$MAC134$MAC135,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,12-01-2016$12-01-2016:,1619,2142
-100036,2415,2ROM size,29-11-2015,$2BB100036,$MAC137$MAC138,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,13-01-2016$13-01-2016:,2415,2224
-100037,2381,2ROM size,29-11-2015,$2BB100037,$MAC140$MAC141,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,14-01-2016$14-01-2016:,2381,1015
-100038,872,7ROM size,29-11-2015,1AA100038$2BB100038,$MAC143$MAC144,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,15-01-2016$15-01-2016,872,1229
-100039,1835,9ROM size,29-11-2015,1AA100039$2BB100039,$$MAC147,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,16-01-2016$16-01-2016,1835,1750
-10004,2597,1ROM size,29-11-2015,1AA10004$2BB10004,$$MAC150,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,17-01-2016$17-01-2016,2597,1717
-100040,1969,9ROM size,29-11-2015,1AA100040$2BB100040,$$MAC153,,18-01-2016$18-01-2016,1969,2078
-100041,2133,8ROM size,29-11-2015,$,$$MAC156,,19-01-2016$19-01-2016,2133,2734
-100042,631,9ROM size,29-11-2015,$,$$MAC159,,20-01-2016$20-01-2016,631,2745
-100043,187,4ROM size,29-11-2015,$,$$MAC162,2:Chinese::guangzhou:longhua:mingzhi$2:India::guangzhou:longhua:mingzhi,21-01-2016$21-01-2016,187,571
-100044,1232,5ROM size,29-11-2015,$,$$MAC165,2::Guangdong Province:guangzhou:longhua:mingzhi$2::Guangdong Province:guangzhou:longhua:mingzhi,22-01-2016$22-01-2016,1232,1697
-100045,1602,6ROM size,29-11-2015,$,$$MAC168,4:Chinese:Hunan Province::xiangtan:jianshelu$4:India:Hunan Province::xiangtan:jianshelu,23-01-2016$23-01-2016,1602,2553
-100046,2319,9ROM size,29-11-2015,$,$$MAC171,2:Chinese:Guangdong Province:guangzhou::mingzhi$2:India:Guangdong Province:guangzhou::mingzhi,24-01-2016$24-01-2016,2319,1077
-100047,839,4ROM size,29-11-2015,$,$$MAC174,5:Chinese:Hunan Province:zhuzhou:tianyuan:$5:India:Hunan Province:zhuzhou:tianyuan:,25-01-2016$25-01-2016,839,1823
-100048,1184,2ROM size,29-11-2015,$,$$MAC177,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,26-01-2016$:,1184,2399
-100049,2705,2ROM size,29-11-2015,$,$$MAC180,2:Chinese:Guangdong Province$2:India:Guangdong Province,27-01-2016$:,2705,2890
-10005,1185,1ROM size,29-11-2015,,$$MAC183,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,28-01-2016$:,1185,1608
-100050,2457,9ROM size,29-11-2015,,$$MAC186,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,29-01-2016$:,2457,29
-100051,2320,8ROM size,29-11-2015,,$$MAC189,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,30-01-2016$:,2320,1407
-100052,2300,0ROM size,29-11-2015,,$$,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,31-01-2016$:,2300,845
-100053,1210,4ROM size,29-11-2015,,$$,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,01-02-2016$:,1210,1655
-100054,1689,8ROM size,29-11-2015,,$$,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,02-02-2016$:,1689,1368
-100055,2823,2ROM size,29-11-2015,,$$,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,$03-02-2016:03-02-2016,2823,1728
-100056,68,6ROM size,29-11-2015,,$$,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,$04-02-2016:04-02-2016,68,750
-100057,716,0ROM size,29-11-2015,,$$,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,$05-02-2016:05-02-2016,716,2288
-100058,864,6ROM size,29-11-2015,,$$,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,$06-02-2016:06-02-2016,864,2635
-100059,499,6ROM size,29-11-2015,,$$,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,$07-02-2016:07-02-2016,499,1337
-10006,1429,3ROM size,29-11-2015,,$$,:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$:India:Guangdong Province:guangzhou:longhua:mingzhi,$08-02-2016:08-02-2016,1429,2478
-100060,2176,2ROM size,29-11-2015,,$$,:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$:India:Hunan Province:xiangtan:xiangtan:jianshelu,$09-02-2016:09-02-2016,2176,538
-100061,2563,7ROM size,29-11-2015,,,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,$10-02-2016:10-02-2016,2563,1407
-100062,2594,3ROM size,29-11-2015,,,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,$11-02-2016:11-02-2016,2594,2952
-100063,2142,1ROM size,29-11-2015,,,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,$12-02-2016:12-02-2016,2142,1226
-100064,138,0ROM size,29-11-2015,1AA100064$2BB100064,,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,$13-02-2016:13-02-2016,138,865
-100065,1168,6ROM size,29-11-2015,1AA100065$2BB100065,,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,$14-02-2016:14-02-2016,1168,901
-100066,2828,5ROM size,29-11-2015,1AA100066$2BB100066,,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,$:,2828,1864
-100067,1160,0ROM size,29-11-2015,1AA100067$2BB100067,,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,$:,1160,572
-100068,1890,6ROM size,29-11-2015,1AA100068$2BB100068,,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,$:,1890,412
-100069,1195,4ROM size,29-11-2015,1AA100069$2BB100069,,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,$:,1195,1491
-10007,2797,9ROM size,29-11-2015,1AA10007$2BB10007,,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,$:,2797,1350
-100070,44,5ROM size,29-11-2015,1AA100070$2BB100070,,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,$:,44,1567
-100071,1683,6ROM size,29-11-2015,1AA100071$2BB100071,,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,$:,1683,1973
-100072,1085,1ROM size,29-11-2015,1AA100072$2BB100072,,3:Chinese:Hunan Province:changsha:yuhua:shazitang$3:India:Hunan Province:changsha:yuhua:shazitang,22-02-2016$22-02-2016:22-02-2016,1085,448
-100073,776,7ROM size,29-11-2015,1AA100073$2BB100073,,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,23-02-2016$23-02-2016:23-02-2016,776,2488
-100074,2074,9ROM size,29-11-2015,1AA100074$2BB100074,MAC262$MAC263$,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,24-02-2016$24-02-2016:24-02-2016,2074,907
-100075,1062,2ROM size,29-11-2015,1AA100075$2BB100075,MAC265$MAC266$,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,25-02-2016$25-02-2016:25-02-2016,1062,2507
-100076,987,7ROM size,29-11-2015,1AA100076$2BB100076,MAC268$MAC269$,6:Chinese:Hubei Province:wuhan:hongshan:hongshan$6:India:New Delhi:wuhan:hongshan:hongshan,26-02-2016$26-02-2016:26-02-2016,987,732
-100077,2799,9ROM size,29-11-2015,1AA100077$2BB100077,MAC271$MAC272$,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,27-02-2016$27-02-2016:27-02-2016,2799,2077
-100078,2765,1ROM size,29-11-2015,1AA100078$2BB100078,MAC274$MAC275$,7:Chinese:Hubei Province:yichang:yichang:yichang$7:India:New Delhi:delhi:delhi:delhi,28-02-2016$28-02-2016:28-02-2016,2765,1434
-100079,2164,1ROM size,29-11-2015,1AA100079$2BB100079,MAC277$MAC278$,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,29-02-2016$29-02-2016:29-02-2016,2164,1098
-10008,1624,6ROM size,29-11-2015,1AA10008$2BB10008,MAC280$MAC281$,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,01-03-2016$01-03-2016:01-03-2016,1624,813
-100080,2355,1ROM size,29-11-2015,1AA100080$2BB100080,MAC283$MAC284$MAC285,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,02-03-2016$02-03-2016:02-03-2016,2355,954
-100081,1650,6ROM size,29-11-2015,1AA100081$2BB100081,MAC286$MAC287$MAC288,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan,03-03-2016$03-03-2016:03-03-2016,1650,613
-100082,2761,3ROM size,29-11-2015,1AA100082$2BB100082,MAC289$MAC290$MAC291,4:Chinese:Hunan Province:xiangtan:xiangtan:jianshelu$4:India:Hunan Province:xiangtan:xiangtan:jianshelu,04-03-2016$04-03-2016:04-03-2016,2761,2348
-100083,1856,3ROM size,29-11-2015,1AA100083$2BB100083,MAC292$MAC293$MAC294,5:Chinese:Hunan Province:zhuzhou:tianyuan:tianyua$5:India:Hunan Province:zhuzhou:tianyuan:tianyua,05-03-2016$05-03-2016:05-03-2016,1856,2192
-100084,1841,7ROM size,29-11-2015,1AA100084$2BB100084,MAC295$MAC296$MAC297,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,06-03-2016$06-03-2016:06-03-2016,1841,2826
-100085,1841,7ROM size,29-11-2015,1AA100084$2BB100084,MAC295$MAC296$MAC297,2:Chinese:Guangdong Province:guangzhou:longhua:mingzhi$2:India:Guangdong Province:guangzhou:longhua:mingzhi,06-03-2016$06-03-2016:06-03-2016,1841,2826
diff --git a/examples/spark/src/main/resources/data.csv b/examples/spark/src/main/resources/data.csv
deleted file mode 100644
index 5696978..0000000
--- a/examples/spark/src/main/resources/data.csv
+++ /dev/null
@@ -1,11 +0,0 @@
-ID,date,country,name,phonetype,serialname,salary,floatField
-1,2015/7/23,china,aaa1,phone197,ASD69643,15000,2.34
-2,2015/7/24,china,aaa2,phone756,ASD42892,15001,2.34
-3,2015/7/25,china,aaa3,phone1904,ASD37014,15002,2.34
-4,2015/7/26,china,aaa4,phone2435,ASD66902,15003,2.34
-5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,2.34
-6,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.5
-7,2015/7/29,china,aaa7,phone610,ASD14875,15006,2.34
-8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,2.34
-9,2015/7/18,china,aaa9,phone706,ASD86717,15008,2.34
-10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,2.34
\ No newline at end of file
diff --git a/examples/spark/src/main/resources/dimSample.csv b/examples/spark/src/main/resources/dimSample.csv
deleted file mode 100644
index 0c8f27a..0000000
--- a/examples/spark/src/main/resources/dimSample.csv
+++ /dev/null
@@ -1,21 +0,0 @@
-id,name,city
-1,David,Beijing
-2,Mark,Paris
-3,Bill,NewYork
-4,Sara,Tokyo
-5,John,Beijing
-6,Michel,Chicago
-7,Robert,Houston
-8,Sunny,Boston
-9,Mary,Tokyo
-10,Edward,Paris
-11,James,Washington
-12,Maria,Berlin
-13,Adam,Athens
-14,Peter,Boston
-15,George,Paris
-16,Paul,Shanghai
-17,Lisa,Hangzhou
-18,Angel,Beijing
-19,Emily,Bangalore
-20,Kevin,Singapore
\ No newline at end of file
diff --git a/examples/spark/src/main/resources/factSample.csv b/examples/spark/src/main/resources/factSample.csv
deleted file mode 100644
index 9693156..0000000
--- a/examples/spark/src/main/resources/factSample.csv
+++ /dev/null
@@ -1,51 +0,0 @@
-id,name,city,salary
-1,David,Beijing,15000
-1,David,Tokyo,20000
-1,David,Hangzhou,18000
-2,Mark,Paris,12000
-2,Mark,Boston,15000
-2,Mark,Chicago,18000
-3,Bill,NewYork,20000
-3,Bill,Boston,23000
-4,Sara,Tokyo,11000
-4,Sara,Paris,15000
-4,Sara,Chicago,21000
-4,Sara,Hangzhou,17000
-5,John,Beijing,15000
-5,John,Shanghai,16000
-6,Michel,Chicago,11000
-6,Michel,Boston,12000
-6,Michel,Tokyo,11000
-8,Sunny,Boston,14000
-8,Sunny,Beijing,22000
-8,Sunny,Tokyo,20000
-9,Mary,Tokyo,13000
-9,Mary,NewYork,18000
-9,Mary,Paris,16000
-9,Mary,Washington,20000
-9,Mary,Boston,17000
-10,Edward,Paris,20000
-10,Edward,Beijing,12000
-10,Edward,Berlin,15000
-11,James,Washington,16000
-12,Maria,Berlin,15000
-12,Maria,Beijing,16000
-13,Adam,Athens,21000
-13,Adam,Berlin,18000
-13,Adam,Hangzhou,17000
-14,Peter,Boston,20000
-14,Peter,Berlin,21000
-14,Peter,Shanghai,18000
-15,George,Paris,17000
-15,George,Tokyo,12000
-15,George,Beijing,15000
-15,George,Berlin,18000
-16,Paul,Shanghai,22000
-16,Paul,Tokyo,19000
-16,Paul,Paris,24000
-16,Paul,Hangzhou,22000
-18,Angel,Beijing,22000
-18,Angel,NewYork,25000
-18,Angel,Tokyo,22000
-20,Kevin,Singapore,18000
-20,Kevin,Bangalore,16000
\ No newline at end of file
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
deleted file mode 100644
index 9012dcf..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.{AllDictionaryUtil, ExampleUtils}
-
-object AllDictionaryExample {
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("AllDictionaryExample")
-    val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
-    val csvHeader = "ID,date,country,name,phonetype,serialname,salary"
-    val dictCol = "|date|country|name|phonetype|serialname|"
-    val allDictFile = ExampleUtils.currentPath + "/src/main/resources/data.dictionary"
-    // extract all dictionary files from source data
-    AllDictionaryUtil.extractDictionary(cc.sparkContext,
-      testData, allDictFile, csvHeader, dictCol)
-    // Specify date format based on raw data
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
-
-    cc.sql("DROP TABLE IF EXISTS t3")
-
-    cc.sql("""
-           CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Date, country String,
-           name String, phonetype String, serialname String, salary Int,floatField float)
-           STORED BY 'carbondata'
-           """)
-
-    cc.sql(s"""
-           LOAD DATA LOCAL INPATH '$testData' into table t3
-           options('ALL_DICTIONARY_PATH'='$allDictFile')
-           """)
-
-    cc.sql("""
-           SELECT * FROM t3
-           """).show()
-
-    cc.sql("""
-           SELECT * FROM t3 where floatField=3.5
-           """).show()
-
-    cc.sql("DROP TABLE IF EXISTS t3")
-
-    // clean local dictionary files
-    AllDictionaryUtil.cleanDictionary(allDictFile)
-  }
-
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
deleted file mode 100644
index 12901b5..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.ExampleUtils
-
-/**
- * configure alluxio:
- * 1.start alluxio
- * 2.upload the jar :"/alluxio_path/core/client/target/
- * alluxio-core-client-YOUR-VERSION-jar-with-dependencies.jar"
- * 3.Get more detail at:http://www.alluxio.org/docs/master/en/Running-Spark-on-Alluxio.html
- */
-
-object AlluxioExample {
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("AlluxioExample")
-    cc.sparkContext.hadoopConfiguration.set("fs.alluxio.impl", "alluxio.hadoop.FileSystem")
-    FileFactory.getConfiguration.set("fs.alluxio.impl", "alluxio.hadoop.FileSystem")
-
-    // Specify date format based on raw data
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
-
-    cc.sql("DROP TABLE IF EXISTS t3")
-
-    cc.sql("""
-           CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Date, country String,
-           name String, phonetype String, serialname String, salary Int)
-           STORED BY 'carbondata'
-           """)
-
-    cc.sql(s"""
-           LOAD DATA LOCAL INPATH 'alluxio://localhost:19998/data.csv' into table t3
-           """)
-
-    cc.sql("""
-           SELECT country, count(salary) AS amount
-           FROM t3
-           WHERE country IN ('china','france')
-           GROUP BY country
-           """).show()
-
-    cc.sql("DROP TABLE IF EXISTS t3")
-  }
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
deleted file mode 100644
index 36013a8..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object CarbonExample {
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("CarbonExample")
-    val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
-
-    // Specify date format based on raw data
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
-
-    cc.sql("DROP TABLE IF EXISTS t3")
-
-    // Create table, 6 dimensions, 1 measure
-    cc.sql("""
-           CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Date, country String,
-           name String, phonetype String, serialname char(10), salary Int)
-           STORED BY 'carbondata'
-           """)
-
-    // Load data
-    cc.sql(s"""
-           LOAD DATA LOCAL INPATH '$testData' into table t3
-           """)
-
-    // Perform a query
-    cc.sql("""
-           SELECT country, count(salary) AS amount
-           FROM t3
-           WHERE country IN ('china','france')
-           GROUP BY country
-           """).show()
-
-    // Drop table
-    cc.sql("DROP TABLE IF EXISTS t3")
-  }
-
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
deleted file mode 100644
index 9ceadea..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import scala.collection.mutable.LinkedHashMap
-
-import org.apache.spark.sql.AnalysisException
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object CarbonPartitionExample {
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("CarbonPartitionExample")
-    val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
-    val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-    // none partition table
-    cc.sql("DROP TABLE IF EXISTS t0")
-    cc.sql("""
-                | CREATE TABLE IF NOT EXISTS t0
-                | (
-                | vin String,
-                | logdate Timestamp,
-                | phonenumber Int,
-                | country String,
-                | area String
-                | )
-                | STORED BY 'carbondata'
-              """.stripMargin)
-
-    // range partition
-    cc.sql("DROP TABLE IF EXISTS t1")
-    cc.sql("""
-                | CREATE TABLE IF NOT EXISTS t1(
-                | vin STRING,
-                | phonenumber INT,
-                | country STRING,
-                | area STRING
-                | )
-                | PARTITIONED BY (logdate TIMESTAMP)
-                | STORED BY 'carbondata'
-                | TBLPROPERTIES('PARTITION_TYPE'='RANGE',
-                | 'RANGE_INFO'='2014/01/01,2015/01/01,2016/01/01')
-              """.stripMargin)
-
-    // hash partition
-    cc.sql("""
-                | CREATE TABLE IF NOT EXISTS t3(
-                | logdate Timestamp,
-                | phonenumber Int,
-                | country String,
-                | area String
-                | )
-                | PARTITIONED BY (vin String)
-                | STORED BY 'carbondata'
-                | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='5')
-                """.stripMargin)
-
-    // list partition
-    cc.sql("DROP TABLE IF EXISTS t5")
-    cc.sql("""
-               | CREATE TABLE IF NOT EXISTS t5(
-               | vin String,
-               | logdate Timestamp,
-               | phonenumber Int,
-               | area String
-               | )
-               | PARTITIONED BY (country string)
-               | STORED BY 'carbondata'
-               | TBLPROPERTIES('PARTITION_TYPE'='LIST',
-               | 'LIST_INFO'='(China,United States),UK ,japan,(Canada,Russia), South Korea ')
-       """.stripMargin)
-
-    cc.sql(s"DROP TABLE IF EXISTS partitionDB.t9")
-    cc.sql(s"DROP DATABASE IF EXISTS partitionDB")
-    cc.sql(s"CREATE DATABASE partitionDB")
-    cc.sql(s"""
-                | CREATE TABLE IF NOT EXISTS partitionDB.t9(
-                | logdate Timestamp,
-                | phonenumber Int,
-                | country String,
-                | area String
-                | )
-                | PARTITIONED BY (vin String)
-                | STORED BY 'carbondata'
-                | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='5')
-                """.stripMargin)
-    // hive partition table
-    cc.sql("DROP TABLE IF EXISTS t7")
-    cc.sql("""
-       | create table t7(id int, name string) partitioned by (city string)
-       | row format delimited fields terminated by ','
-       """.stripMargin)
-    cc.sql("alter table t7 add partition (city = 'Hangzhou')")
-    // hive partition table
-    cc.sql(s"DROP TABLE IF EXISTS hiveDB.t7")
-    cc.sql(s"CREATE DATABASE IF NOT EXISTS hiveDB")
-    cc.sql("""
-       | create table hiveDB.t7(id int, name string) partitioned by (city string)
-       | row format delimited fields terminated by ','
-       """.stripMargin)
-    cc.sql("alter table hiveDB.t7 add partition (city = 'Shanghai')")
-    //  show partitions
-    try {
-      cc.sql("SHOW PARTITIONS t0").show(100, false)
-    } catch {
-      case ex: AnalysisException => LOGGER.error(ex.getMessage())
-    }
-    cc.sql("SHOW PARTITIONS t1").show(100, false)
-    cc.sql("SHOW PARTITIONS t3").show(100, false)
-    cc.sql("SHOW PARTITIONS t5").show(100, false)
-    cc.sql("SHOW PARTITIONS t7").show(100, false)
-    cc.sql("use hiveDB").show()
-    cc.sql("SHOW PARTITIONS t7").show(100, false)
-    cc.sql("use default").show()
-    cc.sql("SHOW PARTITIONS partitionDB.t9").show(100, false)
-
-    cc.sql("DROP TABLE IF EXISTS t0")
-    cc.sql("DROP TABLE IF EXISTS t1")
-    cc.sql("DROP TABLE IF EXISTS t3")
-    cc.sql("DROP TABLE IF EXISTS t5")
-    cc.sql("DROP TABLE IF EXISTS t7")
-    cc.sql(s"DROP TABLE IF EXISTS hiveDb.t7")
-    cc.sql(s"DROP TABLE IF EXISTS partitionDB.t9")
-    cc.sql(s"DROP DATABASE IF EXISTS partitionDB")
-
-  }
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/CaseClassDataFrameAPIExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/CaseClassDataFrameAPIExample.scala
deleted file mode 100644
index 21bd002..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/CaseClassDataFrameAPIExample.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{DataFrame, SaveMode}
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-case class People(name: String, occupation: String, id: Int)
-
-object CaseClassDataFrameAPIExample {
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("CaseClassDataFrameAPIExample")
-    import cc.implicits._
-
-    val people = List(People("sangeeta", "engineer", 1), People("pallavi", "consultant", 2))
-    val peopleRDD: RDD[People] = cc.sc.parallelize(people)
-    val peopleDF: DataFrame = peopleRDD.toDF("name", "occupation", "id")
-
-    // writing data to carbon table
-    peopleDF.write
-      .format("carbondata")
-      .option("tableName", "carbon2")
-      .option("compress", "true")
-      .mode(SaveMode.Overwrite)
-      .save()
-
-    cc.sql("SELECT * FROM carbon2").show()
-
-    cc.sql("DROP TABLE IF EXISTS carbon2")
-  }
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala
deleted file mode 100644
index 992c3f9..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-/**
- * Carbon supports the complex types ARRAY and STRUCT.
- * The complex type columns can be used with all SQL clauses.
- */
-object ComplexTypeExample {
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("ComplexTypeExample")
-    val dataPath = ExampleUtils.currentPath + "/src/main/resources/complexdata.csv"
-    val tableName = "complexTypeTable"
-
-    cc.sql(s"DROP TABLE IF EXISTS $tableName")
-    cc.sql(s"""CREATE TABLE $tableName (
-                 deviceInformationId int,
-                 channelsId string,
-                 ROMSize string,
-                 purchasedate string,
-                 mobile struct<imei:string,
-                              imsi:string>,
-                 MAC array<string>,
-                 locationinfo array<struct<ActiveAreaId:int,
-                                           ActiveCountry:string,
-                                           ActiveProvince:string,
-                                           Activecity:string,
-                                           ActiveDistrict:string,
-                                           ActiveStreet:string>>,
-                  proddate struct<productionDate: string,
-                                 activeDeactivedate: array<string>>,
-                  gamePointId double,
-                  contractNumber double)
-              STORED BY 'org.apache.carbondata.format' """)
-
-    cc.sql(s"load data local inpath '$dataPath' into table $tableName " +
-      "options ('COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
-
-    // filter on complex ARRAY type with index filter
-    cc.sql(s"SELECT mobile, proddate.activeDeactivedate, MAC[0] FROM $tableName " +
-      "WHERE MAC[0] LIKE 'MAC1%'").show
-
-    // filter on complex STRUCT type
-    cc.sql(s"SELECT mobile, proddate.activeDeactivedate FROM $tableName " +
-      "WHERE mobile.imei = '1AA1' or mobile.imsi = ''").show
-
-    // filter on complex STRUCT<ARRAY>
-    cc.sql(s"SELECT mobile, proddate.activeDeactivedate[0] FROM $tableName " +
-      "WHERE proddate.activeDeactivedate[0] = '29-11-2015'").show
-
-    // filter on complex ARRAY<STRUCT>
-    cc.sql(s"SELECT mobile, locationinfo[0] FROM $tableName " +
-      "WHERE locationinfo[0].ActiveCountry = 'Chinese'").show
-
-    // complex type aggregation and group by complex type
-    cc.sql(s"SELECT mobile, count(proddate) FROM $tableName GROUP BY mobile").show
-
-    cc.sql(s"DROP TABLE IF EXISTS $tableName")
-  }
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala
deleted file mode 100644
index db5def9..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-// scalastyle:off println
-object DataFrameAPIExample {
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("DataFrameAPIExample")
-    ExampleUtils.writeSampleCarbonFile(cc, "carbon1", 1000)
-
-    // use datasource api to read
-    val in = cc.read
-      .format("carbondata")
-      .option("tableName", "carbon1")
-      .load()
-
-    import cc.implicits._
-    var count = in.where($"c3" > 500).select($"*").count()
-    println(s"count after 1 load: $count")
-
-    // append new data, query answer should be 1000
-    ExampleUtils.appendSampleCarbonFile(cc, "carbon1")
-    count = in.where($"c3" > 500).select($"*").count()
-    println(s"count after 2 load: $count")
-
-    // use SQL to read
-    cc.sql("SELECT c1, count(c3) FROM carbon1 where c3 > 500 group by c1 limit 10").show
-
-    // delete carbondata file
-    ExampleUtils.cleanSampleCarbonFile(cc, "carbon1")
-  }
-}
-// scalastyle:on println
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala
deleted file mode 100644
index 551a008..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object DataManagementExample {
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("DataManagementExample")
-
-    cc.sql("DROP TABLE IF EXISTS t3")
-
-    // create a table using CarbonData
-    cc.sql(
-      """
-           CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int)
-           STORED BY 'carbondata'
-      """
-    )
-
-    // data.csv has 1000 lines
-    val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
-
-    // load data 5 times, each load of data is called a segment in CarbonData
-    (1 to 5).map { i =>
-      cc.sql(s"LOAD DATA LOCAL INPATH '$testData' into table t3")
-    }
-    cc.sql("SHOW SEGMENTS FOR TABLE t3 ").show
-
-    // delete the first segment
-    cc.sql("DELETE FROM TABLE T3 WHERE SEGMENT.ID IN (0)")
-    cc.sql("SHOW SEGMENTS FOR TABLE t3 LIMIT 10").show
-
-    // this query will be executed on last 4 segments, it should return 4000 rows
-    cc.sql("SELECT count(*) AS amount FROM t3").show
-
-    // force a major compaction to compact all segments into one
-    cc.sql("ALTER TABLE t3 COMPACT 'MAJOR' ")
-    cc.sql("SHOW SEGMENTS FOR TABLE t3 LIMIT 10").show
-
-    // load again, add another 1000 rows
-    cc.sql(s"LOAD DATA LOCAL INPATH '$testData' into table t3")
-    cc.sql("SHOW SEGMENTS FOR TABLE t3 LIMIT 10").show
-
-    // this query will be executed on 2 segments, it should return 5000 rows
-    cc.sql("SELECT count(*) AS amount FROM t3").show
-
-    // delete all segments whose loading time is before '2099-01-01 01:00:00'
-    cc.sql("DELETE FROM TABLE T3 WHERE SEGMENT.STARTTIME BEFORE '2099-01-01 01:00:00'")
-    cc.sql("SHOW SEGMENTS FOR TABLE t3 ").show
-
-    // this query will be executed on 0 segments, it should return 0 rows
-    cc.sql("SELECT count(*) AS amount FROM t3").show
-
-    // force clean up all 'MARKED_FOR_DELETE' and 'COMPACTED' segments immediately
-    cc.sql("CLEAN FILES FOR TABLE t3")
-    cc.sql("SHOW SEGMENTS FOR TABLE t3").show
-
-    cc.sql("DROP TABLE IF EXISTS t3")
-  }
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
deleted file mode 100644
index 830a819..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import java.io.File
-import java.text.SimpleDateFormat
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.{CarbonContext, DataFrame, Row, SaveMode, SQLContext}
-import org.apache.spark.sql.types.{DataTypes, StructField, StructType}
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object DataUpdateDeleteExample {
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("DataUpdateDeleteExample")
-
-    // for local files
-    var rootPath = ExampleUtils.currentPath
-    // for hdfs files
-    // var rootPath = "hdfs://hdfs-host/carbon"
-
-    val testData = rootPath + "/src/main/resources/data.csv"
-
-    // Specify date format based on raw data
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
-
-    cc.sql("DROP TABLE IF EXISTS t3")
-    cc.sql("DROP TABLE IF EXISTS t5")
-
-    // Create table, 6 dimensions, 1 measure
-    cc.sql("""
-           CREATE TABLE IF NOT EXISTS t3
-           (id Int, date Date, country String,
-           name String, phonetype String, serialname char(10), salary Int)
-           STORED BY 'carbondata'
-           """)
-
-    cc.sql(s"""
-           LOAD DATA LOCAL INPATH '$testData' INTO TABLE t3
-           """)
-
-    // Specify date format based on raw data
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy-MM-dd")
-
-    // Simulate data and write to table t5
-    var fields = Seq[StructField]()
-    fields = fields :+ DataTypes.createStructField("t5_id", DataTypes.IntegerType, false)
-    fields = fields :+ DataTypes.createStructField("t5_date", DataTypes.DateType, false)
-    fields = fields :+ DataTypes.createStructField("t5_country", DataTypes.StringType, false)
-    fields = fields :+ DataTypes.createStructField("t5_name", DataTypes.StringType, false)
-    fields = fields :+ DataTypes.createStructField("t5_phonetype", DataTypes.StringType, false)
-    fields = fields :+ DataTypes.createStructField("t5_serialname", DataTypes.StringType, false)
-    fields = fields :+ DataTypes.createStructField("t5_salary", DataTypes.IntegerType, false)
-    var schema = StructType(fields)
-    var sdf = new SimpleDateFormat("yyyy-MM-dd")
-    var data = cc.sparkContext.parallelize(1 to 10).map { x =>
-      val day = x % 20 + 1
-      var dateStr = ""
-      if (day >= 10) {
-        dateStr = "2017-07-" + day
-      } else {
-        dateStr = "2017-07-0" + day
-      }
-      val dt = new java.sql.Date(sdf.parse(dateStr).getTime);
-      var row = Seq[Any]()
-      row = row :+ x
-      row = row :+ dt
-      row = row :+ "china"
-      row = row :+ "bbb" + x
-      row = row :+ "phone" + 100 * x
-      row = row :+ "ASD" + (1000 * x - x)
-      row = row :+ (25000 + x)
-      Row.fromSeq(row)
-    }
-    var df = cc.createDataFrame(data, schema)
-    df.write
-      .format("carbondata")
-      .option("tableName", "t5")
-      .option("tempCSV", "true")
-      .option("compress", "true")
-      .mode(SaveMode.Overwrite)
-      .save()
-    cc.sql("""
-           SELECT * FROM t5 ORDER BY t5_id
-           """).show()
-
-    // 1.Update data with simple SET
-    cc.sql("""
-           SELECT * FROM t3 ORDER BY t3.id
-           """).show()
-
-    // Update data where salary < 15003
-    val dateStr = "2018-08-08"
-    cc.sql(s"""
-           UPDATE t3 SET (t3.date, t3.country) = ('$dateStr', 'india') WHERE t3.salary < 15003
-           """).show()
-    // Query data again after the above update
-    cc.sql("""
-           SELECT * FROM t3 ORDER BY t3.id
-           """).show()
-
-    cc.sql("""
-           UPDATE t3 SET (t3.salary) = (t3.salary + 9) WHERE t3.name = 'aaa1'
-           """).show()
-    // Query data again after the above update
-    cc.sql("""
-           SELECT * FROM t3 ORDER BY t3.id
-           """).show()
-
-    // 2.Update data with subquery result SET
-    cc.sql("""
-         UPDATE t3
-         SET (t3.country, t3.name) = (SELECT t5_country, t5_name FROM t5 WHERE t5_id = 5)
-         WHERE t3.id < 5""").show()
-    cc.sql("""
-         UPDATE t3
-         SET (t3.date, t3.serialname, t3.salary) =
-         (SELECT '2099-09-09', t5_serialname, '9999' FROM t5  WHERE t5_id = 5)
-         WHERE t3.id < 5""").show()
-
-    // Query data again after the above update
-    cc.sql("""
-           SELECT * FROM t3 ORDER BY t3.id
-           """).show()
-
-    // 3.Update data with join query result SET
-    cc.sql("""
-         UPDATE t3
-         SET (t3.country, t3.salary) =
-         (SELECT t5_country, t5_salary FROM t5 FULL JOIN t3 u
-         WHERE u.id = t5_id and t5_id=6) WHERE t3.id >6""").show()
-
-    // Query data again after the above update
-    cc.sql("""
-           SELECT * FROM t3 ORDER BY t3.id
-           """).show()
-
-    // 4.Delete data where salary > 15005
-    cc.sql("""
-           DELETE FROM t3 WHERE t3.salary > 15005
-           """).show()
-
-    // Query data again after delete data
-    cc.sql("""
-           SELECT * FROM t3 ORDER BY t3.id
-           """).show()
-
-    // 5.Delete data WHERE id in (1, 2, $key)
-    var key = 3
-    cc.sql(s"""
-           DELETE FROM t3 WHERE t3.id in (1, 2, $key)
-           """).show()
-
-    // Query data again after delete data
-    cc.sql("""
-           SELECT * FROM t3 ORDER BY t3.id
-           """).show()
-
-    // Drop table
-    cc.sql("DROP TABLE IF EXISTS t3")
-    cc.sql("DROP TABLE IF EXISTS t5")
-  }
-
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala
deleted file mode 100644
index a3af2c3..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.spark.sql.{SaveMode, SQLContext}
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-object DatasourceExample {
-
-  def main(args: Array[String]) {
-    // use CarbonContext to write CarbonData files
-    val cc = ExampleUtils.createCarbonContext("DatasourceExample")
-    ExampleUtils.writeSampleCarbonFile(cc, "table1")
-
-    // Use SQLContext to read CarbonData files
-    val sqlContext = new SQLContext(cc.sparkContext)
-    sqlContext.sql(
-      s"""
-        | CREATE TEMPORARY TABLE source
-        | USING org.apache.spark.sql.CarbonSource
-        | OPTIONS (path '${cc.storePath}/default/table1')
-      """.stripMargin)
-    sqlContext.sql("SELECT c1, c2, count(*) FROM source WHERE c3 > 100 GROUP BY c1, c2").show
-
-    // delete carbondata file
-    ExampleUtils.cleanSampleCarbonFile(cc, "table1")
-  }
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
deleted file mode 100644
index 6a66b93..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.spark.sql.SQLContext
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.examples.util.ExampleUtils
-
-/**
- * This example needs Spark 1.6 or later version to run
- */
-object DirectSQLExample {
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("DirectSQLExample")
-    ExampleUtils.writeSampleCarbonFile(cc, "table1")
-
-    // Use SQLContext to read CarbonData files without creating table
-    val sqlContext = new SQLContext(cc.sparkContext)
-    sqlContext.sql(
-      s"""
-        | SELECT c1, c2, count(*)
-        | FROM carbondata.`${cc.storePath}/${CarbonCommonConstants.DATABASE_DEFAULT_NAME}/table1`
-        | WHERE c3 > 100
-        | GROUP BY c1, c2
-      """.stripMargin).show
-
-    // delete carbondata file
-    ExampleUtils.cleanSampleCarbonFile(cc, "table1")
-  }
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
deleted file mode 100644
index 94d35b0..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.spark.sql.{CarbonContext, CarbonEnv, CarbonRelation}
-
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.examples.util.ExampleUtils
-import org.apache.carbondata.processing.util.CarbonLoaderUtil
-
-/**
- * example for global dictionary generation
- * pls check files under directory of target/store/default/dictSample/Metadata
- * and verify global dictionary values
- */
-object GenerateDictionaryExample {
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("GenerateDictionaryExample")
-    val factFilePath = ExampleUtils.currentPath + "/src/main/resources/factSample.csv"
-    val carbonTablePath = CarbonStorePath.getCarbonTablePath(ExampleUtils.storeLocation,
-      new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "dictSample", "1"))
-    val dictFolderPath = carbonTablePath.getMetadataDirectoryPath
-
-    // execute sql statement
-    cc.sql("DROP TABLE IF EXISTS dictSample")
-
-    cc.sql("""
-           CREATE TABLE IF NOT EXISTS dictSample(id Int, name String, city String, salary Int)
-           STORED BY 'org.apache.carbondata.format'
-           """)
-
-    cc.sql(s"""
-           LOAD DATA LOCAL INPATH '$factFilePath' INTO TABLE dictSample
-           """)
-
-    // check generated dictionary
-    val tableIdentifier =
-      new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "dictSample", "1")
-    printDictionary(cc, tableIdentifier, dictFolderPath)
-  }
-
-  def printDictionary(cc: CarbonContext, carbonTableIdentifier: CarbonTableIdentifier,
-                      dictFolderPath: String) {
-    val dataBaseName = carbonTableIdentifier.getDatabaseName
-    val tableName = carbonTableIdentifier.getTableName
-    val carbonRelation = CarbonEnv.get.carbonMetastore.lookupRelation1(Option(dataBaseName),
-        tableName)(cc).asInstanceOf[CarbonRelation]
-    val carbonTable = carbonRelation.tableMeta.carbonTable
-    val dimensions = carbonTable.getDimensionByTableName(tableName.toLowerCase())
-      .toArray.map(_.asInstanceOf[CarbonDimension])
-    // scalastyle:off println
-    // print dictionary information
-    println("**********************************************************************************")
-    println(s"table:$tableName in " + s"database:$dataBaseName")
-    for (dimension <- dimensions) {
-      println("**********************************************************************************")
-      println(s"dictionary of dimension: ${dimension.getColName}")
-      println(s"Key\t\t\tValue")
-      val columnIdentifier = new DictionaryColumnUniqueIdentifier(carbonTableIdentifier,
-        dimension.getColumnIdentifier, dimension.getDataType,
-        CarbonStorePath
-          .getCarbonTablePath(carbonTable.getStorePath, carbonTable.getCarbonTableIdentifier))
-      val dict = CarbonLoaderUtil.getDictionary(columnIdentifier, cc.storePath)
-      var index: Int = 1
-      var distinctValue = dict.getDictionaryValueForKey(index)
-      while (distinctValue != null) {
-        println(index + s"\t\t\t" + distinctValue)
-        index += 1
-        distinctValue = dict.getDictionaryValueForKey(index)
-      }
-    }
-    println("**********************************************************************************")
-    // scalastyle:on println
-  }
-
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
deleted file mode 100644
index d471ab6..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import org.apache.hadoop.conf.Configuration
-
-import org.apache.carbondata.examples.util.ExampleUtils
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonProjection}
-
-// scalastyle:off println
-object HadoopFileExample {
-
-  def main(args: Array[String]): Unit = {
-    val cc = ExampleUtils.createCarbonContext("HadoopFileExample")
-    ExampleUtils.writeSampleCarbonFile(cc, "carbon1")
-
-    // read two columns
-    val projection = new CarbonProjection
-    projection.addColumn("c1")  // column c1
-    projection.addColumn("c3")  // column c3
-    val conf = new Configuration()
-    CarbonInputFormat.setColumnProjection(conf, projection)
-
-    val sc = cc.sparkContext
-    val input = sc.newAPIHadoopFile(s"${cc.storePath}/default/carbon1",
-      classOf[CarbonInputFormat[Array[Object]]],
-      classOf[Void],
-      classOf[Array[Object]],
-      conf)
-    val result = input.map(x => x._2.toList).collect
-    result.foreach(x => println(x.mkString(", ")))
-
-    // delete carbondata file
-    ExampleUtils.cleanSampleCarbonFile(cc, "carbon1")
-  }
-}
-// scalastyle:on println
-
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
deleted file mode 100644
index 824730f..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import java.io.File
-
-import scala.util.Random
-
-import org.apache.spark.sql.{CarbonContext, DataFrame, Row, SaveMode, SQLContext}
-import org.apache.spark.sql.types.{DataTypes, StructType}
-
-import org.apache.carbondata.examples.PerfTest._
-import org.apache.carbondata.examples.util.ExampleUtils
-
-// scalastyle:off println
-
-/**
- * represent one query
- */
-class Query(val queryType: String, val queryNo: Int, val sqlString: String) {
-
-  /**
-   * run the query in a batch and calculate average time
-   *
-   * @param sqlContext context to run the query
-   * @param runs run how many time
-   * @param datasource datasource to run
-   */
-  def run(sqlContext: SQLContext, runs: Int, datasource: String): QueryResult = {
-    // run repeated and calculate average time elapsed
-    require(runs >= 1)
-    val sqlToRun = makeSQLString(datasource)
-
-    val firstTime = withTime {
-      sqlContext.sql(sqlToRun).collect
-    }
-
-    var totalTime: Long = 0
-    var result: Array[Row] = null
-    (1 to (runs - 1)).foreach { x =>
-      totalTime += withTime {
-        result = sqlContext.sql(sqlToRun).collect
-      }
-    }
-
-    val avgTime = totalTime / (runs - 1)
-    QueryResult(datasource, result, avgTime, firstTime)
-  }
-
-  private def makeSQLString(datasource: String): String = {
-    sqlString.replaceFirst("tableName", PerfTest.makeTableName(datasource))
-  }
-
-}
-
-/**
- * query performance result
- */
-case class QueryResult(datasource: String, result: Array[Row], avgTime: Long, firstTime: Long)
-
-class QueryRunner(sqlContext: SQLContext, dataFrame: DataFrame, datasources: Seq[String]) {
-
-  /**
-   * run a query on each datasource
-   */
-  def run(query: Query, runs: Int): Seq[QueryResult] = {
-    var results = Seq[QueryResult]()
-    datasources.foreach { datasource =>
-      val result = query.run(sqlContext, runs, datasource)
-      results :+= result
-    }
-    checkResult(results)
-    results
-  }
-
-  private def checkResult(results: Seq[QueryResult]): Unit = {
-    results.foldLeft(results.head) { (last, cur) =>
-      if (last.result.sortBy(_.toString()).sameElements(cur.result.sortBy(_.toString()))) cur
-      else sys.error(s"result is not the same between " +
-          s"${last.datasource} and " +
-          s"${cur.datasource}")
-    }
-  }
-
-  private def loadToNative(datasource: String): Unit = {
-    val savePath = PerfTest.savePath(datasource)
-    println(s"loading data into $datasource, path: $savePath")
-    dataFrame.write
-        .mode(SaveMode.Overwrite)
-        .format(datasource)
-        .save(savePath)
-    sqlContext.read
-        .format(datasource)
-        .load(savePath)
-        .registerTempTable(PerfTest.makeTableName(datasource))
-  }
-
-  /**
-   * load data to each datasource
-   */
-  def loadData: Seq[QueryResult] = {
-    // load data into all datasources
-    var results = Seq[QueryResult]()
-    datasources.foreach { datasource =>
-      val time = withTime {
-        datasource match {
-          case "parquet" =>
-            dataFrame.sqlContext.setConf(s"spark.sql.$datasource.compression.codec", "snappy")
-            loadToNative(datasource)
-          case "orc" =>
-            dataFrame.sqlContext.sparkContext.hadoopConfiguration.set("orc.compress", "SNAPPY")
-            loadToNative(datasource)
-          case "carbon" =>
-            sqlContext.sql(s"DROP TABLE IF EXISTS ${PerfTest.makeTableName(datasource)}")
-            println(s"loading data into $datasource, path: " +
-                s"${dataFrame.sqlContext.asInstanceOf[CarbonContext].storePath}")
-            dataFrame.write
-                .format("org.apache.spark.sql.CarbonSource")
-                .option("tableName", PerfTest.makeTableName(datasource))
-                .mode(SaveMode.Overwrite)
-                .save()
-          case _ => sys.error("unsupported data source")
-        }
-      }
-      println(s"load data into $datasource completed, time taken ${time/1000000}ms")
-      results :+= QueryResult(datasource, null, time, time)
-    }
-    results
-  }
-
-  def shutDown(): Unit = {
-    // drop all tables and temp files
-    datasources.foreach {
-        case datasource @ ("parquet" | "orc") =>
-          val f = new File(PerfTest.savePath(datasource))
-          if (f.exists()) f.delete()
-        case "carbon" =>
-          sqlContext.sql(s"DROP TABLE IF EXISTS ${PerfTest.makeTableName("carbon")}")
-        case _ => sys.error("unsupported data source")
-    }
-  }
-}
-
-/**
- * template for table data generation
- *
- * @param dimension number of dimension columns and their cardinality
- * @param measure number of measure columns
- */
-case class TableTemplate(dimension: Seq[(Int, Int)], measure: Int)
-
-/**
- * utility to generate random data according to template
- */
-class TableGenerator(sqlContext: SQLContext) {
-
-  /**
-   * generate a dataframe from random data
-   */
-  def genDataFrame(template: TableTemplate, rows: Int): DataFrame = {
-    val measures = template.measure
-    val dimensions = template.dimension.foldLeft(0) {(x, y) => x + y._1}
-    val cardinality = template.dimension.foldLeft(Seq[Int]()) {(x, y) =>
-      x ++ (1 to y._1).map(z => y._2)
-    }
-    print(s"generating data: $rows rows of $dimensions dimensions and $measures measures. ")
-    println("cardinality for each dimension: " + cardinality.mkString(", "))
-
-    val dimensionFields = (1 to dimensions).map { id =>
-      DataTypes.createStructField(s"c$id", DataTypes.StringType, false)
-    }
-    val measureFields = (dimensions + 1 to dimensions + measures).map { id =>
-      DataTypes.createStructField(s"c$id", DataTypes.IntegerType, false)
-    }
-    val schema = StructType(dimensionFields ++ measureFields)
-    val data = sqlContext.sparkContext.parallelize(1 to rows).map { x =>
-      val random = new Random()
-      val dimSeq = (1 to dimensions).map { y =>
-        s"P${y}_${random.nextInt(cardinality(y - 1))}"
-      }
-      val msrSeq = (1 to measures).map { y =>
-        random.nextInt(10)
-      }
-      Row.fromSeq(dimSeq ++ msrSeq)
-    }
-    val df = sqlContext.createDataFrame(data, schema)
-    df.write.mode(SaveMode.Overwrite).parquet(PerfTest.savePath("temp"))
-    sqlContext.parquetFile(PerfTest.savePath("temp"))
-  }
-}
-
-object PerfTest {
-
-  private val olap: Seq[String] = Seq(
-    """SELECT c3, c4, sum(c8) FROM tableName
-      |WHERE c1 = 'P1_23' and c2 = 'P2_43'
-      |GROUP BY c3, c4""".stripMargin,
-
-    """SELECT c2, c3, sum(c9) FROM tableName
-      |WHERE c1 = 'P1_432' and c4 = 'P4_3' and c5 = 'P5_2'
-      |GROUP by c2, c3 """.stripMargin,
-
-    """SELECT c2, count(distinct c1), sum(c8) FROM tableName
-      |WHERE c3="P3_4" and c5="P5_4"
-      |GROUP BY c2 """.stripMargin,
-
-    """SELECT c2, c5, count(distinct c1), sum(c7) FROM tableName
-      |WHERE c4="P4_4" and c5="P5_7" and c8>4
-      |GROUP BY c2, c5 """.stripMargin
-  )
-
-  private val point: Seq[String] = Seq(
-    """SELECT c4 FROM tableName
-      |WHERE c1="P1_43" """.stripMargin,
-
-    """SELECT c3 FROM tableName
-      |WHERE c1="P1_542" and c2="P2_23" """.stripMargin,
-
-    """SELECT c3, c5 FROM tableName
-      |WHERE c1="P1_52" and c7=4""".stripMargin,
-
-    """SELECT c4, c9 FROM tableName
-      |WHERE c1="P1_43" and c8<3""".stripMargin
-  )
-
-  private val filter: Seq[String] = Seq(
-    """SELECT * FROM tableName
-      |WHERE c2="P2_43" """.stripMargin,
-
-    """SELECT * FROM tableName
-      |WHERE c3="P3_3"  """.stripMargin,
-
-    """SELECT * FROM tableName
-      |WHERE c2="P2_32" and c3="P3_23" """.stripMargin,
-
-    """SELECT * FROM tableName
-      |WHERE c3="P3_28" and c4="P4_3" """.stripMargin
-  )
-
-  private val scan: Seq[String] = Seq(
-    """SELECT sum(c7), sum(c8), avg(c9), max(c10) FROM tableName """.stripMargin,
-
-    """SELECT sum(c7) FROM tableName
-      |WHERE c2="P2_32" """.stripMargin,
-
-    """SELECT sum(c7), sum(c8), sum(9), sum(c10) FROM tableName
-      |WHERE c4="P4_4" """.stripMargin,
-
-    """SELECT sum(c7), sum(c8), sum(9), sum(c10) FROM tableName
-      |WHERE c2="P2_75" and c6<5 """.stripMargin
-  )
-
-  def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("PerfTest")
-
-    // prepare performance queries
-    var workload = Seq[Query]()
-    olap.zipWithIndex.foreach(x => workload :+= new Query("OLAP Query", x._2, x._1))
-    point.zipWithIndex.foreach(x => workload :+= new Query("Point Query", x._2, x._1))
-    filter.zipWithIndex.foreach(x => workload :+= new Query("Filter Query", x._2, x._1))
-    scan.zipWithIndex.foreach(x => workload :+= new Query("Scan Query", x._2, x._1))
-
-    // prepare data
-    val rows = 3 * 1000 * 1000
-    val dimension = Seq((1, 1 * 1000), (1, 100), (1, 50), (2, 10)) // cardinality for each column
-    val measure = 5 // number of measure
-    val template = TableTemplate(dimension, measure)
-    val df = new TableGenerator(cc).genDataFrame(template, rows)
-    println("generate data completed")
-
-    // run all queries against all data sources
-    val datasource = Seq("parquet", "orc", "carbon")
-    val runner = new QueryRunner(cc, df, datasource)
-
-    val results = runner.loadData
-    println(s"load performance: ${results.map(_.avgTime / 1000000L).mkString(", ")}")
-
-    var parquetTime: Double = 0
-    var orcTime: Double = 0
-    var carbonTime: Double = 0
-
-    println(s"query id: ${datasource.mkString(", ")}, result in millisecond")
-    workload.foreach { query =>
-      // run 4 times each round, will print performance of first run and avg time of last 3 runs
-      print(s"${query.queryType} ${query.queryNo}: ")
-      val results = runner.run(query, 4)
-      print(s"${results.map(_.avgTime / 1000000L).mkString(", ")} ")
-      println(s"[sql: ${query.sqlString.replace('\n', ' ')}]")
-      parquetTime += results(0).avgTime
-      orcTime += results(1).avgTime
-      carbonTime += results(2).avgTime
-    }
-
-    println(s"Total time: ${parquetTime / 1000000}, ${orcTime / 1000000}, " +
-        s"${carbonTime / 1000000} = 1 : ${parquetTime / orcTime} : ${parquetTime / carbonTime}")
-    runner.shutDown()
-  }
-
-  def makeTableName(datasource: String): String = {
-    s"${datasource}_perftest_table"
-  }
-
-  def savePath(datasource: String): String =
-      s"${ExampleUtils.currentPath}/target/perftest/${datasource}"
-
-  def withTime(body: => Unit): Long = {
-    val start = System.nanoTime()
-    body
-    System.nanoTime() - start
-  }
-
-}
-// scalastyle:on println
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
deleted file mode 100644
index 3ab61bf..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples.util
-
-import java.io.DataOutputStream
-
-import scala.collection.mutable.{ArrayBuffer, HashSet}
-
-import org.apache.spark.SparkContext
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory
-
-object AllDictionaryUtil {
-  private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-  def extractDictionary(sc: SparkContext,
-                        srcData: String,
-                        outputPath: String,
-                        fileHeader: String,
-                        dictCol: String): Unit = {
-    val fileHeaderArr = fileHeader.split(",")
-    val isDictCol = new Array[Boolean](fileHeaderArr.length)
-    for (i <- 0 until fileHeaderArr.length) {
-      if (dictCol.contains("|" + fileHeaderArr(i).toLowerCase() + "|")) {
-        isDictCol(i) = true
-      } else {
-        isDictCol(i) = false
-      }
-    }
-    val dictionaryRdd = sc.textFile(srcData).flatMap(x => {
-      val tokens = x.split(",")
-      val result = new ArrayBuffer[(Int, String)]()
-      for (i <- 0 until isDictCol.length) {
-        if (isDictCol(i)) {
-          try {
-            result += ((i, tokens(i)))
-          } catch {
-            case ex: ArrayIndexOutOfBoundsException =>
-              LOGGER.error("Read a bad record: " + x)
-          }
-        }
-      }
-      result
-    }).groupByKey().flatMap(x => {
-      val distinctValues = new HashSet[(Int, String)]()
-      for (value <- x._2) {
-        distinctValues.add(x._1, value)
-      }
-      distinctValues
-    })
-    val dictionaryValues = dictionaryRdd.map(x => x._1 + "," + x._2).collect()
-    saveToFile(dictionaryValues, outputPath)
-  }
-
-  def cleanDictionary(outputPath: String): Unit = {
-    try {
-      val fileType = FileFactory.getFileType(outputPath)
-      val file = FileFactory.getCarbonFile(outputPath, fileType)
-      if (file.exists()) {
-        file.delete()
-      }
-    } catch {
-      case ex: Exception =>
-        LOGGER.error("Clean dictionary catching exception:" + ex)
-    }
-  }
-
-  def saveToFile(contents: Array[String], outputPath: String): Unit = {
-    var writer: DataOutputStream = null
-    try {
-      val fileType = FileFactory.getFileType(outputPath)
-      val file = FileFactory.getCarbonFile(outputPath, fileType)
-      if (!file.exists()) {
-        file.createNewFile()
-      }
-      writer = FileFactory.getDataOutputStream(outputPath, fileType)
-      for (content <- contents) {
-        writer.writeBytes(content + "\n")
-      }
-    } catch {
-      case ex: Exception =>
-        LOGGER.error("Save dictionary to file catching exception:" + ex)
-    } finally {
-      if (writer != null) {
-        try {
-          writer.close()
-        } catch {
-          case ex: Exception =>
-            LOGGER.error("Close output stream catching exception:" + ex)
-        }
-      }
-    }
-  }
-}
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
deleted file mode 100644
index f98ec3b..0000000
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples.util
-
-import java.io.File
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.{CarbonContext, SaveMode}
-
-import org.apache.carbondata.core.util.CarbonProperties
-
-// scalastyle:off println
-
-object ExampleUtils {
-
-  def currentPath: String = new File(this.getClass.getResource("/").getPath + "../../")
-      .getCanonicalPath
-  val storeLocation = currentPath + "/target/store"
-
-  def createCarbonContext(appName: String): CarbonContext = {
-    val sc = new SparkContext(new SparkConf()
-        .setAppName(appName)
-        .setMaster("local[2]"))
-    sc.setLogLevel("ERROR")
-
-    println(s"Starting $appName using spark version ${sc.version}")
-
-    val cc = new CarbonContext(sc, storeLocation, currentPath + "/target/carbonmetastore")
-
-    CarbonProperties.getInstance()
-      .addProperty("carbon.storelocation", storeLocation)
-    cc
-  }
-
-  /**
-   * This func will write a sample CarbonData file containing following schema:
-   * c1: String, c2: String, c3: Double
-   * Returns table path
-   */
-  def writeSampleCarbonFile(cc: CarbonContext, tableName: String, numRows: Int = 1000): String = {
-    cc.sql(s"DROP TABLE IF EXISTS $tableName")
-    writeDataframe(cc, tableName, numRows, SaveMode.Overwrite)
-    s"$storeLocation/default/$tableName"
-  }
-
-  /**
-   * This func will append data to the CarbonData file
-   * Returns table path
-   */
-  def appendSampleCarbonFile(cc: CarbonContext, tableName: String, numRows: Int = 1000): String = {
-    writeDataframe(cc, tableName, numRows, SaveMode.Append)
-    s"$storeLocation/default/$tableName"
-  }
-
-  /**
-   * create a new dataframe and write to CarbonData file, based on save mode
-   */
-  private def writeDataframe(
-      cc: CarbonContext, tableName: String, numRows: Int, mode: SaveMode): Unit = {
-    // use CarbonContext to write CarbonData files
-    import cc.implicits._
-    val sc = cc.sparkContext
-    val df = sc.parallelize(1 to numRows, 2)
-        .map(x => ("a", "b", x))
-        .toDF("c1", "c2", "c3")
-
-    // save dataframe directl to carbon file without tempCSV
-    df.write
-      .format("carbondata")
-      .option("tableName", tableName)
-      .option("compress", "true")
-      .option("tempCSV", "false")
-      .mode(mode)
-      .save()
-  }
-
-  def cleanSampleCarbonFile(cc: CarbonContext, tableName: String): Unit = {
-    cc.sql(s"DROP TABLE IF EXISTS $tableName")
-  }
-}
-// scalastyle:on println
-
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index 94af8ec..af25771 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -38,20 +38,6 @@
       <groupId>org.apache.carbondata</groupId>
       <artifactId>carbondata-spark2</artifactId>
       <version>${project.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-hive-thriftserver_2.10</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-repl_2.10</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-sql_2.10</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index 13bd581..e0ad499 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -67,20 +67,6 @@
             <groupId>org.apache.carbondata</groupId>
             <artifactId>carbondata-spark2</artifactId>
             <version>${project.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.spark</groupId>
-                    <artifactId>spark-hive-thriftserver_2.10</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.spark</groupId>
-                    <artifactId>spark-repl_2.10</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.spark</groupId>
-                    <artifactId>spark-sql_2.10</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.hive</groupId>
@@ -116,12 +102,6 @@
             <groupId>org.apache.carbondata</groupId>
             <artifactId>carbondata-hadoop</artifactId>
             <version>${project.version}</version>
-            <exclusions>
-              <exclusion>
-                <groupId>org.apache.spark</groupId>
-                <artifactId>spark-sql_2.10</artifactId>
-              </exclusion>
-            </exclusions>
         </dependency>
     </dependencies>
 
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index e11ce4c..13d351d 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -244,10 +244,6 @@
           <groupId>com.fasterxml.jackson.core</groupId>
           <artifactId>jackson-databind</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-sql_2.10</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -371,10 +367,6 @@
           <groupId>com.fasterxml.jackson.core</groupId>
           <artifactId>jackson-databind</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-sql_2.10</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
diff --git a/integration/spark-common-cluster-test/pom.xml b/integration/spark-common-cluster-test/pom.xml
index 9728a5c..e529035 100644
--- a/integration/spark-common-cluster-test/pom.xml
+++ b/integration/spark-common-cluster-test/pom.xml
@@ -40,12 +40,6 @@
       <artifactId>carbondata-spark-common</artifactId>
       <version>${project.version}</version>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-hive-thriftserver_2.10</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
@@ -167,16 +161,6 @@
           <artifactId>carbondata-spark</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive-thriftserver_2.10</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_2.10</artifactId>
-            </exclusion>
-          </exclusions>
         </dependency>
       </dependencies>
     </profile>
@@ -188,16 +172,6 @@
           <artifactId>carbondata-spark</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive-thriftserver_2.10</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_2.10</artifactId>
-            </exclusion>
-          </exclusions>
         </dependency>
       </dependencies>
     </profile>
@@ -212,16 +186,6 @@
           <artifactId>carbondata-spark2</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive-thriftserver_2.10</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_2.10</artifactId>
-            </exclusion>
-          </exclusions>
         </dependency>
       </dependencies>
     </profile>
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index 7c67000..b2ee316 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -91,12 +91,6 @@
       <artifactId>carbondata-spark-common</artifactId>
       <version>${project.version}</version>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-hive-thriftserver_2.10</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
@@ -341,16 +335,6 @@
           <artifactId>carbondata-spark</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive-thriftserver_2.10</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_2.10</artifactId>
-            </exclusion>
-          </exclusions>
         </dependency>
       </dependencies>
     </profile>
@@ -362,16 +346,6 @@
           <artifactId>carbondata-spark</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive-thriftserver_2.10</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_2.10</artifactId>
-            </exclusion>
-          </exclusions>
         </dependency>
       </dependencies>
     </profile>
@@ -386,16 +360,6 @@
           <artifactId>carbondata-spark2</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive-thriftserver_2.10</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_2.10</artifactId>
-            </exclusion>
-          </exclusions>
         </dependency>
       </dependencies>
     </profile>
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index 82ff7a4..d40e213 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -38,16 +38,6 @@
       <groupId>org.apache.carbondata</groupId>
       <artifactId>carbondata-processing</artifactId>
       <version>${project.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-hive-thriftserver_2.10</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-sql_2.10</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.carbondata</groupId>
diff --git a/integration/spark/CARBON_SPARK_INTERFACELogResource.properties b/integration/spark/CARBON_SPARK_INTERFACELogResource.properties
deleted file mode 100644
index 61856cf..0000000
--- a/integration/spark/CARBON_SPARK_INTERFACELogResource.properties
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-#
-carbon.spark.interface = {0}
\ No newline at end of file
diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml
deleted file mode 100644
index 5060809..0000000
--- a/integration/spark/pom.xml
+++ /dev/null
@@ -1,194 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-  <modelVersion>4.0.0</modelVersion>
-
-  <parent>
-    <groupId>org.apache.carbondata</groupId>
-    <artifactId>carbondata-parent</artifactId>
-    <version>1.3.0-SNAPSHOT</version>
-    <relativePath>../../pom.xml</relativePath>
-  </parent>
-
-  <artifactId>carbondata-spark</artifactId>
-  <name>Apache CarbonData :: Spark</name>
-
-  <properties>
-    <dev.path>${basedir}/../../dev</dev.path>
-  </properties>
-
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.carbondata</groupId>
-      <artifactId>carbondata-common</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.carbondata</groupId>
-      <artifactId>carbondata-core</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.carbondata</groupId>
-      <artifactId>carbondata-processing</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.carbondata</groupId>
-      <artifactId>carbondata-hadoop</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.carbondata</groupId>
-      <artifactId>carbondata-spark-common</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.scala-lang</groupId>
-      <artifactId>scala-compiler</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.scala-lang</groupId>
-      <artifactId>scala-reflect</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.scala-lang</groupId>
-      <artifactId>scala-library</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-repl_${scala.binary.version}</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.scalatest</groupId>
-      <artifactId>scalatest_${scala.binary.version}</artifactId>
-      <version>2.2.1</version>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
-
-  <build>
-    <testSourceDirectory>src/test/scala</testSourceDirectory>
-    <resources>
-      <resource>
-        <directory>src/resources</directory>
-      </resource>
-      <resource>
-        <directory>.</directory>
-        <includes>
-          <include>CARBON_SPARK_INTERFACELogResource.properties</include>
-        </includes>
-      </resource>
-    </resources>
-    <plugins>
-      <plugin>
-        <groupId>org.scala-tools</groupId>
-        <artifactId>maven-scala-plugin</artifactId>
-        <version>2.15.2</version>
-        <executions>
-          <execution>
-            <id>compile</id>
-            <goals>
-              <goal>compile</goal>
-            </goals>
-            <phase>compile</phase>
-          </execution>
-          <execution>
-            <id>testCompile</id>
-            <goals>
-              <goal>testCompile</goal>
-            </goals>
-            <phase>test</phase>
-          </execution>
-          <execution>
-            <phase>process-resources</phase>
-            <goals>
-              <goal>compile</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <configuration>
-          <source>1.7</source>
-          <target>1.7</target>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.18</version>
-        <!-- Note config is repeated in scalatest config -->
-        <configuration>
-          <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-          <argLine>-Xmx3g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m</argLine>
-          <systemProperties>
-            <java.awt.headless>true</java.awt.headless>
-          </systemProperties>
-          <failIfNoTests>false</failIfNoTests>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.scalatest</groupId>
-        <artifactId>scalatest-maven-plugin</artifactId>
-        <version>1.0</version>
-        <!-- Note config is repeated in surefire config -->
-        <configuration>
-          <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-          <junitxml>.</junitxml>
-          <filereports>CarbonTestSuite.txt</filereports>
-          <argLine> ${argLine} -ea -Xmx3g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m
-          </argLine>
-          <stderr />
-          <environmentVariables>
-          </environmentVariables>
-          <systemProperties>
-            <java.awt.headless>true</java.awt.headless>
-          </systemProperties>
-        </configuration>
-        <executions>
-          <execution>
-            <id>test</id>
-            <goals>
-              <goal>test</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
-  <profiles>
-    <profile>
-      <id>sdvtest</id>
-      <properties>
-        <maven.test.skip>true</maven.test.skip>
-      </properties>
-    </profile>
-  </profiles>
-
-</project>
\ No newline at end of file
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
deleted file mode 100644
index 92c8402..0000000
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.readsupport;
-
-import java.io.IOException;
-import java.sql.Date;
-import java.sql.Timestamp;
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.hadoop.readsupport.impl.DictionaryDecodeReadSupport;
-
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.catalyst.expressions.GenericRow;
-import org.apache.spark.unsafe.types.UTF8String;
-
-public class SparkRowReadSupportImpl extends DictionaryDecodeReadSupport<Row> {
-
-  @Override public void initialize(CarbonColumn[] carbonColumns,
-      AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
-    super.initialize(carbonColumns, absoluteTableIdentifier);
-    //can initialize and generate schema here.
-  }
-
-  @Override public Row readRow(Object[] data) {
-    for (int i = 0; i < dictionaries.length; i++) {
-      if (data[i] == null) {
-        continue;
-      }
-      if (dictionaries[i] != null) {
-        data[i] = DataTypeUtil
-            .getDataBasedOnDataType(dictionaries[i].getDictionaryValueForKeyInBytes((int) data[i]),
-                (CarbonDimension) carbonColumns[i]);
-        if (data[i] == null) {
-          continue;
-        }
-        if (dataTypes[i] == DataTypes.STRING) {
-          data[i] = UTF8String.fromString(data[i].toString());
-        } else if (dataTypes[i] == DataTypes.TIMESTAMP) {
-          data[i] = new Timestamp((long) data[i]);
-        } else if (dataTypes[i] == DataTypes.DATE) {
-          data[i] = new Date((long) data[i]);
-        } else if (dataTypes[i] == DataTypes.LONG) {
-          data[i] = data[i];
-        }
-      }
-      else if (carbonColumns[i].hasEncoding(Encoding.DIRECT_DICTIONARY)) {
-        //convert the long to timestamp in case of direct dictionary column
-        if (DataTypes.TIMESTAMP == carbonColumns[i].getDataType()) {
-          data[i] = new Timestamp((long) data[i] / 1000L);
-        } else if (DataTypes.DATE == carbonColumns[i].getDataType()) {
-          data[i] = new Date((long) data[i]);
-        }
-      }
-    }
-    return new GenericRow(data);
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
deleted file mode 100644
index 7881b93..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark
-
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql._
-import org.apache.spark.sql.execution.command.LoadTable
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonType}
-
-class CarbonDataFrameWriter(val dataFrame: DataFrame) {
-
-  private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
-  def saveAsCarbonFile(parameters: Map[String, String] = Map()): Unit = {
-    checkContext()
-    val cc = CarbonContext.getInstance(dataFrame.sqlContext.sparkContext)
-
-    // create a new table using dataframe's schema and write its content into the table
-    cc.sql(makeCreateTableString(dataFrame.schema, new CarbonOption(parameters)))
-    writeToCarbonFile(parameters)
-  }
-
-  def appendToCarbonFile(parameters: Map[String, String] = Map()): Unit = {
-    // append the data as a new load
-    checkContext()
-    writeToCarbonFile(parameters)
-  }
-
-  private def writeToCarbonFile(parameters: Map[String, String] = Map()): Unit = {
-    val options = new CarbonOption(parameters)
-    val cc = CarbonContext.getInstance(dataFrame.sqlContext.sparkContext)
-    if (options.tempCSV) {
-      loadTempCSV(options, cc)
-    } else {
-      loadDataFrame(options, cc)
-    }
-  }
-
-  /**
-   * Firstly, saving DataFrame to CSV files
-   * Secondly, load CSV files
-   * @param options
-   * @param cc
-   */
-  private def loadTempCSV(options: CarbonOption, cc: CarbonContext): Unit = {
-    // temporary solution: write to csv file, then load the csv into carbon
-    val storePath = CarbonEnv.get.carbonMetastore.storePath
-    val tempCSVFolder = new StringBuilder(storePath).append(CarbonCommonConstants.FILE_SEPARATOR)
-      .append("tempCSV")
-      .append(CarbonCommonConstants.UNDERSCORE).append(options.dbName)
-      .append(CarbonCommonConstants.UNDERSCORE).append(options.tableName)
-      .append(CarbonCommonConstants.UNDERSCORE).append(System.nanoTime()).toString
-    writeToTempCSVFile(tempCSVFolder, options)
-
-    val tempCSVPath = new Path(tempCSVFolder)
-    val fs = tempCSVPath.getFileSystem(dataFrame.sqlContext.sparkContext.hadoopConfiguration)
-
-    def countSize(): Double = {
-      var size: Double = 0
-      val itor = fs.listFiles(tempCSVPath, true)
-      while (itor.hasNext) {
-        val f = itor.next()
-        if (f.getPath.getName.startsWith("part-")) {
-          size += f.getLen
-        }
-      }
-      size
-    }
-
-    LOGGER.info(s"temporary CSV file size: ${countSize / 1024 / 1024} MB")
-
-    try {
-      cc.sql(makeLoadString(tempCSVFolder, options))
-    } finally {
-      fs.delete(tempCSVPath, true)
-    }
-  }
-
-  private def checkContext(): Unit = {
-    // To avoid derby problem, dataframe need to be writen and read using CarbonContext
-    require(dataFrame.sqlContext.isInstanceOf[CarbonContext],
-      "Error in saving dataframe to carbon file, must use CarbonContext to save dataframe"
-    )
-  }
-
-  private def writeToTempCSVFile(tempCSVFolder: String, options: CarbonOption): Unit = {
-
-    val strRDD = dataFrame.rdd.mapPartitions { case iter =>
-      new Iterator[String] {
-        override def hasNext = iter.hasNext
-
-        def convertToCSVString(seq: Seq[Any]): String = {
-          val build = new java.lang.StringBuilder()
-          if (seq.head != null) {
-            build.append(seq.head.toString)
-          }
-          val itemIter = seq.tail.iterator
-          while (itemIter.hasNext) {
-            build.append(CarbonCommonConstants.COMMA)
-            val value = itemIter.next()
-            if (value != null) {
-              build.append(value.toString)
-            }
-          }
-          build.toString
-        }
-
-        override def next: String = {
-          convertToCSVString(iter.next.toSeq)
-        }
-      }
-    }
-
-    if (options.compress) {
-      strRDD.saveAsTextFile(tempCSVFolder, classOf[GzipCodec])
-    } else {
-      strRDD.saveAsTextFile(tempCSVFolder)
-    }
-  }
-
-  /**
-   * Loading DataFrame directly without saving DataFrame to CSV files.
-   * @param options
-   * @param cc
-   */
-  private def loadDataFrame(options: CarbonOption, cc: CarbonContext): Unit = {
-    val header = dataFrame.columns.mkString(",")
-    LoadTable(
-      Some(options.dbName),
-      options.tableName,
-      null,
-      Seq(),
-      Map("fileheader" -> header) ++ options.toMap,
-      isOverwriteExist = false,
-      null,
-      Some(dataFrame),
-      None).run(cc)
-  }
-
-  private def convertToCarbonType(sparkType: DataType): String = {
-    sparkType match {
-      case StringType => CarbonType.STRING.getName
-      case IntegerType => CarbonType.INT.getName
-      case ShortType => "smallint"
-      case LongType => "bigint"
-      case FloatType => CarbonType.DOUBLE.getName
-      case DoubleType => CarbonType.DOUBLE.getName
-      case TimestampType => CarbonType.TIMESTAMP.getName
-      case DateType => CarbonType.DATE.getName
-      case decimal: DecimalType => s"${CarbonType.DECIMAL.getName} (${decimal.precision}" +
-                                   s", ${decimal.scale})"
-      case other => sys.error(s"unsupported type: $other")
-    }
-  }
-
-  private def makeCreateTableString(schema: StructType, options: CarbonOption): String = {
-    val properties = Map(
-      "DICTIONARY_INCLUDE" -> options.dictionaryInclude,
-      "DICTIONARY_EXCLUDE" -> options.dictionaryExclude
-    ).filter(_._2.isDefined).map(p => s"'${p._1}' = '${p._2.get}'").mkString(",")
-    val carbonSchema = schema.map { field =>
-      s"${ field.name } ${ convertToCarbonType(field.dataType) }"
-    }
-    s"""
-          CREATE TABLE IF NOT EXISTS ${options.dbName}.${options.tableName}
-          (${ carbonSchema.mkString(", ") })
-          STORED BY '${ CarbonContext.datasourceName }'
-          ${ if (properties.nonEmpty) " TBLPROPERTIES (" + properties + ")" else ""}
-      """
-  }
-
-  private def makeLoadString(csvFolder: String, options: CarbonOption): String = {
-    s"""
-          LOAD DATA INPATH '$csvFolder'
-          INTO TABLE ${options.dbName}.${options.tableName}
-          OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}',
-          'SINGLE_PASS' = '${options.singlePass}')
-      """
-  }
-
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
deleted file mode 100644
index 6eeeaf9..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ /dev/null
@@ -1,1088 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.rdd
-
-import java.text.SimpleDateFormat
-import java.util
-import java.util.concurrent._
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ListBuffer
-import scala.util.Random
-import scala.util.control.Breaks._
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.io.NullWritable
-import org.apache.hadoop.mapreduce.Job
-import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
-import org.apache.spark.{SparkEnv, SparkException, TaskContext}
-import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, NewHadoopRDD, RDD}
-import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
-import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
-import org.apache.spark.sql.hive.DistributionUtil
-import org.apache.spark.util.SparkUtil
-
-import org.apache.carbondata.common.constants.LoggerAction
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
-import org.apache.carbondata.core.dictionary.server.DictionaryServer
-import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
-import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
-import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
-import org.apache.carbondata.core.metadata.schema.partition.PartitionType
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil
-import org.apache.carbondata.core.scan.partition.PartitionUtil
-import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager}
-import org.apache.carbondata.core.util.{ByteUtil, CarbonProperties}
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.exception.DataLoadingException
-import org.apache.carbondata.processing.loading.{DataLoadProcessBuilder, FailureCauses}
-import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat, StringArrayWritable}
-import org.apache.carbondata.processing.loading.exception.{CarbonDataLoadingException, NoRetryException}
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel
-import org.apache.carbondata.processing.loading.sort.SortScopeOptions
-import org.apache.carbondata.processing.merger.{CarbonCompactionUtil, CarbonDataMergerUtil, CompactionType}
-import org.apache.carbondata.processing.splits.TableSplit
-import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil, CarbonLoaderUtil, CarbonQueryUtil}
-import org.apache.carbondata.spark._
-import org.apache.carbondata.spark.load.DataLoadProcessBuilderOnSpark
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, Util}
-
-/**
- * This is the factory class which can create different RDD depends on user needs.
- *
- */
-object CarbonDataRDDFactory {
-
-  private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
-  def alterTableForCompaction(sqlContext: SQLContext,
-      alterTableModel: AlterTableModel,
-      carbonLoadModel: CarbonLoadModel,
-      storePath: String,
-      storeLocation: String): Unit = {
-    var compactionSize: Long = 0
-    var compactionType: CompactionType = CompactionType.MINOR_COMPACTION
-    if (alterTableModel.compactionType.equalsIgnoreCase("major")) {
-      compactionSize = CarbonDataMergerUtil.getCompactionSize(CompactionType.MAJOR_COMPACTION)
-      compactionType = CompactionType.MAJOR_COMPACTION
-    } else if (alterTableModel.compactionType
-      .equalsIgnoreCase(CompactionType.IUD_UPDDEL_DELTA_COMPACTION.toString)) {
-      compactionType = CompactionType.IUD_UPDDEL_DELTA_COMPACTION
-      if (alterTableModel.segmentUpdateStatusManager.get != None) {
-        carbonLoadModel
-          .setSegmentUpdateStatusManager((alterTableModel.segmentUpdateStatusManager.get))
-        carbonLoadModel
-          .setLoadMetadataDetails(alterTableModel.segmentUpdateStatusManager.get
-            .getLoadMetadataDetails.toList.asJava)
-      }
-    }
-    else {
-      compactionType = CompactionType.MINOR_COMPACTION
-    }
-
-    LOGGER.audit(s"Compaction request received for table " +
-        s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-    val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
-
-    if (null == carbonLoadModel.getLoadMetadataDetails) {
-      CommonUtil.readLoadMetadataDetails(carbonLoadModel)
-    }
-    // reading the start time of data load.
-    val loadStartTime = CarbonUpdateUtil.readCurrentTime();
-    carbonLoadModel.setFactTimeStamp(loadStartTime)
-
-    val isCompactionTriggerByDDl = true
-    val compactionModel = CompactionModel(compactionSize,
-      compactionType,
-      carbonTable,
-      isCompactionTriggerByDDl
-    )
-
-    val isConcurrentCompactionAllowed = CarbonProperties.getInstance()
-        .getProperty(CarbonCommonConstants.ENABLE_CONCURRENT_COMPACTION,
-          CarbonCommonConstants.DEFAULT_ENABLE_CONCURRENT_COMPACTION
-        )
-        .equalsIgnoreCase("true")
-
-    // if system level compaction is enabled then only one compaction can run in the system
-    // if any other request comes at this time then it will create a compaction request file.
-    // so that this will be taken up by the compaction process which is executing.
-    if (!isConcurrentCompactionAllowed) {
-      LOGGER.info("System level compaction lock is enabled.")
-      handleCompactionForSystemLocking(sqlContext,
-        carbonLoadModel,
-        storePath,
-        storeLocation,
-        compactionType,
-        carbonTable,
-        compactionModel
-      )
-    } else {
-      // normal flow of compaction
-      val lock = CarbonLockFactory
-          .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
-            LockUsage.COMPACTION_LOCK
-          )
-
-      if (lock.lockWithRetries()) {
-        LOGGER.info("Acquired the compaction lock for table" +
-            s" ${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-        try {
-          startCompactionThreads(sqlContext,
-            carbonLoadModel,
-            storePath,
-            storeLocation,
-            compactionModel,
-            lock
-          )
-        } catch {
-          case e: Exception =>
-            LOGGER.error(s"Exception in start compaction thread. ${ e.getMessage }")
-            lock.unlock()
-        }
-      } else {
-        LOGGER.audit("Not able to acquire the compaction lock for table " +
-            s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-        LOGGER.error(s"Not able to acquire the compaction lock for table" +
-            s" ${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-        sys.error("Table is already locked for compaction. Please try after some time.")
-      }
-    }
-  }
-
-  def handleCompactionForSystemLocking(sqlContext: SQLContext,
-      carbonLoadModel: CarbonLoadModel,
-      storePath: String,
-      storeLocation: String,
-      compactionType: CompactionType,
-      carbonTable: CarbonTable,
-      compactionModel: CompactionModel): Unit = {
-    val lock = CarbonLockFactory
-        .getCarbonLockObj(CarbonCommonConstants.SYSTEM_LEVEL_COMPACTION_LOCK_FOLDER,
-          LockUsage.SYSTEMLEVEL_COMPACTION_LOCK
-        )
-    if (lock.lockWithRetries()) {
-      LOGGER.info(s"Acquired the compaction lock for table ${ carbonLoadModel.getDatabaseName }" +
-          s".${ carbonLoadModel.getTableName }")
-      try {
-        startCompactionThreads(sqlContext,
-          carbonLoadModel,
-          storePath,
-          storeLocation,
-          compactionModel,
-          lock
-        )
-      } catch {
-        case e: Exception =>
-          LOGGER.error(s"Exception in start compaction thread. ${ e.getMessage }")
-          lock.unlock()
-          // if the compaction is a blocking call then only need to throw the exception.
-          if (compactionModel.isDDLTrigger) {
-            throw e
-          }
-      }
-    } else {
-      LOGGER.audit("Not able to acquire the system level compaction lock for table " +
-          s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-      LOGGER.error("Not able to acquire the compaction lock for table " +
-          s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-      CarbonCompactionUtil
-          .createCompactionRequiredFile(carbonTable.getMetaDataFilepath, compactionType)
-      // do sys error only in case of DDL trigger.
-      if (compactionModel.isDDLTrigger) {
-        sys.error("Compaction is in progress, compaction request for table " +
-            s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }" +
-            " is in queue.")
-      } else {
-        LOGGER.error("Compaction is in progress, compaction request for table " +
-            s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }" +
-            " is in queue.")
-      }
-    }
-  }
-
-  def startCompactionThreads(sqlContext: SQLContext,
-      carbonLoadModel: CarbonLoadModel,
-      storePath: String,
-      storeLocation: String,
-      compactionModel: CompactionModel,
-      compactionLock: ICarbonLock): Unit = {
-    val executor: ExecutorService = Executors.newFixedThreadPool(1)
-    // update the updated table status.
-    CommonUtil.readLoadMetadataDetails(carbonLoadModel)
-    val compactionThread = new Thread {
-      override def run(): Unit = {
-
-        try {
-          // compaction status of the table which is triggered by the user.
-          var triggeredCompactionStatus = false
-          var exception: Exception = null
-          try {
-            DataManagementFunc.executeCompaction(carbonLoadModel: CarbonLoadModel,
-              compactionModel: CompactionModel,
-              executor, sqlContext, storeLocation
-            )
-            triggeredCompactionStatus = true
-          } catch {
-            case e: Exception =>
-              LOGGER.error(s"Exception in compaction thread ${ e.getMessage }")
-              exception = e
-          }
-          // continue in case of exception also, check for all the tables.
-          val isConcurrentCompactionAllowed = CarbonProperties.getInstance()
-              .getProperty(CarbonCommonConstants.ENABLE_CONCURRENT_COMPACTION,
-                CarbonCommonConstants.DEFAULT_ENABLE_CONCURRENT_COMPACTION
-              ).equalsIgnoreCase("true")
-
-          if (!isConcurrentCompactionAllowed) {
-            LOGGER.info("System level compaction lock is enabled.")
-            val skipCompactionTables = ListBuffer[CarbonTableIdentifier]()
-            var table: CarbonTable = CarbonCompactionUtil
-                .getNextTableToCompact(CarbonEnv.get.carbonMetastore.metadata.
-                  tablesMeta.map(_.carbonTable).toArray,
-                  skipCompactionTables.toList.asJava)
-            while (null != table) {
-              LOGGER.info("Compaction request has been identified for table " +
-                  s"${ table.getDatabaseName }." +
-                  s"${ table.getFactTableName }")
-              val metadataPath = table.getMetaDataFilepath
-              val compactionType = CarbonCompactionUtil.determineCompactionType(metadataPath)
-
-              val newCarbonLoadModel = new CarbonLoadModel()
-              DataManagementFunc.prepareCarbonLoadModel(table, newCarbonLoadModel)
-
-              val compactionSize = CarbonDataMergerUtil
-                  .getCompactionSize(CompactionType.MAJOR_COMPACTION)
-
-              val newcompactionModel = CompactionModel(compactionSize,
-                compactionType,
-                table,
-                compactionModel.isDDLTrigger
-              )
-              // proceed for compaction
-              try {
-                DataManagementFunc.executeCompaction(newCarbonLoadModel,
-                  newcompactionModel,
-                  executor, sqlContext, storeLocation
-                )
-              } catch {
-                case e: Exception =>
-                  LOGGER.error("Exception in compaction thread for table " +
-                      s"${ table.getDatabaseName }." +
-                      s"${ table.getFactTableName }")
-                // not handling the exception. only logging as this is not the table triggered
-                // by user.
-              } finally {
-                // delete the compaction required file in case of failure or success also.
-                if (!CarbonCompactionUtil
-                    .deleteCompactionRequiredFile(metadataPath, compactionType)) {
-                  // if the compaction request file is not been able to delete then
-                  // add those tables details to the skip list so that it wont be considered next.
-                  skipCompactionTables.+=:(table.getCarbonTableIdentifier)
-                  LOGGER.error("Compaction request file can not be deleted for table " +
-                      s"${ table.getDatabaseName }." +
-                      s"${ table.getFactTableName }")
-                }
-              }
-              // ********* check again for all the tables.
-              table = CarbonCompactionUtil
-                  .getNextTableToCompact(CarbonEnv.get.carbonMetastore.metadata
-                      .tablesMeta.map(_.carbonTable).toArray, skipCompactionTables.asJava
-                  )
-            }
-            // giving the user his error for telling in the beeline if his triggered table
-            // compaction is failed.
-            if (!triggeredCompactionStatus) {
-              throw new Exception("Exception in compaction " + exception.getMessage)
-            }
-          }
-        } finally {
-          executor.shutdownNow()
-          DataManagementFunc.deletePartialLoadsInCompaction(carbonLoadModel)
-          compactionLock.unlock()
-        }
-      }
-    }
-    // calling the run method of a thread to make the call as blocking call.
-    // in the future we may make this as concurrent.
-    compactionThread.run()
-  }
-
-  def loadCarbonData(sqlContext: SQLContext,
-      carbonLoadModel: CarbonLoadModel,
-      storePath: String,
-      columnar: Boolean,
-      partitionStatus: String = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS,
-      result: Option[DictionaryServer],
-      overwriteTable: Boolean,
-      dataFrame: Option[DataFrame] = None,
-      updateModel: Option[UpdateTableModel] = None): Unit = {
-    val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
-    val isAgg = false
-    // for handling of the segment Merging.
-    def handleSegmentMerging(): Unit = {
-      LOGGER.info(s"compaction need status is" +
-          s" ${ CarbonDataMergerUtil.checkIfAutoLoadMergingRequired() }")
-      if (CarbonDataMergerUtil.checkIfAutoLoadMergingRequired()) {
-        LOGGER.audit(s"Compaction request received for table " +
-            s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-        val compactionSize = 0
-        val isCompactionTriggerByDDl = false
-        val compactionModel = CompactionModel(compactionSize,
-          CompactionType.MINOR_COMPACTION,
-          carbonTable,
-          isCompactionTriggerByDDl
-        )
-        var storeLocation = ""
-        val configuredStore = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
-        if (null != configuredStore && configuredStore.nonEmpty) {
-          storeLocation = configuredStore(Random.nextInt(configuredStore.length))
-        }
-        if (storeLocation == null) {
-          storeLocation = System.getProperty("java.io.tmpdir")
-        }
-        storeLocation = storeLocation + "/carbonstore/" + System.nanoTime()
-
-        val isConcurrentCompactionAllowed = CarbonProperties.getInstance()
-          .getProperty(CarbonCommonConstants.ENABLE_CONCURRENT_COMPACTION,
-            CarbonCommonConstants.DEFAULT_ENABLE_CONCURRENT_COMPACTION
-          )
-          .equalsIgnoreCase("true")
-
-        if (!isConcurrentCompactionAllowed) {
-
-          handleCompactionForSystemLocking(sqlContext,
-            carbonLoadModel,
-            storePath,
-            storeLocation,
-            CompactionType.MINOR_COMPACTION,
-            carbonTable,
-            compactionModel
-          )
-        } else {
-          val lock = CarbonLockFactory
-              .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
-                LockUsage.COMPACTION_LOCK
-              )
-
-          if (lock.lockWithRetries()) {
-            LOGGER.info("Acquired the compaction lock.")
-            try {
-              startCompactionThreads(sqlContext,
-                carbonLoadModel,
-                storePath,
-                storeLocation,
-                compactionModel,
-                lock
-              )
-            } catch {
-              case e: Exception =>
-                LOGGER.error(s"Exception in start compaction thread. ${ e.getMessage }")
-                lock.unlock()
-                throw e
-            }
-          } else {
-            LOGGER.audit("Not able to acquire the compaction lock for table " +
-                s"${ carbonLoadModel.getDatabaseName }.${
-                  carbonLoadModel
-                      .getTableName
-                }")
-            LOGGER.error("Not able to acquire the compaction lock for table " +
-                s"${ carbonLoadModel.getDatabaseName }.${
-                  carbonLoadModel
-                      .getTableName
-                }")
-          }
-        }
-      }
-    }
-
-    def updateStatus(loadStatus: String,
-        stat: Array[(String, (LoadMetadataDetails, ExecutionErrors))]) = {
-      val metadataDetails = if (stat != null && stat(0) != null) {
-        stat(0)._2._1
-      } else {
-        new LoadMetadataDetails
-      }
-      CarbonLoaderUtil
-        .populateNewLoadMetaEntry(metadataDetails,
-          loadStatus,
-          carbonLoadModel.getFactTimeStamp,
-          true)
-      val status = CarbonLoaderUtil.recordLoadMetadata(metadataDetails,
-        carbonLoadModel, false, overwriteTable)
-      if (!status) {
-        val errorMessage = "Dataload failed due to failure in table status updation."
-        LOGGER.audit("Data load is failed for " +
-                     s"${ carbonLoadModel.getDatabaseName }.${
-                       carbonLoadModel
-                         .getTableName
-                     }")
-        LOGGER.error("Dataload failed due to failure in table status updation.")
-        throw new Exception(errorMessage)
-      }
-    }
-
-    try {
-      LOGGER.audit(s"Data load request has been received for table" +
-          s" ${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-      // Check if any load need to be deleted before loading new data
-      DataManagementFunc.deleteLoadsAndUpdateMetadata(carbonLoadModel.getDatabaseName,
-        carbonLoadModel.getTableName, storePath, false, carbonTable)
-      // get partition way from configuration
-      // val isTableSplitPartition = CarbonProperties.getInstance().getProperty(
-      // CarbonCommonConstants.TABLE_SPLIT_PARTITION,
-      // CarbonCommonConstants.TABLE_SPLIT_PARTITION_DEFAULT_VALUE).toBoolean
-      val isTableSplitPartition = false
-      var blocksGroupBy: Array[(String, Array[BlockDetails])] = null
-      var status: Array[(String, (LoadMetadataDetails, ExecutionErrors))] = null
-      var res: Array[List[(String, (LoadMetadataDetails, ExecutionErrors))]] = null
-
-      def loadDataFile(): Unit = {
-        if (isTableSplitPartition) {
-          /*
-         * when data handle by table split partition
-         * 1) get partition files, direct load or not will get the different files path
-         * 2) get files blocks by using SplitUtils
-         * 3) output Array[(partitionID,Array[BlockDetails])] to blocksGroupBy
-         */
-          var splits = Array[TableSplit]()
-          if (carbonLoadModel.isDirectLoad) {
-            // get all table Splits, this part means files were divide to different partitions
-            splits = CarbonQueryUtil.getTableSplitsForDirectLoad(carbonLoadModel.getFactFilePath)
-            // get all partition blocks from file list
-            blocksGroupBy = splits.map {
-              split =>
-                val pathBuilder = new StringBuilder()
-                for (path <- split.getPartition.getFilesPath.asScala) {
-                  pathBuilder.append(path).append(",")
-                }
-                if (pathBuilder.nonEmpty) {
-                  pathBuilder.substring(0, pathBuilder.size - 1)
-                }
-                (split.getPartition.getUniqueID, SparkUtil.getSplits(pathBuilder.toString(),
-                  sqlContext.sparkContext
-                ))
-            }
-          } else {
-            // get all table Splits,when come to this, means data have been partition
-            splits = CarbonQueryUtil.getTableSplits(carbonLoadModel.getDatabaseName,
-              carbonLoadModel.getTableName, null)
-            // get all partition blocks from factFilePath/uniqueID/
-            blocksGroupBy = splits.map {
-              split =>
-                val pathBuilder = new StringBuilder()
-                pathBuilder.append(carbonLoadModel.getFactFilePath)
-                if (!carbonLoadModel.getFactFilePath.endsWith("/")
-                    && !carbonLoadModel.getFactFilePath.endsWith("\\")) {
-                  pathBuilder.append("/")
-                }
-                pathBuilder.append(split.getPartition.getUniqueID).append("/")
-                (split.getPartition.getUniqueID,
-                  SparkUtil.getSplits(pathBuilder.toString, sqlContext.sparkContext))
-            }
-          }
-        } else {
-          /*
-           * when data load handle by node partition
-           * 1)clone the hadoop configuration,and set the file path to the configuration
-           * 2)use org.apache.hadoop.mapreduce.lib.input.TextInputFormat to get splits,size info
-           * 3)use CarbonLoaderUtil.nodeBlockMapping to get mapping info of node and block,
-           *   for locally writing carbondata files(one file one block) in nodes
-           *   use NewCarbonDataLoadRDD to load data and write to carbondata files
-           */
-          val hadoopConfiguration = new Configuration(sqlContext.sparkContext.hadoopConfiguration)
-          // FileUtils will skip file which is no csv, and return all file path which split by ','
-          val filePaths = carbonLoadModel.getFactFilePath
-          hadoopConfiguration.set(FileInputFormat.INPUT_DIR, filePaths)
-          hadoopConfiguration.set(FileInputFormat.INPUT_DIR_RECURSIVE, "true")
-          hadoopConfiguration.set("io.compression.codecs",
-            """org.apache.hadoop.io.compress.GzipCodec,
-               org.apache.hadoop.io.compress.DefaultCodec,
-               org.apache.hadoop.io.compress.BZip2Codec""".stripMargin)
-
-          CommonUtil.configSplitMaxSize(sqlContext.sparkContext, filePaths, hadoopConfiguration)
-
-          val inputFormat = new org.apache.hadoop.mapreduce.lib.input.TextInputFormat
-          val jobContext = new Job(hadoopConfiguration)
-          val rawSplits = inputFormat.getSplits(jobContext).toArray
-          val blockList = rawSplits.map { inputSplit =>
-            val fileSplit = inputSplit.asInstanceOf[FileSplit]
-            new TableBlockInfo(fileSplit.getPath.toString,
-              fileSplit.getStart, "1",
-              fileSplit.getLocations, fileSplit.getLength, ColumnarFormatVersion.V1, null
-            ).asInstanceOf[Distributable]
-          }
-          // group blocks to nodes, tasks
-          val startTime = System.currentTimeMillis
-          val activeNodes = DistributionUtil
-              .ensureExecutorsAndGetNodeList(blockList, sqlContext.sparkContext)
-          val nodeBlockMapping =
-            CarbonLoaderUtil
-                .nodeBlockMapping(blockList.toSeq.asJava, -1, activeNodes.toList.asJava).asScala
-                .toSeq
-          val timeElapsed: Long = System.currentTimeMillis - startTime
-          LOGGER.info("Total Time taken in block allocation: " + timeElapsed)
-          LOGGER.info(s"Total no of blocks: ${ blockList.length }, " +
-              s"No.of Nodes: ${nodeBlockMapping.size}")
-          var str = ""
-          nodeBlockMapping.foreach(entry => {
-            val tableBlock = entry._2
-            str = str + "#Node: " + entry._1 + " no.of.blocks: " + tableBlock.size()
-            tableBlock.asScala.foreach(tableBlockInfo =>
-              if (!tableBlockInfo.getLocations.exists(hostentry =>
-                hostentry.equalsIgnoreCase(entry._1)
-              )) {
-                str = str + " , mismatch locations: " + tableBlockInfo.getLocations
-                    .foldLeft("")((a, b) => a + "," + b)
-              }
-            )
-            str = str + "\n"
-          }
-          )
-          LOGGER.info(str)
-          blocksGroupBy = nodeBlockMapping.map(entry => {
-            val blockDetailsList =
-              entry._2.asScala.map(distributable => {
-                val tableBlock = distributable.asInstanceOf[TableBlockInfo]
-                new BlockDetails(new Path(tableBlock.getFilePath),
-                  tableBlock.getBlockOffset, tableBlock.getBlockLength, tableBlock.getLocations
-                )
-              }).toArray
-            (entry._1, blockDetailsList)
-          }
-          ).toArray
-        }
-
-        status = new NewCarbonDataLoadRDD(sqlContext.sparkContext,
-          new DataLoadResultImpl(),
-          carbonLoadModel,
-          blocksGroupBy,
-          isTableSplitPartition).collect()
-      }
-
-      def loadDataFrame(): Unit = {
-        try {
-          val rdd = dataFrame.get.rdd
-          val nodeNumOfData = rdd.partitions.flatMap[String, Array[String]] { p =>
-            DataLoadPartitionCoalescer.getPreferredLocs(rdd, p).map(_.host)
-          }.distinct.size
-          val nodes = DistributionUtil.ensureExecutorsByNumberAndGetNodeList(nodeNumOfData,
-            sqlContext.sparkContext)
-          val newRdd = new DataLoadCoalescedRDD[Row](rdd, nodes.toArray.distinct)
-          var numPartitions = DistributionUtil.getNodeList(sqlContext.sparkContext).length
-          numPartitions = Math.max(1, Math.min(numPartitions, rdd.partitions.length))
-          val coalesceRdd = rdd.coalesce(numPartitions, shuffle = false)
-
-          status = new NewDataFrameLoaderRDD(sqlContext.sparkContext,
-            new DataLoadResultImpl(),
-            carbonLoadModel,
-            newRdd).collect()
-        } catch {
-          case ex: Exception =>
-            LOGGER.error(ex, "load data frame failed")
-            throw ex
-        }
-      }
-
-      def loadDataFrameForUpdate(): Unit = {
-        val segmentUpdateParallelism = CarbonProperties.getInstance().getParallelismForSegmentUpdate
-
-        def triggerDataLoadForSegment(key: String, taskNo: Int,
-            iter: Iterator[Row]): Iterator[(String, (LoadMetadataDetails, ExecutionErrors))] = {
-          val rddResult = new updateResultImpl()
-          val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
-          val resultIter = new Iterator[(String, (LoadMetadataDetails, ExecutionErrors))] {
-            var partitionID = "0"
-            val loadMetadataDetails = new LoadMetadataDetails
-            val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
-            var uniqueLoadStatusId = ""
-            try {
-              val segId = key
-              val index = taskNo
-              uniqueLoadStatusId = carbonLoadModel.getTableName +
-                                   CarbonCommonConstants.UNDERSCORE +
-                                   (index + "_0")
-
-              // convert timestamp
-              val timeStampInLong = updateModel.get.updatedTimeStamp + ""
-              loadMetadataDetails.setPartitionCount(partitionID)
-              loadMetadataDetails.setLoadName(segId)
-              loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)
-              carbonLoadModel.setPartitionId(partitionID)
-              carbonLoadModel.setSegmentId(segId)
-              carbonLoadModel.setTaskNo(String.valueOf(index))
-              carbonLoadModel.setFactTimeStamp(updateModel.get.updatedTimeStamp)
-
-              // During Block Spill case Increment of File Count and proper adjustment of Block
-              // naming is only done when AbstractFactDataWriter.java : initializeWriter get
-              // CarbondataFileName as null. For handling Block Spill not setting the
-              // CarbondataFileName in case of Update.
-              // carbonLoadModel.setCarbondataFileName(newBlockName)
-
-              // storeLocation = CarbonDataLoadRDD.initialize(carbonLoadModel, index)
-              loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS)
-              UpdateDataLoad.DataLoadForUpdate(segId,
-                index,
-                iter,
-                carbonLoadModel,
-                loadMetadataDetails)
-            } catch {
-              case e: NoRetryException =>
-                loadMetadataDetails
-                  .setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
-                executionErrors.failureCauses = FailureCauses.BAD_RECORDS
-                executionErrors.errorMsg = e.getMessage
-                LOGGER.info("Bad Record Found")
-              case e: Exception =>
-                LOGGER.info("DataLoad failure")
-                LOGGER.error(e)
-                throw e
-            }
-
-            var finished = false
-
-            override def hasNext: Boolean = !finished
-
-            override def next(): (String, (LoadMetadataDetails, ExecutionErrors)) = {
-              finished = true
-              rddResult
-                .getKey(uniqueLoadStatusId,
-                  (loadMetadataDetails, executionErrors))
-            }
-          }
-          resultIter
-        }
-
-        val updateRdd = dataFrame.get.rdd
-
-        // return directly if no rows to update
-        val noRowsToUpdate = updateRdd.isEmpty()
-        if (noRowsToUpdate) {
-          res = Array[List[(String, (LoadMetadataDetails, ExecutionErrors))]]()
-          return
-        }
-
-        // splitting as (key, value) i.e., (segment, updatedRows)
-        val keyRDD = updateRdd.map(row =>
-          (row.get(row.size - 1).toString, Row(row.toSeq.slice(0, row.size - 1): _*)))
-
-        val loadMetadataDetails = SegmentStatusManager.readLoadMetadata(
-          carbonTable.getMetaDataFilepath)
-        val segmentIds = loadMetadataDetails.map(_.getLoadName)
-        val segmentIdIndex = segmentIds.zipWithIndex.toMap
-        val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonLoadModel.getStorePath,
-          carbonTable.getCarbonTableIdentifier)
-        val segmentId2maxTaskNo = segmentIds.map { segId =>
-          (segId, CarbonUpdateUtil.getLatestTaskIdForSegment(segId, carbonTablePath))
-        }.toMap
-
-        class SegmentPartitioner(segIdIndex: Map[String, Int], parallelism: Int)
-          extends org.apache.spark.Partitioner {
-          override def numPartitions: Int = segmentIdIndex.size * parallelism
-
-          override def getPartition(key: Any): Int = {
-            val segId = key.asInstanceOf[String]
-            // partitionId
-            segmentIdIndex(segId) * parallelism + Random.nextInt(parallelism)
-          }
-        }
-
-        val partitionByRdd = keyRDD.partitionBy(new SegmentPartitioner(segmentIdIndex,
-          segmentUpdateParallelism))
-
-        // because partitionId=segmentIdIndex*parallelism+RandomPart and RandomPart<parallelism,
-        // so segmentIdIndex=partitionId/parallelism, this has been verified.
-        res = partitionByRdd.map(_._2).mapPartitions { partition =>
-          val partitionId = TaskContext.getPartitionId()
-          val segIdIndex = partitionId / segmentUpdateParallelism
-          val randomPart = partitionId - segIdIndex * segmentUpdateParallelism
-          val segId = segmentIds(segIdIndex)
-          val newTaskNo = segmentId2maxTaskNo(segId) + randomPart + 1
-
-          List(triggerDataLoadForSegment(segId, newTaskNo, partition).toList).toIterator
-        }.collect()
-      }
-
-      def loadDataForPartitionTable(): Unit = {
-        try {
-          val rdd = repartitionInputData(sqlContext, dataFrame, carbonLoadModel)
-          status = new PartitionTableDataLoaderRDD(sqlContext.sparkContext,
-            new DataLoadResultImpl(),
-            carbonLoadModel,
-            rdd).collect()
-        } catch {
-          case ex: Exception =>
-            LOGGER.error(ex, "load data failed for partition table")
-            throw ex
-        }
-      }
-
-      // create new segment folder  in carbon store
-      if (!updateModel.isDefined) {
-        CarbonLoaderUtil.checkAndCreateCarbonDataLocation(storePath,
-          carbonLoadModel.getSegmentId, carbonTable)
-      }
-      var loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
-      var errorMessage: String = "DataLoad failure"
-      var executorMessage: String = ""
-      val configuration = DataLoadProcessBuilder.createConfiguration(carbonLoadModel)
-      val sortScope = CarbonDataProcessorUtil.getSortScope(configuration)
-      try {
-        if (updateModel.isDefined) {
-          loadDataFrameForUpdate()
-        } else if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null) {
-          loadDataForPartitionTable()
-        } else if (configuration.isSortTable &&
-            sortScope.equals(SortScopeOptions.SortScope.GLOBAL_SORT)) {
-          LOGGER.audit("Using global sort for loading.")
-          status = DataLoadProcessBuilderOnSpark.loadDataUsingGlobalSort(sqlContext.sparkContext,
-            dataFrame, carbonLoadModel)
-        } else if (dataFrame.isDefined) {
-          loadDataFrame()
-        } else {
-          loadDataFile()
-        }
-        if (updateModel.isDefined) {
-
-          res.foreach(resultOfSeg => resultOfSeg.foreach(
-            resultOfBlock => {
-              if (resultOfBlock._2._1.getLoadStatus
-                .equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)) {
-                loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
-                if (resultOfBlock._2._2.failureCauses == FailureCauses.NONE) {
-                  updateModel.get.executorErrors.failureCauses = FailureCauses.EXECUTOR_FAILURE
-                  updateModel.get.executorErrors.errorMsg = "Failure in the Executor."
-                }
-                else {
-                  updateModel.get.executorErrors = resultOfBlock._2._2
-                }
-              } else if (resultOfBlock._2._1.getLoadStatus
-                .equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)) {
-                loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
-                updateModel.get.executorErrors.failureCauses = resultOfBlock._2._2.failureCauses
-                updateModel.get.executorErrors.errorMsg = resultOfBlock._2._2.errorMsg
-              }
-            }
-          ))
-
-        }
-        else {
-          val newStatusMap = scala.collection.mutable.Map.empty[String, String]
-        if (status.nonEmpty) {
-          status.foreach { eachLoadStatus =>
-            val state = newStatusMap.get(eachLoadStatus._1)
-            state match {
-              case Some(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) =>
-                newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2._1.getLoadStatus)
-              case Some(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
-                if eachLoadStatus._2._1.getLoadStatus ==
-                    CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS =>
-                newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2._1.getLoadStatus)
-              case _ =>
-                newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2._1.getLoadStatus)
-            }
-          }
-
-          newStatusMap.foreach {
-            case (key, value) =>
-              if (value == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
-                loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
-              } else if (value == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
-                  !loadStatus.equals(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)) {
-                loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
-              }
-          }
-        } else {
-          loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
-        }
-
-        if (loadStatus != CarbonCommonConstants.STORE_LOADSTATUS_FAILURE &&
-            partitionStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS) {
-          loadStatus = partitionStatus
-        }
-      }
-      } catch {
-        case ex: Throwable =>
-          loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
-          ex match {
-            case sparkException: SparkException =>
-              if (sparkException.getCause.isInstanceOf[DataLoadingException] ||
-                  sparkException.getCause.isInstanceOf[CarbonDataLoadingException]) {
-                executorMessage = sparkException.getCause.getMessage
-                errorMessage = errorMessage + ": " + executorMessage
-              }
-            case _ =>
-              executorMessage = ex.getCause.getMessage
-              errorMessage = errorMessage + ": " + executorMessage
-          }
-          LOGGER.info(errorMessage)
-          LOGGER.error(ex)
-      }
-      // handle the status file updation for the update cmd.
-      if (updateModel.isDefined) {
-
-      if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
-      // updateModel.get.executorErrors.errorMsg = errorMessage
-          if (updateModel.get.executorErrors.failureCauses == FailureCauses.NONE) {
-            updateModel.get.executorErrors.failureCauses = FailureCauses.EXECUTOR_FAILURE
-            if (null != executorMessage && !executorMessage.isEmpty) {
-              updateModel.get.executorErrors.errorMsg = executorMessage
-            } else {
-              updateModel.get.executorErrors.errorMsg = "Update failed as the data load has failed."
-            }
-          }
-          return
-        } else if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
-                   updateModel.get.executorErrors.failureCauses == FailureCauses.BAD_RECORDS &&
-                   carbonLoadModel.getBadRecordsAction.split(",")(1) == LoggerAction.FAIL.name) {
-          return
-        } else {
-          // in success case handle updation of the table status file.
-          // success case.
-          val segmentDetails = new util.HashSet[String]()
-
-          var resultSize = 0
-
-          res.foreach(resultOfSeg => {
-            resultSize = resultSize + resultOfSeg.size
-            resultOfSeg.foreach(
-            resultOfBlock => {
-              segmentDetails.add(resultOfBlock._2._1.getLoadName)
-            }
-          )}
-          )
-
-          // this means that the update doesnt have any records to update so no need to do table
-          // status file updation.
-          if (resultSize == 0) {
-            LOGGER.audit("Data update is successful with 0 rows updation for " +
-                         s"${carbonLoadModel.getDatabaseName}.${carbonLoadModel.getTableName}")
-            return
-          }
-
-          if (
-            CarbonUpdateUtil
-              .updateTableMetadataStatus(segmentDetails,
-                carbonTable,
-                updateModel.get.updatedTimeStamp + "",
-                true,
-                new util.ArrayList[String](0))) {
-            LOGGER.audit("Data update is successful for " +
-                         s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-          }
-          else {
-            val errorMessage = "Data update failed due to failure in table status updation."
-            LOGGER.audit("Data update is failed for " +
-                         s"${carbonLoadModel.getDatabaseName}.${carbonLoadModel.getTableName}")
-            LOGGER.error("Data update failed due to failure in table status updation.")
-            updateModel.get.executorErrors.errorMsg = errorMessage
-            updateModel.get.executorErrors.failureCauses = FailureCauses
-              .STATUS_FILE_UPDATION_FAILURE
-            return
-          }
-
-        }
-
-        return
-      }
-      if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
-        LOGGER.info("********starting clean up**********")
-        CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
-        LOGGER.info("********clean up done**********")
-        LOGGER.audit(s"Data load is failed for " +
-            s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-        LOGGER.warn("Cannot write load metadata file as data load failed")
-        updateStatus(loadStatus, status)
-        throw new Exception(errorMessage)
-      } else {
-        // check if data load fails due to bad record and throw data load failure due to
-        // bad record exception
-        if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
-            status(0)._2._2.failureCauses == FailureCauses.BAD_RECORDS &&
-            carbonLoadModel.getBadRecordsAction.split(",")(1) == LoggerAction.FAIL.name) {
-          LOGGER.info("********starting clean up**********")
-          CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
-          LOGGER.info("********clean up done**********")
-          LOGGER.audit(s"Data load is failed for " +
-                       s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-          updateStatus(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE, status)
-          throw new Exception(status(0)._2._2.errorMsg)
-        }
-        if (!isAgg) {
-            writeDictionary(carbonLoadModel, result)
-            updateStatus(loadStatus, status)
-        } else if (!carbonLoadModel.isRetentionRequest) {
-          // TODO : Handle it
-          LOGGER.info("********Database updated**********")
-        }
-
-        if (CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS.equals(loadStatus)) {
-          LOGGER.audit("Data load is partially successful for " +
-                       s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-        } else {
-          LOGGER.audit("Data load is successful for " +
-                       s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
-        }
-        try {
-          // compaction handling
-          handleSegmentMerging()
-        } catch {
-          case e: Exception =>
-            throw new Exception(
-              "Dataload is success. Auto-Compaction has failed. Please check logs.")
-        }
-      }
-    }
-
-  }
-
-  /**
-   * repartition the input data for partiton table.
-   * @param sqlContext
-   * @param dataFrame
-   * @param carbonLoadModel
-   * @return
-   */
-  private def repartitionInputData(sqlContext: SQLContext,
-      dataFrame: Option[DataFrame],
-      carbonLoadModel: CarbonLoadModel): RDD[Row] = {
-    val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
-    val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
-    val partitionColumn = partitionInfo.getColumnSchemaList.get(0).getColumnName
-    val partitionColumnDataType = partitionInfo.getColumnSchemaList.get(0).getDataType
-    val columns = carbonLoadModel.getCsvHeaderColumns
-    var partitionColumnIndex = -1
-    breakable {
-      for (i <- 0 until columns.length) {
-        if (partitionColumn.equalsIgnoreCase(columns(i))) {
-          partitionColumnIndex = i
-          break
-        }
-      }
-    }
-    if (partitionColumnIndex == -1) {
-      throw new DataLoadingException("Partition column not found.")
-    }
-
-    val dateFormatMap = CarbonDataProcessorUtil.getDateFormatMap(carbonLoadModel.getDateFormat())
-    val specificFormat = Option(dateFormatMap.get(partitionColumn.toLowerCase))
-    val timeStampFormat = if (specificFormat.isDefined) {
-      new SimpleDateFormat(specificFormat.get)
-    } else {
-      val timestampFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
-        .CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
-      new SimpleDateFormat(timestampFormatString)
-    }
-
-    val dateFormat = if (specificFormat.isDefined) {
-      new SimpleDateFormat(specificFormat.get)
-    } else {
-      val dateFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
-        .CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
-      new SimpleDateFormat(dateFormatString)
-    }
-
-    // generate RDD[(K, V)] to use the partitionBy method of PairRDDFunctions
-    val inputRDD: RDD[(String, Row)] = if (dataFrame.isDefined) {
-      // input data from DataFrame
-      val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
-      val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
-      val serializationNullFormat =
-        carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
-      dataFrame.get.rdd.map { row =>
-        if (null != row && row.length > partitionColumnIndex &&
-          null != row.get(partitionColumnIndex)) {
-          (CarbonScalaUtil.getString(row.get(partitionColumnIndex), serializationNullFormat,
-            delimiterLevel1, delimiterLevel2, timeStampFormat, dateFormat), row)
-        } else {
-          (null, row)
-        }
-      }
-    } else {
-      // input data from csv files
-      val hadoopConfiguration = new Configuration()
-      CommonUtil.configureCSVInputFormat(hadoopConfiguration, carbonLoadModel)
-      hadoopConfiguration.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
-      val columnCount = columns.length
-      new NewHadoopRDD[NullWritable, StringArrayWritable](
-        sqlContext.sparkContext,
-        classOf[CSVInputFormat],
-        classOf[NullWritable],
-        classOf[StringArrayWritable],
-        hadoopConfiguration
-      ).map { currentRow =>
-        if (null == currentRow || null == currentRow._2) {
-          val row = new StringArrayRow(new Array[String](columnCount))
-          (null, row)
-        } else {
-          val row = new StringArrayRow(new Array[String](columnCount))
-          val values = currentRow._2.get()
-          if (values != null && values.length > partitionColumnIndex) {
-            (currentRow._2.get()(partitionColumnIndex), row.setValues(currentRow._2.get()))
-          } else {
-            (null, row.setValues(currentRow._2.get()))
-          }
-        }
-      }
-    }
-
-    val partitioner = PartitionFactory.getPartitioner(partitionInfo)
-    if (partitionColumnDataType == DataTypes.STRING) {
-      if (partitionInfo.getPartitionType == PartitionType.RANGE) {
-        inputRDD.map { row => (ByteUtil.toBytes(row._1), row._2) }
-          .partitionBy(partitioner)
-          .map(_._2)
-      } else {
-        inputRDD.partitionBy(partitioner)
-          .map(_._2)
-      }
-    } else {
-      inputRDD.map { row =>
-        (PartitionUtil.getDataBasedOnDataType(row._1, partitionColumnDataType, timeStampFormat,
-          dateFormat), row._2)
-      }
-        .partitionBy(partitioner)
-        .map(_._2)
-    }
-  }
-
-    private def writeDictionary(carbonLoadModel: CarbonLoadModel,
-        result: Option[DictionaryServer]) = {
-    // write dictionary file and shutdown dictionary server
-    val uniqueTableName: String = s"${ carbonLoadModel.getDatabaseName }_${
-      carbonLoadModel.getTableName }"
-    result match {
-      case Some(server) =>
-        try {
-          server.writeTableDictionary(carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
-            .getCarbonTableIdentifier.getTableId)
-        } catch {
-          case ex: Exception =>
-            LOGGER.error(s"Error while writing dictionary file for $uniqueTableName")
-            throw new Exception("Dataload failed due to error while writing dictionary file!")
-        }
-      case _ =>
-    }
-  }
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
deleted file mode 100644
index f8275d1..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.thriftserver
-
-import java.io.File
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.CarbonContext
-import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.util.CarbonProperties
-
-object CarbonThriftServer {
-
-  def main(args: Array[String]): Unit = {
-    val conf = new SparkConf()
-      .setAppName("Carbon Thrift Server")
-    if (!conf.contains("carbon.properties.filepath")) {
-      val sparkHome = System.getenv.get("SPARK_HOME")
-      if (sparkHome != null) {
-        val file = new File(sparkHome + '/' + "conf" + '/' + "carbon.properties")
-        if (file.exists()) {
-          conf.set("carbon.properties.filepath", file.getCanonicalPath)
-          System.setProperty("carbon.properties.filepath", file.getCanonicalPath)
-        }
-      }
-    } else {
-      System.setProperty("carbon.properties.filepath", conf.get("carbon.properties.filepath"))
-    }
-    if (org.apache.spark.SPARK_VERSION.startsWith("1.6")) {
-      conf.set("spark.sql.hive.thriftServer.singleSession", "true")
-    }
-    val sc = new SparkContext(conf)
-    val warmUpTime = CarbonProperties.getInstance().getProperty("carbon.spark.warmUpTime", "5000")
-    try {
-      Thread.sleep(Integer.parseInt(warmUpTime))
-    } catch {
-      case e: Exception =>
-        val LOG = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-        LOG.error(s"Wrong value for carbon.spark.warmUpTime $warmUpTime " +
-                  "Using default Value and proceeding")
-        Thread.sleep(30000)
-    }
-    val storePath = if (args.length > 0) args.head else null
-    val cc = new CarbonContext(sc, storePath)
-
-    HiveThriftServer2.startWithContext(cc)
-  }
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
deleted file mode 100644
index 118249a..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.util
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.hive.{CarbonMetaData, DictionaryMap}
-
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.CarbonUtil
-
-case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
-
-object CarbonSparkUtil {
-
-  def createSparkMeta(carbonTable: CarbonTable): CarbonMetaData = {
-    val dimensionsAttr = carbonTable.getDimensionByTableName(carbonTable.getFactTableName)
-        .asScala.map(x => x.getColName) // wf : may be problem
-    val measureAttr = carbonTable.getMeasureByTableName(carbonTable.getFactTableName)
-        .asScala.map(x => x.getColName)
-    val dictionary =
-      carbonTable.getDimensionByTableName(carbonTable.getFactTableName).asScala.map { f =>
-        (f.getColName.toLowerCase,
-            f.hasEncoding(Encoding.DICTIONARY) && !f.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
-                !f.getDataType.isComplexType)
-      }
-    CarbonMetaData(dimensionsAttr, measureAttr, carbonTable, DictionaryMap(dictionary.toMap))
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
deleted file mode 100644
index 4950227..0000000
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.util
-
-import scala.reflect.ClassTag
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.Job
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
-
-
-/**
- * All the utility functions for carbon plan creation
- */
-object QueryPlanUtil {
-
-  /**
-   * createCarbonInputFormat from query model
-   */
-  def createCarbonInputFormat(absoluteTableIdentifier: AbsoluteTableIdentifier) :
-  (CarbonTableInputFormat[Array[Object]], Job) = {
-    val carbonInputFormat = new CarbonTableInputFormat[Array[Object]]()
-    val jobConf: JobConf = new JobConf(new Configuration)
-    val job: Job = new Job(jobConf)
-    FileInputFormat.addInputPath(job, new Path(absoluteTableIdentifier.getTablePath))
-    (carbonInputFormat, job)
-  }
-
-  def createCarbonInputFormat[V: ClassTag](absoluteTableIdentifier: AbsoluteTableIdentifier,
-      conf: Configuration) : CarbonTableInputFormat[V] = {
-    val carbonInputFormat = new CarbonTableInputFormat[V]()
-    val job: Job = new Job(conf)
-    FileInputFormat.addInputPath(job, new Path(absoluteTableIdentifier.getTablePath))
-    carbonInputFormat
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/CarbonInputMetrics.scala b/integration/spark/src/main/scala/org/apache/spark/CarbonInputMetrics.scala
deleted file mode 100644
index ea75ccb..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/CarbonInputMetrics.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark
-
-import java.lang.Long
-
-import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.executor.{DataReadMethod, InputMetrics}
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.hadoop.{CarbonMultiBlockSplit, InputMetricsStats}
-import org.apache.carbondata.spark.InitInputMetrics
-
-/**
- * It gives statistics of number of bytes and record read
- */
-class CarbonInputMetrics extends InitInputMetrics {
-  @transient val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
-  var inputMetrics: InputMetrics = _
-  var bytesReadCallback: Option[() => scala.Long] = _
-  var carbonMultiBlockSplit: CarbonMultiBlockSplit = _
-
-  def initBytesReadCallback(context: TaskContext,
-                            carbonMultiBlockSplit: CarbonMultiBlockSplit) {
-    inputMetrics = context.taskMetrics().getInputMetricsForReadMethod(DataReadMethod.Hadoop)
-    this.carbonMultiBlockSplit = carbonMultiBlockSplit;
-    bytesReadCallback = carbonMultiBlockSplit match {
-      case _: CarbonMultiBlockSplit =>
-        SparkHadoopUtil.get.getFSBytesReadOnThreadCallback()
-      case _ => None
-    }
-  }
-
-  def incrementRecordRead(recordRead: Long) {
-    inputMetrics.incRecordsRead(recordRead)
-  }
-
-  def updateAndClose() {
-    if (bytesReadCallback.isDefined) {
-      inputMetrics.updateBytesRead()
-    } else if (carbonMultiBlockSplit.isInstanceOf[CarbonMultiBlockSplit]) {
-      // If we can't get the bytes read from the FS stats, fall back to the split size,
-      // which may be inaccurate.
-      try {
-        inputMetrics.incBytesRead(carbonMultiBlockSplit.getLength)
-      } catch {
-        case e: java.io.IOException =>
-          LOGGER.warn("Unable to get input size to set InputMetrics for task:" + e.getMessage)
-      }
-    }
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
deleted file mode 100644
index a6c28a9..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExprId, LeafExpression, NamedExpression}
-import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
-import org.apache.spark.sql.sources.Filter
-import org.apache.spark.sql.types.DataType
-
-import org.apache.carbondata.core.scan.expression.ColumnExpression
-
-case class CastExpr(expr: Expression) extends Filter
-
-case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nullable: Boolean)
-  extends LeafExpression with NamedExpression with CodegenFallback {
-
-  type EvaluatedType = Any
-
-  override def toString: String = s"input[" + colExp.getColIndex + "]"
-
-  override def eval(input: InternalRow): Any = input.get(colExp.getColIndex, dataType)
-
-  override def name: String = colExp.getColumnName
-
-  override def toAttribute: Attribute = throw new UnsupportedOperationException
-
-  override def exprId: ExprId = throw new UnsupportedOperationException
-
-  override def qualifiers: Seq[String] = throw new UnsupportedOperationException
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
deleted file mode 100644
index 024c54b..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.logical.{ UnaryNode, _ }
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.sql.optimizer.CarbonDecoderRelation
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.spark.CarbonAliasDecoderRelation
-import org.apache.carbondata.spark.util.CommonUtil
-
-/**
- * Top command
- */
-case class Top(count: Int, topOrBottom: Int, dim: NamedExpression, msr: NamedExpression,
-    child: LogicalPlan) extends UnaryNode {
-  def output: Seq[Attribute] = child.output
-
-  override def references: AttributeSet = {
-    val list = List(dim, msr)
-    AttributeSet(list.flatMap(_.references))
-  }
-}
-
-object getDB {
-
-  def getDatabaseName(dbName: Option[String], sqlContext: SQLContext): String = {
-    dbName.getOrElse(sqlContext.asInstanceOf[HiveContext].catalog.client.currentDatabase)
-  }
-
-}
-
-/**
- * Shows Loads in a table
- */
-case class ShowLoadsCommand(databaseNameOp: Option[String], table: String, limit: Option[String])
-  extends LogicalPlan with Command {
-
-  override def children: Seq[LogicalPlan] = Seq.empty
-
-  override def output: Seq[Attribute] = {
-    Seq(AttributeReference("SegmentSequenceId", StringType, nullable = false)(),
-      AttributeReference("Status", StringType, nullable = false)(),
-      AttributeReference("Load Start Time", TimestampType, nullable = false)(),
-      AttributeReference("Load End Time", TimestampType, nullable = false)())
-  }
-}
-
-/**
- * Describe formatted for hive table
- */
-case class DescribeFormattedCommand(sql: String, tblIdentifier: TableIdentifier)
-  extends LogicalPlan with Command {
-  override def children: Seq[LogicalPlan] = Seq.empty
-
-  override def output: Seq[AttributeReference] =
-    Seq(AttributeReference("result", StringType, nullable = false)())
-}
-
-case class CarbonDictionaryCatalystDecoder(
-    relations: Seq[CarbonDecoderRelation],
-    profile: CarbonProfile,
-    aliasMap: CarbonAliasDecoderRelation,
-    isOuter: Boolean,
-    child: LogicalPlan) extends UnaryNode {
-  override def output: Seq[Attribute] = child.output
-}
-
-abstract class CarbonProfile(attributes: Seq[Attribute]) extends Serializable {
-  def isEmpty: Boolean = attributes.isEmpty
-}
-
-case class IncludeProfile(attributes: Seq[Attribute]) extends CarbonProfile(attributes)
-
-case class ExcludeProfile(attributes: Seq[Attribute]) extends CarbonProfile(attributes)
-
-case class CreateDatabase(dbName: String, sql: String) extends LogicalPlan with Command {
-  override def children: Seq[LogicalPlan] = Seq.empty
-  override def output: Seq[AttributeReference] = {
-    Seq()
-  }
-}
-
-case class DropDatabase(dbName: String, isCascade: Boolean, sql: String)
-    extends LogicalPlan with Command {
-  override def children: Seq[LogicalPlan] = Seq.empty
-  override def output: Seq[AttributeReference] = {
-    Seq()
-  }
-}
-
-case class UseDatabase(sql: String) extends LogicalPlan with Command {
-  override def children: Seq[LogicalPlan] = Seq.empty
-  override def output: Seq[AttributeReference] = {
-    Seq()
-  }
-}
-
-case class ProjectForUpdate(
-    table: UnresolvedRelation,
-    columns: List[String],
-    children: Seq[LogicalPlan] ) extends LogicalPlan with Command {
-  override def output: Seq[AttributeReference] = Seq.empty
-}
-
-case class UpdateTable(
-    table: UnresolvedRelation,
-    columns: List[String],
-    selectStmt: String,
-    filer: String) extends LogicalPlan {
-  override def children: Seq[LogicalPlan] = Seq.empty
-  override def output: Seq[AttributeReference] = Seq.empty
-}
-
-case class DeleteRecords(
-    statement: String,
-    table: UnresolvedRelation) extends LogicalPlan {
-  override def children: Seq[LogicalPlan] = Seq.empty
-  override def output: Seq[AttributeReference] = Seq.empty
-}
-
-case class ShowPartitions(
-    table: TableIdentifier) extends LogicalPlan {
-  override def children: Seq[LogicalPlan] = Seq.empty
-  override def output: Seq[Attribute] = CommonUtil.partitionInfoOutput
-}
-
-/**
- * A logical plan representing insertion into Hive table.
- * This plan ignores nullability of ArrayType, MapType, StructType unlike InsertIntoTable
- * because Hive table doesn't have nullability for ARRAY, MAP, STRUCT types.
- */
-case class InsertIntoCarbonTable(
-    table: CarbonDatasourceRelation,
-    partition: Map[String, Option[String]],
-    child: LogicalPlan,
-    overwrite: Boolean,
-    ifNotExists: Boolean)
-  extends LogicalPlan with Command {
-
-  override def children: Seq[LogicalPlan] = child :: Nil
-  override def output: Seq[Attribute] = Seq.empty
-
-  // This is the expected schema of the table prepared to be inserted into,
-  // including dynamic partition columns.
-  val tableOutput = table.carbonRelation.output
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
deleted file mode 100644
index da4b210..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import java.io.File
-
-import scala.language.implicitConversions
-
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.catalyst.ParserDialect
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, OverrideCatalog}
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.ExtractPythonUDFs
-import org.apache.spark.sql.execution.datasources.{PreInsertCastAndRename, PreWriteCheck}
-import org.apache.spark.sql.hive._
-import org.apache.spark.sql.optimizer.CarbonOptimizer
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory}
-
-class CarbonContext(
-    val sc: SparkContext,
-    val storePath: String,
-    metaStorePath: String) extends HiveContext(sc) {
-  self =>
-
-  def this(sc: SparkContext) = {
-    this(sc,
-      null,
-      new File(CarbonCommonConstants.METASTORE_LOCATION_DEFAULT_VAL).getCanonicalPath)
-  }
-
-  def this(sc: SparkContext, storePath: String) = {
-    this(sc,
-      storePath,
-      new File(CarbonCommonConstants.METASTORE_LOCATION_DEFAULT_VAL).getCanonicalPath)
-  }
-
-  CarbonContext.addInstance(sc, this)
-  CodeGenerateFactory.init(sc.version)
-  udf.register("getTupleId", () => "")
-  CarbonEnv.init(this)
-
-  var lastSchemaUpdatedTime = System.currentTimeMillis()
-  val hiveClientInterface = metadataHive
-
-  protected[sql] override lazy val conf: SQLConf = new CarbonSQLConf
-
-  @transient
-  override lazy val catalog = {
-    val carbonProperties = CarbonProperties.getInstance()
-    if (storePath != null) {
-      carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION, storePath)
-      // In case if it is in carbon.properties for backward compatible
-    } else if (carbonProperties.getProperty(CarbonCommonConstants.STORE_LOCATION) == null) {
-      carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION,
-        conf.getConfString("spark.sql.warehouse.dir"))
-    }
-    new CarbonMetastore(this, storePath, metadataHive, queryId) with OverrideCatalog
-  }
-
-  @transient
-  override protected[sql] lazy val analyzer =
-    new Analyzer(catalog, functionRegistry, conf) {
-
-      override val extendedResolutionRules =
-        catalog.ParquetConversions ::
-        catalog.CreateTables ::
-        CarbonIUDAnalysisRule ::
-        CarbonPreInsertionCasts ::
-        ExtractPythonUDFs ::
-        ResolveHiveWindowFunction ::
-        PreInsertCastAndRename ::
-        Nil
-
-      override val extendedCheckRules = Seq(
-        PreWriteCheck(catalog)
-      )
-    }
-
-  @transient
-  override protected[sql] lazy val optimizer: Optimizer =
-    CarbonOptimizer.optimizer(
-      CodeGenerateFactory.createDefaultOptimizer(conf, sc),
-      conf.asInstanceOf[CarbonSQLConf],
-      sc.version)
-
-  protected[sql] override def getSQLDialect(): ParserDialect = new CarbonSQLDialect(this)
-
-  experimental.extraStrategies = {
-    val carbonStrategy = new CarbonStrategies(self)
-    Seq(carbonStrategy.CarbonTableScan, carbonStrategy.DDLStrategies)
-  }
-
-  override protected def configure(): Map[String, String] = {
-    sc.hadoopConfiguration.addResource("hive-site.xml")
-    if (sc.hadoopConfiguration.get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
-      val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
-      val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
-      logDebug(s"metastore db is going to be created in location: $hiveMetaStoreDB")
-      super.configure() ++ Map[String, String]((CarbonCommonConstants.HIVE_CONNECTION_URL,
-        s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"),
-        ("hive.metastore.warehouse.dir", metaStorePathAbsolute + "/hivemetadata"))
-    } else {
-      super.configure()
-    }
-  }
-
-  @transient
-  private val LOGGER = LogServiceFactory.getLogService(CarbonContext.getClass.getName)
-
-  var queryId: String = ""
-
-  override def sql(sql: String): DataFrame = {
-    // queryId will be unique for each query, creting query detail holder
-    queryId = System.nanoTime() + ""
-    this.setConf("queryId", queryId)
-
-    CarbonContext.updateCarbonPorpertiesPath(this)
-    val sqlString = sql.toUpperCase
-    LOGGER.info(s"Query [$sqlString]")
-    val recorder = CarbonTimeStatisticsFactory.createDriverRecorder()
-    val statistic = new QueryStatistic()
-    val logicPlan: LogicalPlan = parseSql(sql)
-    statistic.addStatistics(QueryStatisticsConstants.SQL_PARSE, System.currentTimeMillis())
-    recorder.recordStatisticsForDriver(statistic, queryId)
-    val result = new DataFrame(this, logicPlan)
-
-    // We force query optimization to happen right away instead of letting it happen lazily like
-    // when using the query DSL.  This is so DDL commands behave as expected.  This is only
-    // generates the RDD lineage for DML queries, but do not perform any execution.
-    result
-  }
-
-}
-
-object CarbonContext {
-
-  val datasourceName: String = "org.apache.carbondata.format"
-
-  val datasourceShortName: String = "carbondata"
-
-  @transient
-  private val LOGGER = LogServiceFactory.getLogService(CarbonContext.getClass.getName)
-
-  final def updateCarbonPorpertiesPath(hiveContext: HiveContext) {
-    val carbonPropertiesFilePath = hiveContext.getConf("carbon.properties.filepath", null)
-    val systemcarbonPropertiesFilePath = System.getProperty("carbon.properties.filepath", null)
-    if (null != carbonPropertiesFilePath && null == systemcarbonPropertiesFilePath) {
-      System.setProperty("carbon.properties.filepath",
-        carbonPropertiesFilePath + "/" + "carbon.properties")
-    }
-    // configuring the zookeeper URl .
-    val zooKeeperUrl = hiveContext.getConf("spark.deploy.zookeeper.url", "127.0.0.1:2181")
-
-    CarbonProperties.getInstance().addProperty("spark.deploy.zookeeper.url", zooKeeperUrl)
-
-  }
-
-  // this cache is used to avoid creating multiple CarbonContext from same SparkContext,
-  // to avoid the derby problem for metastore
-  private val cache = collection.mutable.Map[SparkContext, CarbonContext]()
-
-  def getInstance(sc: SparkContext): CarbonContext = {
-    cache(sc)
-  }
-
-  def addInstance(sc: SparkContext, cc: CarbonContext): Unit = {
-    if (cache.contains(sc)) {
-      sys.error("creating multiple instances of CarbonContext is not " +
-                "allowed using the same SparkContext instance")
-    }
-    cache(sc) = cc
-  }
-}
-
-object SQLParser {
-  def parse(sql: String, sqlContext: SQLContext): LogicalPlan = sqlContext.parseSql(sql)
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
deleted file mode 100644
index 2c2e954..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import java.text.SimpleDateFormat
-import java.util.Date
-
-import scala.reflect.ClassTag
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileStatus, Path}
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.{Job, JobID}
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-import org.apache.spark._
-import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.mapreduce.SparkHadoopMapReduceUtil
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.sources.{Filter, HadoopFsRelation, OutputWriterFactory}
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.util.SerializableConfiguration
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.scan.expression.logical.AndExpression
-import org.apache.carbondata.core.util.DataTypeUtil
-import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonProjection}
-import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
-import org.apache.carbondata.hadoop.util.{CarbonInputFormatUtil, SchemaReader}
-import org.apache.carbondata.processing.merger.TableMeta
-import org.apache.carbondata.spark.{CarbonFilters, CarbonOption}
-import org.apache.carbondata.spark.rdd.CarbonRDD
-import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
-import org.apache.carbondata.spark.util.SparkDataTypeConverterImpl
-
-private[sql] case class CarbonDatasourceHadoopRelation(
-  sqlContext: SQLContext,
-  paths: Array[String],
-  parameters: Map[String, String],
-  tableSchema: Option[StructType])
-  extends HadoopFsRelation {
-
-  lazy val schemaPath = new Path(CarbonTablePath.getSchemaFilePath(paths.head))
-  if (!schemaPath.getFileSystem(new Configuration).exists(schemaPath)) {
-    throw new IllegalArgumentException("invalid CarbonData file path: " + paths.head)
-  }
-
-  lazy val job = new Job(new JobConf())
-  lazy val options = new CarbonOption(parameters)
-  lazy val absIdentifier = AbsoluteTableIdentifier.fromTablePath(paths.head)
-  lazy val relationRaw: CarbonRelation = {
-    val carbonTable = SchemaReader.readCarbonTableFromStore(absIdentifier)
-    if (carbonTable == null) {
-      sys.error(s"CarbonData file path ${paths.head} is not valid")
-    }
-    CarbonRelation(
-      carbonTable.getDatabaseName,
-      carbonTable.getFactTableName,
-      CarbonSparkUtil.createSparkMeta(carbonTable),
-      new TableMeta(carbonTable.getCarbonTableIdentifier,
-        paths.head, absIdentifier.getTablePath, carbonTable),
-      None
-    )(sqlContext)
-  }
-
-  override def dataSchema: StructType = tableSchema.getOrElse(relationRaw.schema)
-
-  override def prepareJobForWrite(job: Job): OutputWriterFactory = {
-    // TODO
-    throw new UnsupportedOperationException
-  }
-
-  override def buildScan(
-    requiredColumns: Array[String],
-    filters: Array[Filter],
-    inputFiles: Array[FileStatus]): RDD[Row] = {
-    val conf = new Configuration(job.getConfiguration)
-    filters.flatMap { filter =>
-      CarbonFilters.createCarbonFilter(dataSchema, filter)
-    }.reduceOption(new AndExpression(_, _))
-      .foreach(CarbonTableInputFormat.setFilterPredicates(conf, _))
-
-    val projection = new CarbonProjection
-    requiredColumns.foreach(projection.addColumn)
-    CarbonTableInputFormat.setColumnProjection(conf, projection)
-    CarbonTableInputFormat.setCarbonReadSupport(conf, classOf[SparkRowReadSupportImpl])
-
-    new CarbonHadoopFSRDD[Row](sqlContext.sparkContext,
-      new SerializableConfiguration(conf),
-      absIdentifier,
-      classOf[CarbonTableInputFormat[Row]],
-      classOf[Row]
-    )
-  }
-
-}
-
-class CarbonHadoopFSPartition(rddId: Int, val idx: Int,
-  val carbonSplit: SerializableWritable[CarbonInputSplit])
-  extends Partition {
-
-  override val index: Int = idx
-
-  override def hashCode(): Int = 41 * (41 + rddId) + idx
-}
-
-class CarbonHadoopFSRDD[V: ClassTag](
-  @transient sc: SparkContext,
-  conf: SerializableConfiguration,
-  identifier: AbsoluteTableIdentifier,
-  inputFormatClass: Class[_ <: CarbonTableInputFormat[V]],
-  valueClass: Class[V])
-  extends CarbonRDD[V](sc, Nil) with SparkHadoopMapReduceUtil {
-
-  private val jobTrackerId: String = {
-    val formatter = new SimpleDateFormat("yyyyMMddHHmm")
-    formatter.format(new Date())
-  }
-  @transient protected val jobId = new JobID(jobTrackerId, id)
-
-  override def internalCompute(split: Partition,
-    context: TaskContext): Iterator[V] = {
-    val attemptId = newTaskAttemptID(jobTrackerId, id, isMap = true, split.index, 0)
-    val hadoopAttemptContext = newTaskAttemptContext(conf.value, attemptId)
-    val job: Job = new Job(hadoopAttemptContext.getConfiguration)
-    val format = CarbonInputFormatUtil.createCarbonInputFormat(identifier, job)
-    CarbonInputFormat.setDataTypeConverter(hadoopAttemptContext.getConfiguration,
-      new SparkDataTypeConverterImpl)
-    hadoopAttemptContext.getConfiguration.set(FileInputFormat.INPUT_DIR, identifier.getTablePath)
-    val reader =
-      format.createRecordReader(split.asInstanceOf[CarbonHadoopFSPartition].carbonSplit.value,
-        hadoopAttemptContext
-      )
-    reader.initialize(split.asInstanceOf[CarbonHadoopFSPartition].carbonSplit.value,
-      hadoopAttemptContext
-    )
-    new Iterator[V] {
-      private[this] var havePair = false
-      private[this] var finished = false
-
-      override def hasNext: Boolean = {
-        if (context.isInterrupted) {
-          throw new TaskKilledException
-        }
-        if (!finished && !havePair) {
-          finished = !reader.nextKeyValue
-          if (finished) {
-            reader.close()
-          }
-          havePair = !finished
-        }
-        !finished
-      }
-
-      override def next(): V = {
-        if (!hasNext) {
-          throw new java.util.NoSuchElementException("End of stream")
-        }
-        havePair = false
-        reader.getCurrentValue
-      }
-    }
-  }
-
-  override protected def getPartitions: Array[Partition] = {
-    val jobContext = newJobContext(conf.value, jobId)
-    val format = CarbonInputFormatUtil.createCarbonInputFormat(identifier, new Job(conf.value))
-    jobContext.getConfiguration.set(FileInputFormat.INPUT_DIR, identifier.getTablePath)
-    val splits = format.getSplits(jobContext).toArray
-    val carbonInputSplits = splits
-      .map(f => new SerializableWritable(f.asInstanceOf[CarbonInputSplit]))
-    carbonInputSplits.zipWithIndex.map(f => new CarbonHadoopFSPartition(id, f._2, f._1))
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
deleted file mode 100644
index 2e93a6c..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-import scala.language.implicitConversions
-
-import org.apache.hadoop.fs.Path
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.hive.{CarbonMetaData, CarbonMetastoreTypes}
-import org.apache.spark.sql.sources._
-import org.apache.spark.sql.types.{DataType, StructType}
-
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.merger.TableMeta
-import org.apache.carbondata.spark.{CarbonOption, _}
-
-/**
- * Carbon relation provider compliant to data source api.
- * Creates carbon relations
- */
-class CarbonSource extends RelationProvider
-    with CreatableRelationProvider with HadoopFsRelationProvider with DataSourceRegister {
-
-  override def shortName(): String = "carbondata"
-
-  /**
-   * Returns a new base relation with the given parameters.
-   * Note: the parameters' keywords are case insensitive and this insensitivity is enforced
-   * by the Map that is passed to the function.
-   */
-  override def createRelation(
-      sqlContext: SQLContext,
-      parameters: Map[String, String]): BaseRelation = {
-    // if path is provided we can directly create Hadoop relation. \
-    // Otherwise create datasource relation
-    parameters.get("path") match {
-      case Some(path) => CarbonDatasourceHadoopRelation(sqlContext, Array(path), parameters, None)
-      case _ =>
-        val options = new CarbonOption(parameters)
-        val tableIdentifier = options.tableIdentifier.split("""\.""").toSeq
-        val identifier = tableIdentifier match {
-          case Seq(name) => TableIdentifier(name, None)
-          case Seq(db, name) => TableIdentifier(name, Some(db))
-        }
-        CarbonDatasourceRelation(identifier, None)(sqlContext)
-    }
-  }
-
-  override def createRelation(
-      sqlContext: SQLContext,
-      mode: SaveMode,
-      parameters: Map[String, String],
-      data: SchemaRDD): BaseRelation = {
-
-    // To avoid derby problem, dataframe need to be writen and read using CarbonContext
-    require(sqlContext.isInstanceOf[CarbonContext], "Error in saving dataframe to carbon file, " +
-        "must use CarbonContext to save dataframe")
-
-    // User should not specify path since only one store is supported in carbon currently,
-    // after we support multi-store, we can remove this limitation
-    require(!parameters.contains("path"), "'path' should not be specified, " +
-        "the path to store carbon file is the 'storePath' specified when creating CarbonContext")
-
-    val options = new CarbonOption(parameters)
-    val storePath = CarbonContext.getInstance(sqlContext.sparkContext).storePath
-    val tablePath = new Path(storePath + "/" + options.dbName + "/" + options.tableName)
-    val isExists = tablePath.getFileSystem(sqlContext.sparkContext.hadoopConfiguration)
-        .exists(tablePath)
-    val (doSave, doAppend) = (mode, isExists) match {
-      case (SaveMode.ErrorIfExists, true) =>
-        sys.error(s"ErrorIfExists mode, path $storePath already exists.")
-      case (SaveMode.Overwrite, true) =>
-        val cc = CarbonContext.getInstance(sqlContext.sparkContext)
-        cc.sql(s"DROP TABLE IF EXISTS ${ options.dbName }.${ options.tableName }")
-        (true, false)
-      case (SaveMode.Overwrite, false) | (SaveMode.ErrorIfExists, false) =>
-        (true, false)
-      case (SaveMode.Append, _) =>
-        (false, true)
-      case (SaveMode.Ignore, exists) =>
-        (!exists, false)
-    }
-
-    if (doSave) {
-      // save data when the save mode is Overwrite.
-      new CarbonDataFrameWriter(data).saveAsCarbonFile(parameters)
-    } else if (doAppend) {
-      new CarbonDataFrameWriter(data).appendToCarbonFile(parameters)
-    }
-
-    createRelation(sqlContext, parameters)
-  }
-
-  override def createRelation(sqlContext: SQLContext,
-      paths: Array[String],
-      dataSchema: Option[StructType],
-      partitionColumns: Option[StructType],
-      parameters: Map[String, String]): HadoopFsRelation = {
-    CarbonDatasourceHadoopRelation(sqlContext, paths, parameters, dataSchema)
-  }
-}
-
-/**
- *  Creates carbon relation compliant to data source api.
- * This relation is stored to hive metastore
- */
-private[sql] case class CarbonDatasourceRelation(
-    tableIdentifier: TableIdentifier,
-    alias: Option[String])
-    (@transient context: SQLContext)
-    extends BaseRelation with Serializable {
-
-  lazy val carbonRelation: CarbonRelation = {
-    CarbonEnv.get
-        .carbonMetastore.lookupRelation1(tableIdentifier, None)(sqlContext)
-        .asInstanceOf[CarbonRelation]
-  }
-
-  def getDatabaseName(): String = tableIdentifier.database.getOrElse("default")
-
-  def getTable(): String = tableIdentifier.table
-
-  def schema: StructType = carbonRelation.schema
-
-  def sqlContext: SQLContext = context
-
-  override def sizeInBytes: Long = carbonRelation.sizeInBytes
-}
-
-/**
- * Represents logical plan for one carbon table
- */
-case class CarbonRelation(
-    databaseName: String,
-    tableName: String,
-    var metaData: CarbonMetaData,
-    tableMeta: TableMeta,
-    alias: Option[String])(@transient sqlContext: SQLContext)
-    extends LeafNode with MultiInstanceRelation {
-
-  def recursiveMethod(dimName: String, childDim: CarbonDimension): String = {
-    childDim.getDataType.getName.toLowerCase match {
-      case "array" => s"${
-        childDim.getColName.substring(dimName.length + 1)
-      }:array<${ getArrayChildren(childDim.getColName) }>"
-      case "struct" => s"${
-        childDim.getColName.substring(dimName.length + 1)
-      }:struct<${ getStructChildren(childDim.getColName) }>"
-      case dType => s"${ childDim.getColName.substring(dimName.length + 1) }:${ dType }"
-    }
-  }
-
-  def getArrayChildren(dimName: String): String = {
-    metaData.carbonTable.getChildren(dimName).asScala.map(childDim => {
-      childDim.getDataType.getName.toLowerCase match {
-        case "array" => s"array<${ getArrayChildren(childDim.getColName) }>"
-        case "struct" => s"struct<${ getStructChildren(childDim.getColName) }>"
-        case dType => addDecimalScaleAndPrecision(childDim, dType)
-      }
-    }).mkString(",")
-  }
-
-  def getStructChildren(dimName: String): String = {
-    metaData.carbonTable.getChildren(dimName).asScala.map(childDim => {
-      childDim.getDataType.getName.toLowerCase match {
-        case "array" => s"${
-          childDim.getColName.substring(dimName.length + 1)
-        }:array<${ getArrayChildren(childDim.getColName) }>"
-        case "struct" => s"${
-          childDim.getColName.substring(dimName.length + 1)
-        }:struct<${ metaData.carbonTable.getChildren(childDim.getColName)
-            .asScala.map(f => s"${ recursiveMethod(childDim.getColName, f) }").mkString(",")
-        }>"
-        case dType => s"${ childDim.getColName
-            .substring(dimName.length() + 1) }:${ addDecimalScaleAndPrecision(childDim, dType) }"
-      }
-    }).mkString(",")
-  }
-
-  override def newInstance(): LogicalPlan = {
-    CarbonRelation(databaseName, tableName, metaData, tableMeta, alias)(sqlContext)
-        .asInstanceOf[this.type]
-  }
-
-  val dimensionsAttr = {
-    val sett = new java.util.LinkedHashSet(tableMeta.carbonTable
-      .getDimensionByTableName(tableMeta.carbonTableIdentifier.getTableName).asScala.asJava)
-    sett.asScala.toSeq.filter(dim => !dim.isInvisible ||
-                                     (dim.isInvisible && dim.isInstanceOf[CarbonImplicitDimension]))
-      .map(dim => {
-      val dimval = metaData.carbonTable
-          .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName)
-      val output: DataType = dimval.getDataType.getName.toLowerCase match {
-        case "array" =>
-          CarbonMetastoreTypes.toDataType(s"array<${ getArrayChildren(dim.getColName) }>")
-        case "struct" =>
-          CarbonMetastoreTypes.toDataType(s"struct<${ getStructChildren(dim.getColName) }>")
-        case dType =>
-          val dataType = addDecimalScaleAndPrecision(dimval, dType)
-          CarbonMetastoreTypes.toDataType(dataType)
-      }
-
-      AttributeReference(
-        dim.getColName,
-        output,
-        nullable = true)(qualifiers = tableName +: alias.toSeq)
-    })
-  }
-
-  val measureAttr = {
-    val factTable = tableMeta.carbonTable.getFactTableName
-    new java.util.LinkedHashSet(
-      tableMeta.carbonTable.
-          getMeasureByTableName(tableMeta.carbonTable.getFactTableName).
-          asScala.asJava).asScala.toSeq.filter(!_.getColumnSchema.isInvisible)
-        .map(x => AttributeReference(x.getColName, CarbonMetastoreTypes.toDataType(
-          metaData.carbonTable.getMeasureByName(factTable, x.getColName).getDataType.getName
-              .toLowerCase match {
-            case "float" => "double"
-            case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"
-            case others => others
-          }),
-          nullable = true)(qualifiers = tableName +: alias.toSeq))
-  }
-
-  override val output = {
-    val columns = tableMeta.carbonTable.getCreateOrderColumn(tableMeta.carbonTable.getFactTableName)
-        .asScala
-    columns.filter(!_.isInvisible).map { column =>
-      if (column.isDimension()) {
-        val output: DataType = column.getDataType.getName.toLowerCase match {
-          case "array" =>
-            CarbonMetastoreTypes.toDataType(s"array<${getArrayChildren(column.getColName)}>")
-          case "struct" =>
-            CarbonMetastoreTypes.toDataType(s"struct<${getStructChildren(column.getColName)}>")
-          case dType =>
-            val dataType = addDecimalScaleAndPrecision(column, dType)
-            CarbonMetastoreTypes.toDataType(dataType)
-        }
-        AttributeReference(column.getColName, output,
-          nullable = true
-        )(qualifiers = tableName +: alias.toSeq)
-      } else {
-        AttributeReference(column.getColName, CarbonMetastoreTypes.toDataType(
-          column.getDataType.getName.toLowerCase match {
-            case "float" => "double"
-            case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column
-              .getColumnSchema.getScale + ")"
-            case others => others
-          }
-        ),
-          nullable = true
-        )(qualifiers = tableName +: alias.toSeq)
-      }
-    }
-  }
-  // TODO: Use data from the footers.
-  override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes)
-
-  override def equals(other: Any): Boolean = {
-    other match {
-      case p: CarbonRelation =>
-        p.databaseName == databaseName && p.output == output && p.tableName == tableName
-      case _ => false
-    }
-  }
-
-  def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = {
-    var dType = dataType
-    if (dimval.getDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.DECIMAL) {
-      dType +=
-          "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
-    }
-    dType
-  }
-
-  private var tableStatusLastUpdateTime = 0L
-
-  private var sizeInBytesLocalValue = 0L
-
-  def sizeInBytes: Long = {
-    val tableStatusNewLastUpdatedTime = SegmentStatusManager.getTableStatusLastModifiedTime(
-      tableMeta.carbonTable.getAbsoluteTableIdentifier)
-    if (tableStatusLastUpdateTime != tableStatusNewLastUpdatedTime) {
-      val tablePath = CarbonStorePath.getCarbonTablePath(
-        tableMeta.storePath,
-        tableMeta.carbonTableIdentifier).getPath
-      val fileType = FileFactory.getFileType(tablePath)
-      if(FileFactory.isFileExist(tablePath, fileType)) {
-        tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
-        sizeInBytesLocalValue = FileFactory.getDirectorySize(tablePath)
-      }
-    }
-    sizeInBytesLocalValue
-  }
-
-}
-
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
deleted file mode 100644
index c14a61a..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.TaskContext
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors.attachTree
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.execution.{SparkPlan, UnaryNode}
-import org.apache.spark.sql.hive.{CarbonMetastore, CarbonMetastoreTypes}
-import org.apache.spark.sql.optimizer.CarbonDecoderRelation
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
-import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, ColumnIdentifier}
-import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
-import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.util.{CarbonTimeStatisticsFactory, DataTypeUtil}
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.spark.CarbonAliasDecoderRelation
-
-/**
- * It decodes the dictionary key to value
- */
-case class CarbonDictionaryDecoder(
-    relations: Seq[CarbonDecoderRelation],
-    profile: CarbonProfile,
-    aliasMap: CarbonAliasDecoderRelation,
-    child: SparkPlan)
-  (@transient sqlContext: SQLContext)
-  extends UnaryNode {
-
-  override def otherCopyArgs: Seq[AnyRef] = sqlContext :: Nil
-
-  override val output: Seq[Attribute] = {
-    child.output.map { a =>
-      val attr = aliasMap.getOrElse(a, a)
-      val relation = relations.find(p => p.contains(attr))
-      if (relation.isDefined && canBeDecoded(attr)) {
-        val carbonTable = relation.get.carbonRelation.carbonRelation.metaData.carbonTable
-        val carbonDimension = carbonTable
-          .getDimensionByName(carbonTable.getFactTableName, attr.name)
-        if (carbonDimension != null &&
-            carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
-            !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
-            !carbonDimension.isComplex()) {
-          val newAttr = AttributeReference(a.name,
-            convertCarbonToSparkDataType(carbonDimension,
-              relation.get.carbonRelation.carbonRelation),
-            a.nullable,
-            a.metadata)(a.exprId,
-            a.qualifiers).asInstanceOf[Attribute]
-          newAttr
-        } else {
-          a
-        }
-      } else {
-        a
-      }
-    }
-  }
-
-
-  def canBeDecoded(attr: Attribute): Boolean = {
-    profile match {
-      case ip: IncludeProfile if ip.attributes.nonEmpty =>
-        ip.attributes
-          .exists(a => a.name.equalsIgnoreCase(attr.name) && a.exprId == attr.exprId)
-      case ep: ExcludeProfile =>
-        !ep.attributes
-          .exists(a => a.name.equalsIgnoreCase(attr.name) && a.exprId == attr.exprId)
-      case _ => true
-    }
-  }
-
-  def convertCarbonToSparkDataType(carbonDimension: CarbonDimension,
-      relation: CarbonRelation): types.DataType = {
-    carbonDimension.getDataType match {
-      case CarbonDataTypes.STRING => StringType
-      case CarbonDataTypes.SHORT => ShortType
-      case CarbonDataTypes.INT => IntegerType
-      case CarbonDataTypes.LONG => LongType
-      case CarbonDataTypes.DOUBLE => DoubleType
-      case CarbonDataTypes.BOOLEAN => BooleanType
-      case CarbonDataTypes.DECIMAL =>
-        val scale: Int = carbonDimension.getColumnSchema.getScale
-        val precision: Int = carbonDimension.getColumnSchema.getPrecision
-        if (scale == 0 && precision == 0) {
-          DecimalType(18, 2)
-        } else {
-          DecimalType(precision, scale)
-        }
-      case CarbonDataTypes.TIMESTAMP => TimestampType
-      case CarbonDataTypes.DATE => DateType
-      case CarbonDataTypes.STRUCT =>
-        CarbonMetastoreTypes
-          .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName) }>")
-      case CarbonDataTypes.ARRAY =>
-        CarbonMetastoreTypes
-          .toDataType(s"array<${ relation.getArrayChildren(carbonDimension.getColName) }>")
-    }
-  }
-
-  val getDictionaryColumnIds = {
-    val attributes = child.output
-    val dictIds: Array[(String, ColumnIdentifier, DataType, CarbonDimension)] =
-      attributes.map { a =>
-        val attr = aliasMap.getOrElse(a, a)
-        val relation = relations.find(p => p.contains(attr))
-        if (relation.isDefined && canBeDecoded(attr)) {
-          val carbonTable = relation.get.carbonRelation.carbonRelation.metaData.carbonTable
-          val carbonDimension =
-            carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
-          if (carbonDimension != null &&
-              carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
-              !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
-              !carbonDimension.isComplex()) {
-            (carbonTable.getFactTableName, carbonDimension.getColumnIdentifier,
-              carbonDimension.getDataType, carbonDimension)
-          } else {
-            (null, null, null, null)
-          }
-        } else {
-          (null, null, null, null)
-        }
-
-      }.toArray
-    dictIds
-  }
-
-  override def outputsUnsafeRows: Boolean = true
-
-  override def canProcessUnsafeRows: Boolean = true
-
-  override def canProcessSafeRows: Boolean = true
-
-  override def doExecute(): RDD[InternalRow] = {
-    attachTree(this, "execute") {
-      val storePath = sqlContext.catalog.asInstanceOf[CarbonMetastore].storePath
-      val queryId = sqlContext.getConf("queryId", System.nanoTime() + "")
-      val absoluteTableIdentifiers = relations.map { relation =>
-        val carbonTable = relation.carbonRelation.carbonRelation.metaData.carbonTable
-        (carbonTable.getFactTableName, carbonTable.getAbsoluteTableIdentifier)
-      }.toMap
-
-      val recorder = CarbonTimeStatisticsFactory.createExecutorRecorder(queryId)
-      if (isRequiredToDecode) {
-        val dataTypes = child.output.map { attr => attr.dataType }
-        child.execute().mapPartitions { iter =>
-          val cacheProvider: CacheProvider = CacheProvider.getInstance
-          val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] =
-            cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, storePath)
-          val dicts: Seq[Dictionary] = getDictionary(absoluteTableIdentifiers,
-            forwardDictionaryCache)
-          val dictIndex = dicts.zipWithIndex.filter(x => x._1 != null).map(x => x._2)
-          // add a task completion listener to clear dictionary that is a decisive factor for
-          // LRU eviction policy
-          val dictionaryTaskCleaner = TaskContext.get
-          dictionaryTaskCleaner.addTaskCompletionListener(context =>
-            dicts.foreach { dictionary =>
-              if (null != dictionary) {
-                dictionary.clear()
-              }
-            }
-          )
-          new Iterator[InternalRow] {
-            val unsafeProjection = UnsafeProjection.create(output.map(_.dataType).toArray)
-
-            override final def hasNext: Boolean = {
-              iter.hasNext
-            }
-
-            override final def next(): InternalRow = {
-              val row: InternalRow = iter.next()
-              val data = row.toSeq(dataTypes).toArray
-              dictIndex.foreach { index =>
-                if (data(index) != null) {
-                  data(index) = DataTypeUtil.getDataBasedOnDataType(dicts(index)
-                    .getDictionaryValueForKeyInBytes(data(index).asInstanceOf[Int]),
-                    getDictionaryColumnIds(index)._4)
-                }
-              }
-              val result = unsafeProjection(new GenericMutableRow(data))
-              result
-            }
-          }
-        }
-      } else {
-        child.execute()
-      }
-    }
-  }
-
-  private def isRequiredToDecode = {
-    getDictionaryColumnIds.find(p => p._1 != null) match {
-      case Some(value) => true
-      case _ => false
-    }
-  }
-
-  private def getDictionary(atiMap: Map[String, AbsoluteTableIdentifier],
-      cache: Cache[DictionaryColumnUniqueIdentifier, Dictionary]) = {
-    val dictionaryColumnIds = getDictionaryColumnIds.map { dictionaryId =>
-      if (dictionaryId._2 != null) {
-        new DictionaryColumnUniqueIdentifier(
-          atiMap(dictionaryId._1).getCarbonTableIdentifier,
-          dictionaryId._2, dictionaryId._3,
-          CarbonStorePath.getCarbonTablePath(atiMap(dictionaryId._1)))
-      } else {
-        null
-      }
-    }
-    try {
-      val noDictionaryIndexes = new java.util.ArrayList[Int]()
-      dictionaryColumnIds.zipWithIndex.foreach { columnIndex =>
-        if (columnIndex._1 == null) {
-          noDictionaryIndexes.add(columnIndex._2)
-        }
-      }
-      val dict = cache.getAll(dictionaryColumnIds.filter(_ != null).toSeq.asJava);
-      val finalDict = new java.util.ArrayList[Dictionary]()
-      var dictIndex: Int = 0
-      dictionaryColumnIds.zipWithIndex.foreach { columnIndex =>
-        if (!noDictionaryIndexes.contains(columnIndex._2)) {
-          finalDict.add(dict.get(dictIndex))
-          dictIndex += 1
-        } else {
-          finalDict.add(null)
-        }
-      }
-      finalDict.asScala
-    } catch {
-      case t: Throwable => Seq.empty
-    }
-
-  }
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
deleted file mode 100644
index 36cd6f2..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
-import org.apache.spark.sql.hive.{CarbonIUDAnalysisRule, CarbonMetastore}
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.hadoop.readsupport.impl.RawDataReadSupport
-import org.apache.carbondata.spark.rdd.SparkReadSupport
-
-case class CarbonEnv(carbonMetastore: CarbonMetastore)
-
-object CarbonEnv {
-
-  @volatile private var carbonEnv: CarbonEnv = _
-
-  // set readsupport class global so that the executor can get it.
-  SparkReadSupport.readSupportClass = classOf[RawDataReadSupport]
-
-  var initialized = false
-
-  def init(sqlContext: SQLContext): Unit = {
-    if (!initialized) {
-      val cc = sqlContext.asInstanceOf[CarbonContext]
-      val catalog = new CarbonMetastore(cc, cc.storePath, cc.hiveClientInterface, "")
-      carbonEnv = CarbonEnv(catalog)
-      CarbonIUDAnalysisRule.init(sqlContext)
-      initialized = true
-      CarbonProperties.getInstance.addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true")
-    }
-  }
-
-  def get: CarbonEnv = {
-    if (initialized) carbonEnv
-    else throw new RuntimeException("CarbonEnv not initialized")
-  }
-}
-
-
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSQLConf.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSQLConf.scala
deleted file mode 100644
index 6ed8c0d..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSQLConf.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.SQLConf.SQLConfEntry
-import org.apache.spark.sql.hive.CarbonSQLDialect
-
- /**
-  * A trait that enables the setting and getting of mutable config parameters/hints.
-  *
-  */
-class CarbonSQLConf extends SQLConf {
-
-  override def dialect: String = {
-    getConf(SQLConf.DIALECT,
-      classOf[CarbonSQLDialect].getCanonicalName)
-  }
-
-  override def caseSensitiveAnalysis: Boolean = getConf(SQLConf.CASE_SENSITIVE, false)
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
deleted file mode 100644
index a3c6343..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.CarbonInputMetrics
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.execution.LeafNode
-import org.apache.spark.sql.hive.CarbonMetastore
-
-import org.apache.carbondata.core.scan.model._
-import org.apache.carbondata.hadoop.{CarbonProjection, InputMetricsStats}
-import org.apache.carbondata.spark.CarbonFilters
-import org.apache.carbondata.spark.rdd.CarbonScanRDD
-
-case class CarbonScan(
-    var columnProjection: Seq[Attribute],
-    relationRaw: CarbonRelation,
-    dimensionPredicatesRaw: Seq[Expression],
-    useUnsafeCoversion: Boolean = true)(@transient val ocRaw: SQLContext) extends LeafNode {
-  val carbonTable = relationRaw.metaData.carbonTable
-  val selectedDims = scala.collection.mutable.MutableList[QueryDimension]()
-  val selectedMsrs = scala.collection.mutable.MutableList[QueryMeasure]()
-  @transient val carbonCatalog = ocRaw.catalog.asInstanceOf[CarbonMetastore]
-
-  val attributesNeedToDecode = new java.util.LinkedHashSet[AttributeReference]()
-  val unprocessedExprs = new ArrayBuffer[Expression]()
-
-  val buildCarbonPlan: CarbonQueryPlan = {
-    val plan: CarbonQueryPlan = new CarbonQueryPlan(relationRaw.databaseName, relationRaw.tableName)
-    plan.setQueryId(ocRaw.getConf("queryId", System.nanoTime() + ""))
-    processFilterExpressions(plan)
-    plan
-  }
-
-  def processFilterExpressions(plan: CarbonQueryPlan) {
-    if (dimensionPredicatesRaw.nonEmpty) {
-      val expressionVal = CarbonFilters.processExpression(
-        dimensionPredicatesRaw,
-        attributesNeedToDecode,
-        unprocessedExprs,
-        carbonTable)
-      expressionVal match {
-        case Some(ce) =>
-          // adding dimension used in expression in querystats
-          plan.setFilterExpression(ce)
-        case _ =>
-      }
-    }
-    processExtraAttributes(plan)
-  }
-
-  private def processExtraAttributes(plan: CarbonQueryPlan) {
-    if (attributesNeedToDecode.size() > 0) {
-      val attributeOut = new ArrayBuffer[Attribute]() ++ columnProjection
-
-      attributesNeedToDecode.asScala.foreach { attr =>
-        if (!columnProjection.exists(_.name.equalsIgnoreCase(attr.name))) {
-          attributeOut += attr
-        }
-      }
-      columnProjection = attributeOut
-    }
-
-    val columns = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName)
-    columns.addAll(carbonTable.getImplicitDimensionByTableName(carbonTable.getFactTableName))
-    val colAttr = new Array[Attribute](columns.size())
-    columnProjection.foreach { attr =>
-    val column =
-        carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name)
-      if(column != null) {
-        colAttr(columns.indexOf(column)) = attr
-       }
-    }
-
-    columnProjection = colAttr.filter(f => f != null)
-
-    var queryOrder: Integer = 0
-    columnProjection.foreach { attr =>
-      val carbonColumn = carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name)
-      if (carbonColumn != null) {
-        if (carbonColumn.isDimension()) {
-          val dim = new QueryDimension(attr.name)
-          dim.setQueryOrder(queryOrder)
-          queryOrder = queryOrder + 1
-          selectedDims += dim
-         } else {
-          val m1 = new QueryMeasure(attr.name)
-          m1.setQueryOrder(queryOrder)
-          queryOrder = queryOrder + 1
-          selectedMsrs += m1
-        }
-      }
-    }
-
-    // Fill the selected dimensions & measures obtained from
-    // attributes to query plan  for detailed query
-    selectedDims.foreach(plan.addDimension)
-    selectedMsrs.foreach(plan.addMeasure)
-  }
-
-  def inputRdd: CarbonScanRDD = {
-    val projection = new CarbonProjection
-    columnProjection.foreach { attr =>
-      projection.addColumn(attr.name)
-    }
-    val inputMetricsStats: CarbonInputMetrics = new CarbonInputMetrics
-    new CarbonScanRDD(
-      ocRaw.sparkContext,
-      projection,
-      buildCarbonPlan.getFilterExpression,
-      carbonTable.getAbsoluteTableIdentifier,
-      carbonTable.getTableInfo.serialize(),
-      carbonTable.getTableInfo, inputMetricsStats
-    )
-  }
-
-  override def outputsUnsafeRows: Boolean =
-    (attributesNeedToDecode.size() == 0) && useUnsafeCoversion
-
-  override def doExecute(): RDD[InternalRow] = {
-    val outUnsafeRows: Boolean = (attributesNeedToDecode.size() == 0) && useUnsafeCoversion
-    inputRdd.mapPartitions { iter =>
-      val unsafeProjection = UnsafeProjection.create(output.map(_.dataType).toArray)
-      new Iterator[InternalRow] {
-        override def hasNext: Boolean = iter.hasNext
-
-        override def next(): InternalRow = {
-          val value = iter.next
-          if (outUnsafeRows) {
-            unsafeProjection(value)
-          } else {
-            value
-          }
-        }
-      }
-    }
-  }
-
-  def output: Seq[Attribute] = {
-    columnProjection
-  }
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSparkUtil.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSparkUtil.scala
deleted file mode 100644
index bc62a55..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSparkUtil.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.hive.{CarbonMetaData, DictionaryMap}
-
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.CarbonUtil
-
-case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
-
-object CarbonSparkUtil {
-
-  def createSparkMeta(carbonTable: CarbonTable): CarbonMetaData = {
-    val dimensionsAttr = carbonTable.getDimensionByTableName(carbonTable.getFactTableName)
-        .asScala.map(x => x.getColName)
-    // wf : may be problem
-    val measureAttr = carbonTable.getMeasureByTableName(carbonTable.getFactTableName)
-        .asScala.map(x => x.getColName)
-    val dictionary =
-      carbonTable.getDimensionByTableName(carbonTable.getFactTableName).asScala.map { f =>
-        (f.getColName.toLowerCase,
-            f.hasEncoding(Encoding.DICTIONARY) && !f.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
-                !f.getDataType.isComplexType)
-      }
-    CarbonMetaData(dimensionsAttr, measureAttr, carbonTable, DictionaryMap(dictionary.toMap))
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
deleted file mode 100644
index 9dc9ee2..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ /dev/null
@@ -1,589 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.Map
-import scala.language.implicitConversions
-
-import org.apache.hadoop.hive.ql.lib.Node
-import org.apache.hadoop.hive.ql.parse._
-import org.apache.spark.sql.catalyst._
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit._
-import org.apache.spark.sql.catalyst.analysis._
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.execution.ExplainCommand
-import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.DescribeCommand
-import org.apache.spark.sql.hive.HiveQlWrapper
-import org.apache.spark.sql.types.StructField
-
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.util.CommonUtil
-
-/**
- * Parser for All Carbon DDL, DML cases in Unified context
- */
-class CarbonSqlParser() extends CarbonDDLSqlParser {
-
-  override def parse(input: String): LogicalPlan = {
-    synchronized {
-      // Initialize the Keywords.
-      initLexical
-      phrase(start)(new lexical.Scanner(input)) match {
-        case Success(plan, _) => plan match {
-          case x: LoadTable =>
-            x.inputSqlString = input
-            x
-          case logicalPlan => logicalPlan
-        }
-        case failureOrError => sys.error(failureOrError.toString)
-      }
-    }
-  }
-
-  override protected lazy val start: Parser[LogicalPlan] = explainPlan | startCommand
-
-  protected lazy val startCommand: Parser[LogicalPlan] =
-    createDatabase | dropDatabase | loadManagement | describeTable |
-      showPartitions | showLoads | alterTable | updateTable | deleteRecords | useDatabase |
-      createTable
-
-  protected lazy val loadManagement: Parser[LogicalPlan] =
-    deleteLoadsByID | deleteLoadsByLoadDate | cleanFiles | loadDataNew
-
-  protected lazy val createDatabase: Parser[LogicalPlan] =
-    CREATE ~> (DATABASE | SCHEMA) ~> restInput ^^ {
-      case statement =>
-        val createDbSql = "CREATE DATABASE " + statement
-        var dbName = ""
-        // Get Ast node for create db command
-        val node = HiveQlWrapper.getAst(createDbSql)
-        node match {
-          // get dbname
-          case Token("TOK_CREATEDATABASE", children) =>
-            dbName = BaseSemanticAnalyzer.unescapeIdentifier(children(0).getText)
-        }
-        CreateDatabase(convertDbNameToLowerCase(dbName), createDbSql)
-    }
-
-  protected lazy val dropDatabase: Parser[LogicalPlan] =
-    DROP ~> (DATABASE | SCHEMA) ~> restInput ^^ {
-      case statement =>
-        val dropDbSql = "DROP DATABASE " + statement
-        var dbName = ""
-        var isCascade = false
-        // Get Ast node for drop db command
-        val node = HiveQlWrapper.getAst(dropDbSql)
-        node match {
-          case Token("TOK_DROPDATABASE", children) =>
-            dbName = BaseSemanticAnalyzer.unescapeIdentifier(children(0).getText)
-            // check whether cascade drop db
-            children.collect {
-              case t@Token("TOK_CASCADE", _) =>
-                isCascade = true
-              case _ => // Unsupport features
-            }
-        }
-        DropDatabase(convertDbNameToLowerCase(dbName), isCascade, dropDbSql)
-    }
-
-  protected lazy val alterTable: Parser[LogicalPlan] =
-    ALTER ~> TABLE ~> restInput ^^ {
-      case statement =>
-        try {
-          val alterSql = "alter table " + statement
-          // DDl will be parsed and we get the AST tree from the HiveQl
-          val node = HiveQlWrapper.getAst(alterSql)
-          // processing the AST tree
-          nodeToPlanForAlterTable(node, alterSql)
-        } catch {
-          // MalformedCarbonCommandException need to be throw directly, parser will catch it
-          case ce: MalformedCarbonCommandException =>
-            throw ce
-        }
-    }
-
-  /**
-   * For handling the create table DDl systax compatible to Hive syntax
-   */
-  protected lazy val createTable: Parser[LogicalPlan] =
-    restInput ^^ {
-
-      case statement =>
-        try {
-          // DDl will be parsed and we get the AST tree from the HiveQl
-          val node = HiveQlWrapper.getAst(statement)
-          // processing the AST tree
-          nodeToPlan(node)
-        } catch {
-          // MalformedCarbonCommandException need to be throw directly, parser will catch it
-          case ce: MalformedCarbonCommandException =>
-            throw ce
-          case e: Exception =>
-            sys.error("Parsing error") // no need to do anything.
-        }
-    }
-
-  /**
-   * This function will traverse the tree and logical plan will be formed using that.
-   *
-   * @param node
-   * @return LogicalPlan
-   */
-  protected def nodeToPlan(node: Node): LogicalPlan = {
-    node match {
-      // if create table taken is found then only we will handle.
-      case Token("TOK_CREATETABLE", children) =>
-
-
-        var fields: Seq[Field] = Seq[Field]()
-        var tableComment: String = ""
-        var tableProperties = Map[String, String]()
-        var partitionByFields: Seq[Field] = Seq[Field]()
-        var partitionCols: Seq[PartitionerField] = Seq[PartitionerField]()
-        var likeTableName: String = ""
-        var storedBy: String = ""
-        var ifNotExistPresent: Boolean = false
-        var dbName: Option[String] = None
-        var tableName: String = ""
-        var bucketFields: Option[BucketFields] = None
-
-        try {
-
-          // Checking whether create table request is carbon table
-          children.collect {
-            case Token("TOK_STORAGEHANDLER", child :: Nil) =>
-              storedBy = BaseSemanticAnalyzer.unescapeSQLString(child.getText).trim.toLowerCase
-            case _ =>
-          }
-          if (!(storedBy.equals(CarbonContext.datasourceName) ||
-                storedBy.equals(CarbonContext.datasourceShortName))) {
-            sys.error("Not a carbon format request")
-          }
-
-          children.collect {
-            // collecting all the field  list
-            case list@Token("TOK_TABCOLLIST", _) =>
-              val cols = BaseSemanticAnalyzer.getColumns(list, true)
-              if (cols != null) {
-                val dupColsGrp = cols.asScala.groupBy(x => x.getName) filter {
-                  case (_, colList) => colList.size > 1
-                }
-                if (dupColsGrp.nonEmpty) {
-                  var columnName: String = ""
-                  dupColsGrp.toSeq.foreach(columnName += _._1 + ", ")
-                  columnName = columnName.substring(0, columnName.lastIndexOf(", "))
-                  val errorMessage = "Duplicate column name: " + columnName + " found in table " +
-                                     ".Please check create table statement."
-                  throw new MalformedCarbonCommandException(errorMessage)
-                }
-                cols.asScala.map { col =>
-                  val columnName = col.getName()
-                  val dataType = Option(col.getType)
-                  val name = Option(col.getName())
-                  // This is to parse complex data types
-                  val x = '`' + col.getName + '`' + ' ' + col.getType
-                  val f: Field = anyFieldDef(new lexical.Scanner(x))
-                  match {
-                    case Success(field, _) => field
-                    case failureOrError => throw new MalformedCarbonCommandException(
-                      s"Unsupported data type: $col.getType")
-                  }
-                  // the data type of the decimal type will be like decimal(10,0)
-                  // so checking the start of the string and taking the precision and scale.
-                  // resetting the data type with decimal
-                  if (f.dataType.getOrElse("").startsWith("decimal")) {
-                    val (precision, scale) = getScaleAndPrecision(col.getType)
-                    f.precision = precision
-                    f.scale = scale
-                    f.dataType = Some("decimal")
-                  }
-                  if (f.dataType.getOrElse("").startsWith("char")) {
-                    f.dataType = Some("char")
-                  } else if (f.dataType.getOrElse("").startsWith("float")) {
-                    f.dataType = Some("float")
-                  }
-                  f.rawSchema = x
-                  fields ++= Seq(f)
-                }
-              }
-
-            case Token("TOK_IFNOTEXISTS", _) =>
-              ifNotExistPresent = true
-
-            case t@Token("TOK_TABNAME", _) =>
-              val (db, tblName) = extractDbNameTableName(t)
-              dbName = db
-              tableName = tblName.toLowerCase()
-
-            case Token("TOK_TABLECOMMENT", child :: Nil) =>
-              tableComment = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
-
-            case Token("TOK_TABLEPARTCOLS", list@Token("TOK_TABCOLLIST", _) :: Nil) =>
-              val cols = BaseSemanticAnalyzer.getColumns(list(0), false)
-              if (cols != null) {
-                cols.asScala.map { col =>
-                  val columnName = col.getName()
-                  val dataType = Option(col.getType)
-                  val comment = col.getComment
-                  val rawSchema = '`' + col.getName + '`' + ' ' + col.getType
-                  val field = Field(columnName, dataType, Some(columnName), None)
-
-                  // the data type of the decimal type will be like decimal(10,0)
-                  // so checking the start of the string and taking the precision and scale.
-                  // resetting the data type with decimal
-                  if (field.dataType.getOrElse("").startsWith("decimal")) {
-                    val (precision, scale) = getScaleAndPrecision(col.getType)
-                    field.precision = precision
-                    field.scale = scale
-                    field.dataType = Some("decimal")
-                  }
-                  if (field.dataType.getOrElse("").startsWith("char")) {
-                    field.dataType = Some("char")
-                  } else if (field.dataType.getOrElse("").startsWith("float")) {
-                    field.dataType = Some("float")
-                  }
-                  field.rawSchema = rawSchema
-                  val partitionCol = new PartitionerField(columnName, dataType, comment)
-                  partitionCols ++= Seq(partitionCol)
-                  partitionByFields ++= Seq(field)
-                }
-              }
-            case Token("TOK_TABLEPROPERTIES", list :: Nil) =>
-              val propertySeq: Seq[(String, String)] = getProperties(list)
-              val repeatedProperties = propertySeq.groupBy(_._1).filter(_._2.size > 1).keySet
-              if (repeatedProperties.nonEmpty) {
-                val repeatedPropStr: String = repeatedProperties.mkString(",")
-                throw new MalformedCarbonCommandException("Table properties is repeated: " +
-                                                          repeatedPropStr)
-              }
-              tableProperties ++= propertySeq
-
-            case Token("TOK_LIKETABLE", child :: Nil) =>
-              likeTableName = child.getChild(0).getText()
-            case Token("TOK_ALTERTABLE_BUCKETS",
-            Token("TOK_TABCOLNAME", list) :: numberOfBuckets) =>
-              val cols = list.map(_.getText)
-              if (cols != null) {
-                bucketFields = Some(BucketFields(cols,
-                  numberOfBuckets.head.getText.toInt))
-              }
-
-            case _ => // Unsupport features
-          }
-
-          // validate tblProperties
-          if (!CommonUtil.validateTblProperties(tableProperties, fields)) {
-            throw new MalformedCarbonCommandException("Invalid table properties")
-          }
-
-          if (partitionCols.nonEmpty) {
-            if (!CommonUtil.validatePartitionColumns(tableProperties, partitionCols)) {
-              throw new MalformedCarbonCommandException("Invalid partition definition")
-            }
-            // partition columns should not be part of the schema
-            val colNames = fields.map(_.column)
-            val badPartCols = partitionCols.map(_.partitionColumn).toSet.intersect(colNames.toSet)
-            if (badPartCols.nonEmpty) {
-              throw new MalformedCarbonCommandException(
-                "Partition columns should not be specified in the schema: " +
-                badPartCols.map("\"" + _ + "\"").mkString("[", ",", "]"))
-            }
-            fields ++= partitionByFields
-          }
-
-          // prepare table model of the collected tokens
-          val tableModel: TableModel = prepareTableModel(ifNotExistPresent,
-            dbName,
-            tableName,
-            fields,
-            partitionCols,
-            tableProperties,
-            bucketFields)
-
-          // get logical plan.
-          CreateTable(tableModel)
-        } catch {
-          case ce: MalformedCarbonCommandException =>
-            val message = if (tableName.isEmpty) {
-              "Create table command failed. "
-            }
-            else if (dbName.isEmpty) {
-              s"Create table command failed for $tableName. "
-            }
-            else {
-              s"Create table command failed for ${ dbName.get }.$tableName. "
-            }
-            LOGGER.audit(message + ce.getMessage)
-            throw ce
-        }
-
-    }
-  }
-
-  /**
-   * This function will traverse the tree and logical plan will be formed using that.
-   *
-   * @param node
-   * @return LogicalPlan
-   */
-  protected def nodeToPlanForAlterTable(node: Node, alterSql: String): LogicalPlan = {
-    node match {
-      // if create table taken is found then only we will handle.
-      case Token("TOK_ALTERTABLE", children) =>
-
-        var dbName: Option[String] = None
-        var tableName: String = ""
-        var compactionType: String = ""
-
-        children.collect {
-
-          case t@Token("TOK_TABNAME", _) =>
-            val (db, tblName) = extractDbNameTableName(t)
-            dbName = db
-            tableName = tblName
-
-          case Token("TOK_ALTERTABLE_COMPACT", child :: Nil) =>
-            compactionType = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
-
-          case _ => // Unsupport features
-        }
-
-        val altertablemodel = AlterTableModel(dbName,
-          tableName,
-          None,
-          compactionType,
-          Some(System.currentTimeMillis()),
-          alterSql)
-        AlterTableCompaction(altertablemodel)
-    }
-  }
-
-  protected lazy val loadDataNew: Parser[LogicalPlan] =
-    LOAD ~> DATA ~> opt(LOCAL) ~> INPATH ~> stringLit ~ opt(OVERWRITE) ~
-    (INTO ~> TABLE ~> (ident <~ ".").? ~ ident) ~
-    (OPTIONS ~> "(" ~> repsep(loadOptions, ",") <~ ")").? <~ opt(";") ^^ {
-      case filePath ~ isOverwrite ~ table ~ optionsList =>
-        val (databaseNameOp, tableName) = table match {
-          case databaseName ~ tableName => (databaseName, tableName.toLowerCase())
-        }
-        if (optionsList.isDefined) {
-          validateOptions(optionsList)
-        }
-        val optionsMap = optionsList.getOrElse(List.empty[(String, String)]).toMap
-        LoadTable(convertDbNameToLowerCase(databaseNameOp), tableName, filePath, Seq(), optionsMap,
-          isOverwrite.isDefined)
-    }
-
-  protected lazy val describeTable: Parser[LogicalPlan] =
-    ((DESCRIBE | DESC) ~> opt(EXTENDED | FORMATTED)) ~ (ident <~ ".").? ~ ident ^^ {
-      case ef ~ db ~ tbl =>
-        val tblIdentifier = db match {
-          case Some(dbName) =>
-            TableIdentifier(tbl.toLowerCase, Some(convertDbNameToLowerCase(dbName)))
-          case None =>
-            TableIdentifier(tbl.toLowerCase, None)
-        }
-        if (ef.isDefined && "FORMATTED".equalsIgnoreCase(ef.get)) {
-          new DescribeFormattedCommand("describe formatted " + tblIdentifier,
-            tblIdentifier)
-        } else {
-          new DescribeCommand(UnresolvedRelation(tblIdentifier, None), ef.isDefined)
-        }
-    }
-
-  protected lazy val showLoads: Parser[LogicalPlan] =
-    SHOW ~> SEGMENTS ~> FOR ~> TABLE ~> (ident <~ ".").? ~ ident ~
-    (LIMIT ~> numericLit).? <~
-    opt(";") ^^ {
-      case databaseName ~ tableName ~ limit =>
-        ShowLoadsCommand(convertDbNameToLowerCase(databaseName), tableName.toLowerCase(), limit)
-    }
-
-  protected lazy val deleteLoadsByID: Parser[LogicalPlan] =
-  DELETE ~> FROM ~ TABLE ~> (ident <~ ".").? ~ ident ~
-  (WHERE ~> (SEGMENT ~ "." ~ ID) ~> IN ~> "(" ~> repsep(segmentId, ",")) <~ ")" ~ opt(";") ^^ {
-    case dbName ~ tableName ~ loadids =>
-      DeleteLoadsById(loadids, convertDbNameToLowerCase(dbName), tableName.toLowerCase())
-  }
-
-  protected lazy val deleteLoadsByLoadDate: Parser[LogicalPlan] =
-    DELETE ~> FROM ~> TABLE ~> (ident <~ ".").? ~ ident ~
-    (WHERE ~> (SEGMENT ~ "." ~ STARTTIME ~> BEFORE) ~ stringLit) <~
-    opt(";") ^^ {
-      case database ~ table ~ condition =>
-        condition match {
-          case dateField ~ dateValue =>
-            DeleteLoadsByLoadDate(convertDbNameToLowerCase(database),
-              table.toLowerCase(),
-              dateField,
-              dateValue)
-        }
-    }
-
-  protected lazy val cleanFiles: Parser[LogicalPlan] =
-    CLEAN ~> FILES ~> FOR ~> TABLE ~> (ident <~ ".").? ~ ident <~ opt(";") ^^ {
-      case databaseName ~ tableName =>
-        CleanFiles(convertDbNameToLowerCase(databaseName), tableName.toLowerCase())
-    }
-
-  protected lazy val explainPlan: Parser[LogicalPlan] =
-    (EXPLAIN ~> opt(EXTENDED)) ~ startCommand ^^ {
-      case isExtended ~ logicalPlan =>
-        logicalPlan match {
-          case plan: CreateTable => ExplainCommand(logicalPlan, extended = isExtended.isDefined)
-          case _ => ExplainCommand(OneRowRelation)
-        }
-    }
-
-  protected lazy val deleteRecords: Parser[LogicalPlan] =
-    (DELETE ~> FROM ~> table) ~ restInput.? <~ opt(";") ^^ {
-      case table ~ rest =>
-        val tableName = getTableName(table.tableIdentifier)
-        val alias = table.alias.getOrElse("")
-        DeleteRecords("select tupleId from " + tableName + " " + alias + rest.getOrElse(""), table)
-    }
-
-  protected lazy val updateTable: Parser[LogicalPlan] =
-    UPDATE ~> table ~
-    (SET ~> "(" ~> repsep(element, ",") <~ ")") ~
-    ("=" ~> restInput) <~ opt(";") ^^ {
-      case tab ~ columns ~ rest =>
-        val (sel, where) = splitQuery(rest)
-        val (selectStmt, relation) =
-          if (!sel.toLowerCase.startsWith("select ")) {
-            if (sel.trim.isEmpty) {
-              sys.error("At least one source column has to be specified ")
-            }
-            // only list of expression are given, need to convert that list of expressions into
-            // select statement on destination table
-            val relation = tab match {
-              case r@UnresolvedRelation(tableIdentifier, alias) =>
-                updateRelation(r, tableIdentifier, alias)
-              case _ => tab
-            }
-            ("select " + sel + " from " + getTableName(relation.tableIdentifier) + " " +
-             relation.alias.get, relation)
-          } else {
-            (sel, updateRelation(tab, tab.tableIdentifier, tab.alias))
-          }
-        UpdateTable(relation, columns, selectStmt, where)
-    }
-  protected lazy val showPartitions: Parser[LogicalPlan] =
-    (SHOW ~> PARTITIONS ~> table) <~ opt(";") ^^ {
-      case table =>
-        val tableName = getTableName(table.tableIdentifier)
-        val alias = table.alias.getOrElse("")
-        ShowPartitions(table.tableIdentifier)
-    }
-
-  private def splitQuery(query: String): (String, String) = {
-    val stack = scala.collection.mutable.Stack[Char]()
-    var foundSingleQuotes = false
-    var foundDoubleQuotes = false
-    var foundEscapeChar = false
-    var ignoreChar = false
-    var stop = false
-    var bracketCount = 0
-    val (selectStatement, where) = query.span {
-      ch => {
-        if (stop) {
-          false
-        } else {
-          ignoreChar = false
-          if (foundEscapeChar && (ch == '\'' || ch == '\"' || ch == '\\')) {
-            foundEscapeChar = false
-            ignoreChar = true
-          }
-          // If escaped single or double quotes found, no need to consider
-          if (!ignoreChar) {
-            if (ch == '\\') {
-              foundEscapeChar = true
-            } else if (ch == '\'') {
-              foundSingleQuotes = !foundSingleQuotes
-            } else if (ch == '\"') {
-              foundDoubleQuotes = !foundDoubleQuotes
-            }
-            else if (ch == '(' && !foundSingleQuotes && !foundDoubleQuotes) {
-              bracketCount = bracketCount + 1
-              stack.push(ch)
-            } else if (ch == ')' && !foundSingleQuotes && !foundDoubleQuotes) {
-              bracketCount = bracketCount + 1
-              stack.pop()
-              if (0 == stack.size) {
-                stop = true
-              }
-            }
-          }
-          true
-        }
-      }
-    }
-    if (bracketCount == 0 || bracketCount % 2 != 0) {
-      sys.error("Parsing error, missing bracket ")
-    }
-    val select = selectStatement.trim
-    (select.substring(1, select.length - 1).trim -> where.trim)
-  }
-
-
-  protected lazy val table: Parser[UnresolvedRelation] = {
-    rep1sep(attributeName, ".") ~ opt(ident) ^^ {
-      case tableIdent ~ alias => UnresolvedRelation(tableIdent, alias)
-    }
-  }
-
-  protected lazy val attributeName: Parser[String] = acceptMatch("attribute name", {
-    case lexical.Identifier(str) => str.toLowerCase
-    case lexical.Keyword(str) if !lexical.delimiters.contains(str) => str.toLowerCase
-  })
-
-  private def updateRelation(
-      r: UnresolvedRelation,
-      tableIdentifier: Seq[String],
-      alias: Option[String]): UnresolvedRelation = {
-    alias match {
-      case Some(_) => r
-      case _ =>
-        val tableAlias = tableIdentifier match {
-          case Seq(dbName, tableName) => Some(tableName)
-          case Seq(tableName) => Some(tableName)
-        }
-        UnresolvedRelation(tableIdentifier, tableAlias)
-    }
-  }
-
-  private def getTableName(tableIdentifier: Seq[String]): String = {
-    if (tableIdentifier.size > 1) {
-      tableIdentifier(0) + "." + tableIdentifier(1)
-    } else {
-      tableIdentifier(0)
-    }
-  }
-
-  protected lazy val element: Parser[String] =
-    (ident <~ ".").? ~ ident ^^ {
-      case table ~ column => column.toLowerCase
-    }
-
-  protected lazy val useDatabase: Parser[LogicalPlan] =
-    USE ~> ident <~ opt(";") ^^ {
-      case databaseName => UseDatabase(s"use ${ databaseName.toLowerCase }")
-    }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CodeGenerateFactory.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CodeGenerateFactory.scala
deleted file mode 100644
index 065dfed..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CodeGenerateFactory.scala
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.catalyst.CatalystConf
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.plans.logical.{Expand, LogicalPlan}
-import org.apache.spark.util.{ScalaCompilerUtil, Utils}
-
-private[sql] class CodeGenerateFactory(version: String) {
-
-  val optimizerFactory = if (version.equals("1.6.2") || version.equals("1.6.3")) {
-    ScalaCompilerUtil.compiledCode(CodeTemplates.spark1_6_OptimizerString)
-      .asInstanceOf[AbstractCarbonOptimizerFactory]
-  } else if (version.startsWith("1.6") || version.startsWith("1.5")) {
-    ScalaCompilerUtil.compiledCode(CodeTemplates.defaultOptimizerString)
-      .asInstanceOf[AbstractCarbonOptimizerFactory]
-  } else {
-    throw new UnsupportedOperationException(s"Spark version $version is not supported")
-  }
-
-  val expandFactory = if (version.startsWith("1.5")) {
-    ScalaCompilerUtil.compiledCode(CodeTemplates.spark1_5ExpandString)
-      .asInstanceOf[AbstractCarbonExpandFactory]
-  } else if (version.startsWith("1.6")) {
-    new AbstractCarbonExpandFactory {
-      override def createExpand(expand: Expand, child: LogicalPlan): Expand = {
-        val loader = Utils.getContextOrSparkClassLoader
-        try {
-          val cons = loader.loadClass("org.apache.spark.sql.catalyst.plans.logical.Expand")
-            .getDeclaredConstructors
-          cons.head.setAccessible(true)
-          cons.head.newInstance(expand.projections, expand.output, child).asInstanceOf[Expand]
-        } catch {
-          case e: Exception => null
-        }
-      }
-    }
-  } else {
-    throw new UnsupportedOperationException(s"Spark version $version is not supported")
-  }
-
-}
-
-object CodeGenerateFactory {
-
-  private var codeGenerateFactory: CodeGenerateFactory = _
-
-  def init(version: String): Unit = {
-    if (codeGenerateFactory == null) {
-      codeGenerateFactory = new CodeGenerateFactory(version)
-    }
-  }
-
-  def getInstance(): CodeGenerateFactory = {
-    codeGenerateFactory
-  }
-
-  def createDefaultOptimizer(conf: CatalystConf, sc: SparkContext): Optimizer = {
-    val name = "org.apache.spark.sql.catalyst.optimizer.DefaultOptimizer"
-    val loader = Utils.getContextOrSparkClassLoader
-    try {
-      val cons = loader.loadClass(name + "$").getDeclaredConstructors
-      cons.head.setAccessible(true)
-      cons.head.newInstance().asInstanceOf[Optimizer]
-    } catch {
-      case e: Exception =>
-        loader.loadClass(name).getConstructor(classOf[CatalystConf])
-          .newInstance(conf).asInstanceOf[Optimizer]
-    }
-  }
-
-}
-
-object CodeTemplates {
-
-  val spark1_6_OptimizerString =
-    s"""
-       import org.apache.spark.sql._;
-       import org.apache.spark.sql.optimizer._;
-       import org.apache.spark.sql.catalyst.plans.logical._;
-       import org.apache.spark.sql.catalyst._;
-       import org.apache.spark.sql.catalyst.optimizer.Optimizer;
-
-       new AbstractCarbonOptimizerFactory {
-         override def createOptimizer(optimizer: Optimizer, conf: CarbonSQLConf): Optimizer = {
-           class CarbonOptimizer1(optimizer: Optimizer, conf: CarbonSQLConf)
-             extends Optimizer(conf) {
-             override val batches = Nil;
-             override def execute(plan: LogicalPlan): LogicalPlan = {
-               CarbonOptimizer.execute(plan, optimizer);
-             }
-           }
-           new CarbonOptimizer1(optimizer, conf);
-         }
-       }
-    """
-
-  val defaultOptimizerString =
-    s"""
-       import org.apache.spark.sql._;
-       import org.apache.spark.sql.optimizer._;
-       import org.apache.spark.sql.catalyst.plans.logical._;
-       import org.apache.spark.sql.catalyst._;
-       import org.apache.spark.sql.catalyst.optimizer.Optimizer;
-
-       new AbstractCarbonOptimizerFactory {
-         override def createOptimizer(optimizer: Optimizer, conf: CarbonSQLConf): Optimizer = {
-           class CarbonOptimizer2(optimizer: Optimizer, conf: CarbonSQLConf) extends Optimizer {
-             val batches = Nil;
-             override def execute(plan: LogicalPlan): LogicalPlan = {
-               CarbonOptimizer.execute(plan, optimizer);
-             }
-           }
-           new CarbonOptimizer2(optimizer, conf);
-         }
-       }
-    """
-
-  val spark1_5ExpandString =
-    s"""
-       import org.apache.spark.sql._
-       import org.apache.spark.sql.catalyst.plans.logical.{Expand, LogicalPlan}
-       new AbstractCarbonExpandFactory {
-         override def createExpand(expand: Expand, child: LogicalPlan): Expand = {
-           Expand(expand.bitmasks, expand.groupByExprs, expand.gid, child)
-         }
-       }
-    """
-}
-
-abstract class AbstractCarbonOptimizerFactory {
-  def createOptimizer(optimizer: Optimizer, conf: CarbonSQLConf) : Optimizer
-}
-
-abstract class AbstractCarbonExpandFactory {
-  def createExpand(expand: Expand, child: LogicalPlan) : Expand
-}
-
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
deleted file mode 100644
index d745be2..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodeGenContext, GeneratedExpressionCode}
-import org.apache.spark.sql.types.{DataType, StringType}
-
-/**
- * Custom expression to override the deterministic property
- *
- */
-case class CustomDeterministicExpression(nonDt: Expression ) extends Expression with Serializable{
-  override def nullable: Boolean = true
-
-  override def eval(input: InternalRow): Any = null
-
-  override protected def genCode(ctx: CodeGenContext,
-      ev: GeneratedExpressionCode): String = ev.code
-  override def deterministic: Boolean = true
-
-  override def dataType: DataType = StringType
-
-  override def children: Seq[Expression] = Seq()
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
deleted file mode 100644
index dc2dd7b..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression, GenericMutableRow}
-
-import org.apache.carbondata.core.scan.expression.{ColumnExpression, Expression, ExpressionResult, UnknownExpression}
-import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException
-import org.apache.carbondata.core.scan.filter.intf.{ExpressionType, RowIntf}
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-class SparkUnknownExpression(var sparkExp: SparkExpression)
-  extends UnknownExpression with ConditionalExpression {
-
-  private var evaluateExpression: (InternalRow) => Any = sparkExp.eval
-  private var isExecutor: Boolean = false
-  children.addAll(getColumnList())
-
-  override def evaluate(carbonRowInstance: RowIntf): ExpressionResult = {
-
-    val values = carbonRowInstance.getValues.toSeq.map {
-      case s: String => org.apache.spark.unsafe.types.UTF8String.fromString(s)
-      case d: java.math.BigDecimal => org.apache.spark.sql.types.Decimal.apply(d)
-      case value => value
-    }
-    try {
-      val result = evaluateExpression(
-        new GenericMutableRow(values.map(a => a.asInstanceOf[Any]).toArray))
-      val sparkRes = if (isExecutor) {
-        result.asInstanceOf[InternalRow].get(0, sparkExp.dataType)
-      } else {
-        result
-      }
-      new ExpressionResult(CarbonScalaUtil.convertSparkToCarbonDataType(sparkExp.dataType),
-        sparkRes
-      )
-    } catch {
-      case e: Exception => throw new FilterUnsupportedException(e.getMessage)
-    }
-  }
-
-  override def getFilterExpressionType: ExpressionType = {
-    ExpressionType.UNKNOWN
-  }
-
-  override def getString: String = {
-    sparkExp.toString()
-  }
-
-  def setEvaluateExpression(evaluateExpression: (InternalRow) => Any): Unit = {
-    this.evaluateExpression = evaluateExpression
-    isExecutor = true
-  }
-
-  override def findAndSetChild(oldExpr: Expression, newExpr: Expression): Unit = {}
-
-  def getColumnList: java.util.List[ColumnExpression] = {
-
-    val lst = new java.util.ArrayList[ColumnExpression]()
-    getColumnListFromExpressionTree(sparkExp, lst)
-    lst
-  }
-  def getLiterals: java.util.List[ExpressionResult] = {
-
-    val lst = new java.util.ArrayList[ExpressionResult]()
-    lst
-  }
-
-  def getAllColumnList: java.util.List[ColumnExpression] = {
-    val lst = new java.util.ArrayList[ColumnExpression]()
-    getAllColumnListFromExpressionTree(sparkExp, lst)
-    lst
-  }
-
-  def isSingleColumn: Boolean = {
-    val lst = new java.util.ArrayList[ColumnExpression]()
-    getAllColumnListFromExpressionTree(sparkExp, lst)
-    if (lst.size == 1 && lst.get(0).isDimension) {
-      true
-    } else {
-      false
-    }
-  }
-
-  def getColumnListFromExpressionTree(sparkCurrentExp: SparkExpression,
-      list: java.util.List[ColumnExpression]): Unit = {
-    sparkCurrentExp match {
-      case carbonBoundRef: CarbonBoundReference =>
-        val foundExp = list.asScala
-          .find(p => p.getColumnName == carbonBoundRef.colExp.getColumnName)
-        if (foundExp.isEmpty) {
-          carbonBoundRef.colExp.setColIndex(list.size)
-          list.add(carbonBoundRef.colExp)
-        } else {
-          carbonBoundRef.colExp.setColIndex(foundExp.get.getColIndex)
-        }
-      case _ => sparkCurrentExp.children.foreach(getColumnListFromExpressionTree(_, list))
-    }
-  }
-
-
-  def getAllColumnListFromExpressionTree(sparkCurrentExp: SparkExpression,
-      list: java.util.List[ColumnExpression]): java.util.List[ColumnExpression] = {
-    sparkCurrentExp match {
-      case carbonBoundRef: CarbonBoundReference => list.add(carbonBoundRef.colExp)
-      case _ => sparkCurrentExp.children.foreach(getColumnListFromExpressionTree(_, list))
-    }
-    list
-  }
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
deleted file mode 100644
index cb35960..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ /dev/null
@@ -1,842 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.command
-
-import java.util
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ListBuffer
-
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
-import org.apache.spark.sql.execution.RunnableCommand
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.storage.StorageLevel
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, DeleteDeltaBlockDetails, SegmentUpdateDetails, TupleIdEnum}
-import org.apache.carbondata.core.mutate.data.RowCountDetailsVO
-import org.apache.carbondata.core.statusmanager.{SegmentStatusManager, SegmentUpdateStatusManager}
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl
-import org.apache.carbondata.processing.exception.MultipleMatchingException
-import org.apache.carbondata.processing.loading.FailureCauses
-import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CarbonDataMergerUtilResult, CompactionType}
-import org.apache.carbondata.spark.DeleteDelataResultImpl
-import org.apache.carbondata.spark.util.QueryPlanUtil
-
-
-/**
- * IUD update delete and compaction framework.
- *
- */
-
-private[sql] case class ProjectForDeleteCommand(
-     plan: LogicalPlan,
-     identifier: Seq[String],
-     timestamp: String) extends RunnableCommand {
-
-  val LOG = LogServiceFactory.getLogService(this.getClass.getName)
-  var horizontalCompactionFailed = false
-
-  override def run(sqlContext: SQLContext): Seq[Row] = {
-
-    val dataFrame = DataFrame(sqlContext, plan)
-    val dataRdd = dataFrame.rdd
-
-    val relation = CarbonEnv.get.carbonMetastore
-      .lookupRelation1(deleteExecution.getTableIdentifier(identifier))(sqlContext).
-      asInstanceOf[CarbonRelation]
-    val carbonTable = relation.tableMeta.carbonTable
-    val metadataLock = CarbonLockFactory
-      .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
-        LockUsage.METADATA_LOCK)
-    var lockStatus = false
-    try {
-      lockStatus = metadataLock.lockWithRetries()
-      LOG.audit(s" Delete data request has been received " +
-                s"for ${ relation.databaseName }.${ relation.tableName }.")
-      if (lockStatus) {
-        LOG.info("Successfully able to get the table metadata file lock")
-      }
-      else {
-        throw new Exception("Table is locked for deletion. Please try after some time")
-      }
-      val tablePath = CarbonStorePath.getCarbonTablePath(
-        carbonTable.getStorePath,
-        carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier)
-      var executorErrors = new ExecutionErrors(FailureCauses.NONE, "")
-
-        // handle the clean up of IUD.
-        CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, false)
-
-          if (deleteExecution
-            .deleteDeltaExecution(identifier, sqlContext, dataRdd, timestamp, relation,
-              false, executorErrors)) {
-            // call IUD Compaction.
-            IUDCommon.tryHorizontalCompaction(sqlContext, relation, isUpdateOperation = false)
-          }
-    } catch {
-      case e: HorizontalCompactionException =>
-          LOG.error("Delete operation passed. Exception in Horizontal Compaction." +
-              " Please check logs. " + e.getMessage)
-          CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, e.compactionTimeStamp.toString)
-
-      case e: Exception =>
-        LOG.error("Exception in Delete data operation " + e.getMessage)
-        // ****** start clean up.
-        // In case of failure , clean all related delete delta files
-        CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timestamp)
-
-        // clean up. Null check is required as for executor error some times message is null
-        if (null != e.getMessage) {
-          sys.error("Delete data operation is failed. " + e.getMessage)
-        }
-        else {
-          sys.error("Delete data operation is failed. Please check logs.")
-        }
-    } finally {
-      if (lockStatus) {
-        CarbonLockUtil.fileUnlock(metadataLock, LockUsage.METADATA_LOCK)
-      }
-    }
-    Seq.empty
-  }
-}
-
-private[sql] case class ProjectForUpdateCommand(
-    plan: LogicalPlan, tableIdentifier: Seq[String]) extends RunnableCommand {
-  val LOGGER = LogServiceFactory.getLogService(ProjectForUpdateCommand.getClass.getName)
-
-  override def run(sqlContext: SQLContext): Seq[Row] = {
-
-    val res = plan find {
-      case relation: LogicalRelation if (relation.relation
-        .isInstanceOf[CarbonDatasourceRelation]) =>
-        true
-      case _ => false
-    }
-
-    if (!res.isDefined) {
-      return Seq.empty
-    }
-
-    val relation = CarbonEnv.get.carbonMetastore
-      .lookupRelation1(deleteExecution.getTableIdentifier(tableIdentifier))(sqlContext).
-      asInstanceOf[CarbonRelation]
-    val carbonTable = relation.tableMeta.carbonTable
-    val metadataLock = CarbonLockFactory
-      .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
-        LockUsage.METADATA_LOCK)
-    var lockStatus = false
-    // get the current time stamp which should be same for delete and update.
-    val currentTime = CarbonUpdateUtil.readCurrentTime
-    var dataFrame: DataFrame = null
-    val isPersistEnabledUserValue = CarbonProperties.getInstance
-      .getProperty(CarbonCommonConstants.isPersistEnabled,
-        CarbonCommonConstants.defaultValueIsPersistEnabled)
-   var isPersistEnabled = CarbonCommonConstants.defaultValueIsPersistEnabled.toBoolean
-    if (isPersistEnabledUserValue.equalsIgnoreCase("false")) {
-      isPersistEnabled = false
-    }
-    else if (isPersistEnabledUserValue.equalsIgnoreCase("true")) {
-      isPersistEnabled = true
-    }
-    try {
-      lockStatus = metadataLock.lockWithRetries()
-      if (lockStatus) {
-        logInfo("Successfully able to get the table metadata file lock")
-      }
-      else {
-        throw new Exception("Table is locked for updation. Please try after some time")
-      }
-      val tablePath = CarbonStorePath.getCarbonTablePath(
-        carbonTable.getStorePath,
-        carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier)
-        // Get RDD.
-        dataFrame = if (isPersistEnabled) {
-          DataFrame(sqlContext, plan)
-            .persist(StorageLevel.MEMORY_AND_DISK)
-        }
-        else {
-          DataFrame(sqlContext, plan)
-        }
-        var executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
-
-
-        // handle the clean up of IUD.
-        CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, false)
-
-        // do delete operation.
-        deleteExecution.deleteDeltaExecution(tableIdentifier, sqlContext, dataFrame.rdd,
-          currentTime + "",
-        relation, isUpdateOperation = true, executionErrors)
-
-        if(executionErrors.failureCauses != FailureCauses.NONE) {
-          throw new Exception(executionErrors.errorMsg)
-        }
-
-        // do update operation.
-        UpdateExecution.performUpdate(dataFrame, tableIdentifier, plan,
-          sqlContext, currentTime, executionErrors)
-
-        if(executionErrors.failureCauses != FailureCauses.NONE) {
-          throw new Exception(executionErrors.errorMsg)
-        }
-
-        // Do IUD Compaction.
-        IUDCommon.tryHorizontalCompaction(sqlContext, relation, isUpdateOperation = true)
-    }
-
-    catch {
-      case e: HorizontalCompactionException =>
-        LOGGER.error(
-            "Update operation passed. Exception in Horizontal Compaction. Please check logs." + e)
-        // In case of failure , clean all related delta files
-        CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, e.compactionTimeStamp.toString)
-
-      case e: Exception =>
-        LOGGER.error("Exception in update operation" + e)
-        // ****** start clean up.
-        // In case of failure , clean all related delete delta files
-        CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, currentTime + "")
-
-        // *****end clean up.
-        if (null != e.getMessage) {
-          sys.error("Update operation failed. " + e.getMessage)
-        }
-        if (null != e.getCause && null != e.getCause.getMessage) {
-          sys.error("Update operation failed. " + e.getCause.getMessage)
-        }
-        sys.error("Update operation failed. please check logs.")
-    }
-    finally {
-      if (null != dataFrame && isPersistEnabled) {
-        dataFrame.unpersist()
-      }
-      if (lockStatus) {
-        CarbonLockUtil.fileUnlock(metadataLock, LockUsage.METADATA_LOCK)
-      }
-    }
-    Seq.empty
-  }
-}
-
-object IUDCommon {
-
-  val LOG = LogServiceFactory.getLogService(this.getClass.getName)
-
-  /**
-   * The method does horizontal compaction. After Update and Delete completion
-   * tryHorizontal compaction will be called. In case this method is called after
-   * Update statement then Update Compaction followed by Delete Compaction will be
-   * processed whereas for tryHorizontalCompaction called after Delete statement
-   * then only Delete Compaction will be processed.
-    *
-    * @param sqlContext
-   * @param carbonRelation
-   * @param isUpdateOperation
-   */
-  def tryHorizontalCompaction(sqlContext: SQLContext,
-      carbonRelation: CarbonRelation,
-      isUpdateOperation: Boolean): Unit = {
-
-    var ishorizontalCompaction = CarbonDataMergerUtil.isHorizontalCompactionEnabled()
-
-    if (ishorizontalCompaction == false) {
-      return
-    }
-
-    var compactionTypeIUD = CompactionType.IUD_UPDDEL_DELTA_COMPACTION
-    val carbonTable = carbonRelation.tableMeta.carbonTable
-    val (db, table) = (carbonTable.getDatabaseName, carbonTable.getFactTableName)
-    val absTableIdentifier = carbonTable.getAbsoluteTableIdentifier
-    val updateTimeStamp = System.currentTimeMillis()
-    // To make sure that update and delete timestamps are not same,
-    // required to commit to status metadata and cleanup
-    val deleteTimeStamp = updateTimeStamp + 1
-
-    // get the valid segments
-    var segLists = CarbonDataMergerUtil.getValidSegmentList(absTableIdentifier)
-
-    if (segLists == null || segLists.size() == 0) {
-      return
-    }
-
-    // Should avoid reading Table Status file from Disk every time. Better to load it
-    // in-memory at the starting and pass it along the routines. The constructor of
-    // SegmentUpdateStatusManager reads the Table Status File and Table Update Status
-    // file and save the content in segmentDetails and updateDetails respectively.
-    val segmentUpdateStatusManager: SegmentUpdateStatusManager = new SegmentUpdateStatusManager(
-      absTableIdentifier)
-
-    if (isUpdateOperation == true) {
-
-      // This is only update operation, perform only update compaction.
-      compactionTypeIUD = CompactionType.IUD_UPDDEL_DELTA_COMPACTION
-      performUpdateDeltaCompaction(sqlContext,
-        compactionTypeIUD,
-        carbonTable,
-        absTableIdentifier,
-        segmentUpdateStatusManager,
-        updateTimeStamp,
-        segLists)
-    }
-
-    // After Update Compaction perform delete compaction
-    compactionTypeIUD = CompactionType.IUD_DELETE_DELTA_COMPACTION
-    segLists = CarbonDataMergerUtil.getValidSegmentList(absTableIdentifier)
-    if (segLists == null || segLists.size() == 0) {
-      return
-    }
-
-    // Delete Compaction
-    performDeleteDeltaCompaction(sqlContext,
-      compactionTypeIUD,
-      carbonTable,
-      absTableIdentifier,
-      segmentUpdateStatusManager,
-      deleteTimeStamp,
-      segLists)
-  }
-
-  /**
-   * Update Delta Horizontal Compaction.
-    *
-    * @param sqlContext
-   * @param compactionTypeIUD
-   * @param carbonTable
-   * @param absTableIdentifier
-   * @param segLists
-   */
-  private def performUpdateDeltaCompaction(sqlContext: SQLContext,
-      compactionTypeIUD: CompactionType,
-      carbonTable: CarbonTable,
-      absTableIdentifier: AbsoluteTableIdentifier,
-      segmentUpdateStatusManager: SegmentUpdateStatusManager,
-      factTimeStamp: Long,
-      segLists: util.List[String]): Unit = {
-    val db = carbonTable.getDatabaseName
-    val table = carbonTable.getFactTableName
-    // get the valid segments qualified for update compaction.
-    val validSegList = CarbonDataMergerUtil.getSegListIUDCompactionQualified(segLists,
-      absTableIdentifier,
-      segmentUpdateStatusManager,
-      compactionTypeIUD)
-
-    if (validSegList.size() == 0) {
-      return
-    }
-
-    LOG.info(s"Horizontal Update Compaction operation started for [${db}.${table}].")
-    LOG.audit(s"Horizontal Update Compaction operation started for [${db}.${table}].")
-
-    try {
-      // Update Compaction.
-      val altertablemodel = AlterTableModel(Option(carbonTable.getDatabaseName),
-        carbonTable.getFactTableName,
-        Some(segmentUpdateStatusManager),
-        CompactionType.IUD_UPDDEL_DELTA_COMPACTION.toString,
-        Some(factTimeStamp),
-        "")
-
-      AlterTableCompaction(altertablemodel).run(sqlContext)
-    }
-    catch {
-      case e: Exception =>
-        val msg = if (null != e.getMessage) {
-          e.getMessage
-        } else {
-          "Please check logs for more info"
-        }
-        throw new HorizontalCompactionException(
-          s"Horizontal Update Compaction Failed for [${ db }.${ table }]. " + msg, factTimeStamp)
-    }
-    LOG.info(s"Horizontal Update Compaction operation completed for [${ db }.${ table }].")
-    LOG.audit(s"Horizontal Update Compaction operation completed for [${ db }.${ table }].")
-  }
-
-  /**
-   * Delete Delta Horizontal Compaction.
-    *
-    * @param sqlContext
-   * @param compactionTypeIUD
-   * @param carbonTable
-   * @param absTableIdentifier
-   * @param segLists
-   */
-  private def performDeleteDeltaCompaction(sqlContext: SQLContext,
-      compactionTypeIUD: CompactionType,
-      carbonTable: CarbonTable,
-      absTableIdentifier: AbsoluteTableIdentifier,
-      segmentUpdateStatusManager: SegmentUpdateStatusManager,
-      factTimeStamp: Long,
-      segLists: util.List[String]): Unit = {
-
-    val db = carbonTable.getDatabaseName
-    val table = carbonTable.getFactTableName
-    val deletedBlocksList = CarbonDataMergerUtil.getSegListIUDCompactionQualified(segLists,
-      absTableIdentifier,
-      segmentUpdateStatusManager,
-      compactionTypeIUD)
-
-    if (deletedBlocksList.size() == 0) {
-      return
-    }
-
-    LOG.info(s"Horizontal Delete Compaction operation started for [${db}.${table}].")
-    LOG.audit(s"Horizontal Delete Compaction operation started for [${db}.${table}].")
-
-    try {
-
-      // Delete Compaction RDD
-      val rdd1 = sqlContext.sparkContext
-        .parallelize(deletedBlocksList.asScala.toSeq, deletedBlocksList.size())
-
-      val timestamp = factTimeStamp
-      val updateStatusDetails = segmentUpdateStatusManager.getUpdateStatusDetails
-      val result = rdd1.mapPartitions(iter =>
-        new Iterator[Seq[CarbonDataMergerUtilResult]] {
-          override def hasNext: Boolean = iter.hasNext
-
-          override def next(): Seq[CarbonDataMergerUtilResult] = {
-            val segmentAndBlocks = iter.next
-            val segment = segmentAndBlocks.substring(0, segmentAndBlocks.lastIndexOf("/"))
-            val blockName = segmentAndBlocks
-              .substring(segmentAndBlocks.lastIndexOf("/") + 1, segmentAndBlocks.length)
-
-            val result = CarbonDataMergerUtil.compactBlockDeleteDeltaFiles(segment, blockName,
-              absTableIdentifier,
-              updateStatusDetails,
-              timestamp)
-
-            result.asScala.toList
-
-          }
-        }).collect
-
-      val resultList = ListBuffer[CarbonDataMergerUtilResult]()
-      result.foreach(x => {
-        x.foreach(y => {
-          resultList += y
-        })
-      })
-
-      val updateStatus = CarbonDataMergerUtil.updateStatusFile(resultList.toList.asJava,
-        carbonTable,
-        timestamp.toString,
-        segmentUpdateStatusManager)
-      if (updateStatus == false) {
-        LOG.audit(s"Delete Compaction data operation is failed for [${db}.${table}].")
-        LOG.error("Delete Compaction data operation is failed.")
-        throw new HorizontalCompactionException(
-          s"Horizontal Delete Compaction Failed for [${db}.${table}] ." +
-          s" Please check logs for more info.", factTimeStamp)
-      }
-      else {
-        LOG.info(s"Horizontal Delete Compaction operation completed for [${db}.${table}].")
-        LOG.audit(s"Horizontal Delete Compaction operation completed for [${db}.${table}].")
-      }
-    }
-    catch {
-      case e: Exception =>
-        val msg = if (null != e.getMessage) {
-          e.getMessage
-        } else {
-          "Please check logs for more info"
-        }
-        throw new HorizontalCompactionException(
-          s"Horizontal Delete Compaction Failed for [${ db }.${ table }]. " + msg, factTimeStamp)
-    }
-  }
-}
-
-class HorizontalCompactionException(
-    message: String,
-    // required for cleanup
-    val compactionTimeStamp: Long) extends RuntimeException(message) {
-}
-
-object deleteExecution {
-  val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
-
-  def getTableIdentifier(tableIdentifier: Seq[String]): TableIdentifier = {
-    if (tableIdentifier.size > 1) {
-      TableIdentifier(tableIdentifier(1), Some(tableIdentifier(0)))
-    } else {
-      TableIdentifier(tableIdentifier(0), None)
-    }
-  }
-
-  def deleteDeltaExecution(identifier: Seq[String],
-                           sqlContext: SQLContext,
-                           dataRdd: RDD[Row],
-                           timestamp: String, relation: CarbonRelation, isUpdateOperation: Boolean,
-                           executorErrors: ExecutionErrors): Boolean = {
-
-    var res: Array[List[(String, (SegmentUpdateDetails, ExecutionErrors))]] = null
-    val tableName = getTableIdentifier(identifier).table
-    val database = getDB.getDatabaseName(getTableIdentifier(identifier).database, sqlContext)
-    val relation = CarbonEnv.get.carbonMetastore
-      .lookupRelation1(getTableIdentifier(identifier))(sqlContext).
-      asInstanceOf[CarbonRelation]
-
-    val storeLocation = relation.tableMeta.storePath
-    val absoluteTableIdentifier: AbsoluteTableIdentifier = new
-        AbsoluteTableIdentifier(storeLocation,
-          relation.tableMeta.carbonTableIdentifier)
-    var tablePath = CarbonStorePath
-      .getCarbonTablePath(storeLocation,
-        absoluteTableIdentifier.getCarbonTableIdentifier())
-    var tableUpdateStatusPath = tablePath.getTableUpdateStatusFilePath
-    val totalSegments =
-      SegmentStatusManager.readLoadMetadata(tablePath.getMetadataDirectoryPath).length
-    var factPath = tablePath.getFactDir
-
-    var carbonTable = relation.tableMeta.carbonTable
-    var deleteStatus = true
-    val deleteRdd = if (isUpdateOperation) {
-      val schema =
-        org.apache.spark.sql.types.StructType(Seq(org.apache.spark.sql.types.StructField(
-          CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID,
-          org.apache.spark.sql.types.StringType)))
-      val rdd = dataRdd
-        .map(row => Row(row.get(row.fieldIndex(
-          CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))))
-      sqlContext.createDataFrame(rdd, schema).rdd
-    } else {
-      dataRdd
-    }
-
-    val (carbonInputFormat, job) =
-      QueryPlanUtil.createCarbonInputFormat(absoluteTableIdentifier)
-
-    val keyRdd = deleteRdd.map({ row =>
-      val tupleId: String = row
-        .getString(row.fieldIndex(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))
-      val key = CarbonUpdateUtil.getSegmentWithBlockFromTID(tupleId)
-      (key, row)
-    }).groupByKey()
-
-    // if no loads are present then no need to do anything.
-    if (keyRdd.partitions.size == 0) {
-      return true
-    }
-
-    var blockMappingVO = carbonInputFormat.getBlockRowCount(job, absoluteTableIdentifier)
-    val segmentUpdateStatusMngr = new SegmentUpdateStatusManager(absoluteTableIdentifier)
-    CarbonUpdateUtil
-      .createBlockDetailsMap(blockMappingVO, segmentUpdateStatusMngr)
-
-    val rowContRdd = sqlContext.sparkContext
-      .parallelize(blockMappingVO.getCompleteBlockRowDetailVO.asScala.toSeq,
-        keyRdd.partitions.size)
-
-    val rdd = rowContRdd.join(keyRdd)
-
-    res = rdd.mapPartitionsWithIndex(
-      (index: Int, records: Iterator[((String), (RowCountDetailsVO, Iterable[Row]))]) =>
-        Iterator[List[(String, (SegmentUpdateDetails, ExecutionErrors))]] {
-
-          var result = List[(String, (SegmentUpdateDetails, ExecutionErrors))]()
-          while (records.hasNext) {
-            val ((key), (rowCountDetailsVO, groupedRows)) = records.next
-            result = result ++
-              deleteDeltaFunc(index,
-                key,
-                groupedRows.toIterator,
-                timestamp,
-                rowCountDetailsVO)
-
-          }
-          result
-        }
-    ).collect()
-
-    // if no loads are present then no need to do anything.
-    if (res.isEmpty) {
-      return true
-    }
-
-    // update new status file
-    checkAndUpdateStatusFiles
-
-    // all or none : update status file, only if complete delete opeartion is successfull.
-    def checkAndUpdateStatusFiles: Unit = {
-      val blockUpdateDetailsList = new util.ArrayList[SegmentUpdateDetails]()
-      val segmentDetails = new util.HashSet[String]()
-      res.foreach(resultOfSeg => resultOfSeg.foreach(
-        resultOfBlock => {
-          if (resultOfBlock._1.equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS)) {
-            blockUpdateDetailsList.add(resultOfBlock._2._1)
-            segmentDetails.add(resultOfBlock._2._1.getSegmentName)
-            // if this block is invalid then decrement block count in map.
-            if (CarbonUpdateUtil.isBlockInvalid(resultOfBlock._2._1.getStatus)) {
-              CarbonUpdateUtil.decrementDeletedBlockCount(resultOfBlock._2._1,
-                blockMappingVO.getSegmentNumberOfBlockMapping)
-            }
-          }
-          else {
-            deleteStatus = false
-            // In case of failure , clean all related delete delta files
-            CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timestamp)
-            LOGGER.audit(s"Delete data operation is failed for ${ database }.${ tableName }")
-            val errorMsg =
-              "Delete data operation is failed due to failure in creating delete delta file for " +
-                "segment : " + resultOfBlock._2._1.getSegmentName + " block : " +
-                resultOfBlock._2._1.getBlockName
-            executorErrors.failureCauses = resultOfBlock._2._2.failureCauses
-            executorErrors.errorMsg = resultOfBlock._2._2.errorMsg
-
-            if (executorErrors.failureCauses == FailureCauses.NONE) {
-              executorErrors.failureCauses = FailureCauses.EXECUTOR_FAILURE
-              executorErrors.errorMsg = errorMsg
-            }
-            LOGGER.error(errorMsg)
-            return
-          }
-        }
-      )
-      )
-
-      val listOfSegmentToBeMarkedDeleted = CarbonUpdateUtil
-        .getListOfSegmentsToMarkDeleted(blockMappingVO.getSegmentNumberOfBlockMapping)
-
-
-
-      // this is delete flow so no need of putting timestamp in the status file.
-      if (CarbonUpdateUtil
-        .updateSegmentStatus(blockUpdateDetailsList, carbonTable, timestamp, false) &&
-        CarbonUpdateUtil
-          .updateTableMetadataStatus(segmentDetails,
-            carbonTable,
-            timestamp,
-            !isUpdateOperation,
-            listOfSegmentToBeMarkedDeleted)
-      ) {
-        LOGGER.info(s"Delete data operation is successful for ${ database }.${ tableName }")
-        LOGGER.audit(s"Delete data operation is successful for ${ database }.${ tableName }")
-      }
-      else {
-        // In case of failure , clean all related delete delta files
-        CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timestamp)
-
-        val errorMessage = "Delete data operation is failed due to failure " +
-          "in table status updation."
-        LOGGER.audit(s"Delete data operation is failed for ${ database }.${ tableName }")
-        LOGGER.error("Delete data operation is failed due to failure in table status updation.")
-        executorErrors.failureCauses = FailureCauses.STATUS_FILE_UPDATION_FAILURE
-        executorErrors.errorMsg = errorMessage
-        // throw new Exception(errorMessage)
-      }
-    }
-
-    def deleteDeltaFunc(index: Int,
-                        key: String,
-                        iter: Iterator[Row],
-                        timestamp: String,
-                        rowCountDetailsVO: RowCountDetailsVO):
-    Iterator[(String, (SegmentUpdateDetails, ExecutionErrors))] = {
-
-      val result = new DeleteDelataResultImpl()
-      var deleteStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
-      val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
-      // here key = segment/blockName
-      val blockName = CarbonUpdateUtil
-        .getBlockName(
-          CarbonTablePath.addDataPartPrefix(key.split(CarbonCommonConstants.FILE_SEPARATOR)(1)))
-      val segmentId = key.split(CarbonCommonConstants.FILE_SEPARATOR)(0)
-      var deleteDeltaBlockDetails: DeleteDeltaBlockDetails = new DeleteDeltaBlockDetails(blockName)
-      val resultIter = new Iterator[(String, (SegmentUpdateDetails, ExecutionErrors))] {
-        val segmentUpdateDetails = new SegmentUpdateDetails()
-        var TID = ""
-        var countOfRows = 0
-        try {
-          while (iter.hasNext) {
-            val oneRow = iter.next
-            TID = oneRow
-              .get(oneRow.fieldIndex(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID)).toString
-            val offset = CarbonUpdateUtil.getRequiredFieldFromTID(TID, TupleIdEnum.OFFSET)
-            val blockletId = CarbonUpdateUtil
-              .getRequiredFieldFromTID(TID, TupleIdEnum.BLOCKLET_ID)
-            val pageId = Integer.parseInt(CarbonUpdateUtil
-              .getRequiredFieldFromTID(TID, TupleIdEnum.PAGE_ID))
-            val IsValidOffset = deleteDeltaBlockDetails.addBlocklet(blockletId, offset, pageId)
-            // stop delete operation
-            if(!IsValidOffset) {
-              executorErrors.failureCauses = FailureCauses.MULTIPLE_INPUT_ROWS_MATCHING
-              executorErrors.errorMsg = "Multiple input rows matched for same row."
-              throw new MultipleMatchingException("Multiple input rows matched for same row.")
-            }
-            countOfRows = countOfRows + 1
-          }
-
-          val blockPath = CarbonUpdateUtil.getTableBlockPath(TID, factPath)
-          val completeBlockName = CarbonTablePath
-            .addDataPartPrefix(CarbonUpdateUtil.getRequiredFieldFromTID(TID, TupleIdEnum.BLOCK_ID) +
-              CarbonCommonConstants.FACT_FILE_EXT)
-          val deleteDeletaPath = CarbonUpdateUtil
-            .getDeleteDeltaFilePath(blockPath, blockName, timestamp)
-          val carbonDeleteWriter = new CarbonDeleteDeltaWriterImpl(deleteDeletaPath,
-            FileFactory.getFileType(deleteDeletaPath))
-
-
-
-          segmentUpdateDetails.setBlockName(blockName)
-          segmentUpdateDetails.setActualBlockName(completeBlockName)
-          segmentUpdateDetails.setSegmentName(segmentId)
-          segmentUpdateDetails.setDeleteDeltaEndTimestamp(timestamp)
-          segmentUpdateDetails.setDeleteDeltaStartTimestamp(timestamp)
-
-          val alreadyDeletedRows: Long = rowCountDetailsVO.getDeletedRowsInBlock
-          val totalDeletedRows: Long = alreadyDeletedRows + countOfRows
-          segmentUpdateDetails.setDeletedRowsInBlock(totalDeletedRows.toString)
-          if (totalDeletedRows == rowCountDetailsVO.getTotalNumberOfRows) {
-            segmentUpdateDetails.setStatus(CarbonCommonConstants.MARKED_FOR_DELETE)
-          }
-          else {
-            // write the delta file
-            carbonDeleteWriter.write(deleteDeltaBlockDetails)
-          }
-
-          deleteStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
-        } catch {
-          case e : MultipleMatchingException =>
-            LOGGER.audit(e.getMessage)
-            LOGGER.error(e.getMessage)
-          // dont throw exception here.
-          case e: Exception =>
-            val errorMsg = s"Delete data operation is failed for ${ database }.${ tableName }."
-            LOGGER.audit(errorMsg)
-            LOGGER.error(errorMsg + e.getMessage)
-            throw e
-        }
-
-
-        var finished = false
-
-        override def hasNext: Boolean = {
-          if (!finished) {
-            finished = true
-            finished
-          }
-          else {
-            !finished
-          }
-        }
-
-        override def next(): (String, (SegmentUpdateDetails, ExecutionErrors)) = {
-          finished = true
-          result.getKey(deleteStatus, (segmentUpdateDetails, executorErrors))
-        }
-      }
-      resultIter
-    }
-    true
-  }
-}
-
-
-
-object UpdateExecution {
-
-  def performUpdate(
-         dataFrame: DataFrame,
-         tableIdentifier: Seq[String],
-         plan: LogicalPlan,
-         sqlContext: SQLContext,
-         currentTime: Long,
-         executorErrors: ExecutionErrors): Unit = {
-
-    def isDestinationRelation(relation: CarbonDatasourceRelation): Boolean = {
-
-      val tableName = relation.getTable()
-      val dbName = relation.getDatabaseName()
-      (tableIdentifier.size > 1 &&
-        tableIdentifier(0) == dbName &&
-        tableIdentifier(1) == tableName) ||
-        (tableIdentifier(0) == tableName)
-    }
-    def getHeader(relation: CarbonDatasourceRelation, plan: LogicalPlan): String = {
-      var header = ""
-      var found = false
-
-      plan match {
-        case Project(pList, _) if (!found) =>
-          found = true
-          header = pList
-            .filter(field => !field.name
-              .equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))
-            .map(col => if (col.name.endsWith(CarbonCommonConstants.UPDATED_COL_EXTENSION)) {
-              col.name
-                .substring(0, col.name.lastIndexOf(CarbonCommonConstants.UPDATED_COL_EXTENSION))
-            }
-            else {
-              col.name
-            }).mkString(",")
-      }
-      header
-    }
-    val ex = dataFrame.queryExecution.analyzed
-    val res = ex find {
-      case relation: LogicalRelation if (relation.relation.isInstanceOf[CarbonDatasourceRelation] &&
-        isDestinationRelation(relation.relation
-          .asInstanceOf[CarbonDatasourceRelation])) =>
-        true
-      case _ => false
-    }
-    val carbonRelation: CarbonDatasourceRelation = res match {
-      case Some(relation: LogicalRelation) =>
-        relation.relation.asInstanceOf[CarbonDatasourceRelation]
-      case _ => sys.error("")
-    }
-
-    val updateTableModel = UpdateTableModel(true, currentTime, executorErrors)
-
-    val header = getHeader(carbonRelation, plan)
-
-
-
-    LoadTable(
-      Some(carbonRelation.getDatabaseName()),
-      carbonRelation.getTable(),
-      null,
-      Seq(),
-      Map(("fileheader" -> header)),
-      false,
-      null,
-      Some(dataFrame),
-      Some(updateTableModel)).run(sqlContext)
-
-
-    executorErrors.errorMsg = updateTableModel.executorErrors.errorMsg
-    executorErrors.failureCauses = updateTableModel.executorErrors.failureCauses
-
-    Seq.empty
-
-  }
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
deleted file mode 100644
index 9814cc2..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ /dev/null
@@ -1,1019 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.command
-
-import java.io.File
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ListBuffer
-import scala.language.implicitConversions
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.{RunnableCommand, SparkPlan}
-import org.apache.spark.sql.hive.CarbonMetastore
-import org.apache.spark.sql.types.TimestampType
-import org.apache.spark.util.{CausedBy, FileUtils}
-import org.codehaus.jackson.map.ObjectMapper
-
-import org.apache.carbondata.api.CarbonStore
-import org.apache.carbondata.common.constants.LoggerAction
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.dictionary.server.DictionaryServer
-import org.apache.carbondata.core.exception.InvalidConfigurationException
-import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, ICarbonLock, LockUsage}
-import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.TableInfo
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.exception.DataLoadingException
-import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants
-import org.apache.carbondata.processing.loading.exception.NoRetryException
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.load.ValidateUtil
-import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, DataManagementFunc, DictionaryLoadModel}
-import org.apache.carbondata.spark.util.{CommonUtil, GlobalDictionaryUtil}
-
-object Checker {
-  def validateTableExists(
-      dbName: Option[String],
-      tableName: String,
-      sqlContext: SQLContext): Unit = {
-    val identifier = TableIdentifier(tableName, dbName)
-    if (!CarbonEnv.get.carbonMetastore.tableExists(identifier)(sqlContext)) {
-      val err = s"table $dbName.$tableName not found"
-      LogServiceFactory.getLogService(this.getClass.getName).error(err)
-      throw new IllegalArgumentException(err)
-    }
-  }
-}
-
-/**
- * Command for show table partitions Command
- *
- * @param tableIdentifier
- */
-private[sql] case class ShowCarbonPartitionsCommand(
-    tableIdentifier: TableIdentifier) extends RunnableCommand {
-  val LOGGER = LogServiceFactory.getLogService(ShowCarbonPartitionsCommand.getClass.getName)
-  override val output = CommonUtil.partitionInfoOutput
-  override def run(sqlContext: SQLContext): Seq[Row] = {
-    val relation = CarbonEnv.get.carbonMetastore
-      .lookupRelation1(tableIdentifier)(sqlContext).
-      asInstanceOf[CarbonRelation]
-    val carbonTable = relation.tableMeta.carbonTable
-    var tableName = carbonTable.getFactTableName
-    var partitionInfo = carbonTable.getPartitionInfo(
-      carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getTableName)
-    if (partitionInfo == null) {
-      throw new AnalysisException(
-        s"SHOW PARTITIONS is not allowed on a table that is not partitioned: $tableName")
-    }
-    var partitionType = partitionInfo.getPartitionType
-    var columnName = partitionInfo.getColumnSchemaList.get(0).getColumnName
-    LOGGER.info("partition column name:" + columnName)
-    CommonUtil.getPartitionInfo(columnName, partitionType, partitionInfo)
-  }
-}
-
-/**
- * Command for the compaction in alter table command
- *
- * @param alterTableModel
- */
-private[sql] case class AlterTableCompaction(alterTableModel: AlterTableModel) extends
-  RunnableCommand {
-
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    // TODO : Implement it.
-    val tableName = alterTableModel.tableName
-    val databaseName = getDB.getDatabaseName(alterTableModel.dbName, sqlContext)
-    if (null == CarbonMetadata.getInstance.getCarbonTable(databaseName + "_" + tableName)) {
-      logError(s"alter table failed. table not found: $databaseName.$tableName")
-      sys.error(s"alter table failed. table not found: $databaseName.$tableName")
-    }
-
-    val relation =
-      CarbonEnv.get.carbonMetastore
-        .lookupRelation1(Option(databaseName), tableName)(sqlContext)
-        .asInstanceOf[CarbonRelation]
-    if (relation == null) {
-      sys.error(s"Table $databaseName.$tableName does not exist")
-    }
-    val carbonLoadModel = new CarbonLoadModel()
-
-
-    val table = relation.tableMeta.carbonTable
-    carbonLoadModel.setTableName(table.getFactTableName)
-    val dataLoadSchema = new CarbonDataLoadSchema(table)
-    // Need to fill dimension relation
-    carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
-    carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
-    carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
-    carbonLoadModel.setStorePath(relation.tableMeta.storePath)
-
-    var storeLocation = CarbonProperties.getInstance
-      .getProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
-        System.getProperty("java.io.tmpdir")
-      )
-    storeLocation = storeLocation + "/carbonstore/" + System.nanoTime()
-    try {
-      CarbonDataRDDFactory.alterTableForCompaction(sqlContext,
-          alterTableModel,
-          carbonLoadModel,
-          relation.tableMeta.storePath,
-          storeLocation
-        )
-    } catch {
-      case e: Exception =>
-        if (null != e.getMessage) {
-          sys.error(s"Compaction failed. Please check logs for more info. ${ e.getMessage }")
-        } else {
-          sys.error("Exception in compaction. Please check logs for more info.")
-        }
-    }
-    Seq.empty
-  }
-}
-
-case class CreateTable(cm: TableModel) extends RunnableCommand {
-
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-    cm.databaseName = getDB.getDatabaseName(cm.databaseNameOp, sqlContext)
-    val tbName = cm.tableName
-    val dbName = cm.databaseName
-    LOGGER.audit(s"Creating Table with Database name [$dbName] and Table name [$tbName]")
-
-    val tableInfo: TableInfo = TableNewProcessor(cm)
-
-    // Add validation for sort scope when create table
-    val sortScope = tableInfo.getFactTable.getTableProperties
-      .getOrDefault("sort_scope", CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)
-    if (!CarbonUtil.isValidSortOption(sortScope)) {
-      throw new InvalidConfigurationException(s"Passing invalid SORT_SCOPE '$sortScope'," +
-        s" valid SORT_SCOPE are 'NO_SORT', 'BATCH_SORT', 'LOCAL_SORT' and 'GLOBAL_SORT' ")
-    }
-
-    if (tableInfo.getFactTable.getListOfColumns.isEmpty) {
-      sys.error("No Dimensions found. Table should have at least one dimesnion !")
-    }
-
-    if (sqlContext.tableNames(dbName).exists(_.equalsIgnoreCase(tbName))) {
-      if (!cm.ifNotExistsSet) {
-        LOGGER.audit(
-          s"Table creation with Database name [$dbName] and Table name [$tbName] failed. " +
-          s"Table [$tbName] already exists under database [$dbName]")
-        sys.error(s"Table [$tbName] already exists under database [$dbName]")
-      }
-    } else {
-      // Add Database to catalog and persist
-      val catalog = CarbonEnv.get.carbonMetastore
-      // Need to fill partitioner class when we support partition
-      val tablePath = catalog.createTableFromThrift(tableInfo, dbName, tbName, null)(sqlContext)
-      try {
-        sqlContext.sql(
-          s"""CREATE TABLE $dbName.$tbName USING carbondata""" +
-          s""" OPTIONS (tableName "$dbName.$tbName", tablePath "$tablePath") """)
-          .collect
-      } catch {
-        case e: Exception =>
-          val identifier: TableIdentifier = TableIdentifier(tbName, Some(dbName))
-          // call the drop table to delete the created table.
-
-          CarbonEnv.get.carbonMetastore
-            .dropTable(catalog.storePath, identifier)(sqlContext)
-
-          LOGGER.audit(s"Table creation with Database name [$dbName] " +
-                       s"and Table name [$tbName] failed")
-          throw e
-      }
-
-      LOGGER.audit(s"Table created with Database name [$dbName] and Table name [$tbName]")
-    }
-
-    Seq.empty
-  }
-
-  def setV(ref: Any, name: String, value: Any): Unit = {
-    ref.getClass.getFields.find(_.getName == name).get
-      .set(ref, value.asInstanceOf[AnyRef])
-  }
-}
-
-private[sql] case class DeleteLoadsById(
-    loadids: Seq[String],
-    databaseNameOp: Option[String],
-    tableName: String) extends RunnableCommand {
-
-  val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    Checker.validateTableExists(databaseNameOp, tableName, sqlContext)
-    val carbonTable = CarbonEnv.get.carbonMetastore.lookupRelation1(databaseNameOp,
-      tableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
-    CarbonStore.deleteLoadById(
-      loadids,
-      getDB.getDatabaseName(databaseNameOp, sqlContext),
-      tableName,
-      carbonTable
-    )
-    Seq.empty
-
-  }
-
-  // validates load ids
-  private def validateLoadIds: Unit = {
-    if (loadids.isEmpty) {
-      val errorMessage = "Error: Segment id(s) should not be empty."
-      throw new MalformedCarbonCommandException(errorMessage)
-
-    }
-  }
-}
-
-private[sql] case class DeleteLoadsByLoadDate(
-    databaseNameOp: Option[String],
-    tableName: String,
-    dateField: String,
-    loadDate: String) extends RunnableCommand {
-
-  val LOGGER = LogServiceFactory.getLogService("org.apache.spark.sql.tablemodel.tableSchema")
-
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    Checker.validateTableExists(databaseNameOp, tableName, sqlContext)
-    val carbonTable = CarbonEnv.get.carbonMetastore.lookupRelation1(databaseNameOp,
-      tableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
-    CarbonStore.deleteLoadByDate(
-      loadDate,
-      getDB.getDatabaseName(databaseNameOp, sqlContext),
-      tableName,
-      carbonTable
-    )
-    Seq.empty
-
-  }
-
-}
-
-object LoadTable {
-
-  def updateTableMetadata(carbonLoadModel: CarbonLoadModel,
-      sqlContext: SQLContext,
-      model: DictionaryLoadModel,
-      noDictDimension: Array[CarbonDimension]): Unit = {
-
-    val carbonTablePath = CarbonStorePath.getCarbonTablePath(model.hdfsLocation,
-      model.table)
-    val schemaFilePath = carbonTablePath.getSchemaFilePath
-
-    // read TableInfo
-    val tableInfo = CarbonMetastore.readSchemaFileToThriftTable(schemaFilePath)
-
-    // modify TableInfo
-    val columns = tableInfo.getFact_table.getTable_columns
-    for (i <- 0 until columns.size) {
-      if (noDictDimension.exists(x => columns.get(i).getColumn_id.equals(x.getColumnId))) {
-        columns.get(i).encoders.remove(org.apache.carbondata.format.Encoding.DICTIONARY)
-      }
-    }
-
-    // write TableInfo
-    CarbonMetastore.writeThriftTableToSchemaFile(schemaFilePath, tableInfo)
-
-    // update Metadata
-    val catalog = CarbonEnv.get.carbonMetastore
-    catalog.updateMetadataByThriftTable(schemaFilePath, tableInfo,
-      model.table.getDatabaseName, model.table.getTableName, carbonLoadModel.getStorePath)
-
-    // update CarbonDataLoadSchema
-    val carbonTable = catalog.lookupRelation1(Option(model.table.getDatabaseName),
-      model.table.getTableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
-    carbonLoadModel.setCarbonDataLoadSchema(new CarbonDataLoadSchema(carbonTable))
-  }
-
-}
-
-private[sql] case class LoadTableByInsert(relation: CarbonDatasourceRelation,
-    child: LogicalPlan, isOverwriteExist: Boolean) extends RunnableCommand {
-  val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    val df = new DataFrame(sqlContext, child)
-    val header = relation.carbonRelation.output.map(_.name).mkString(",")
-    val load = LoadTable(
-      Some(relation.carbonRelation.databaseName),
-      relation.carbonRelation.tableName,
-      null,
-      Seq(),
-      scala.collection.immutable.Map("fileheader" -> header),
-      isOverwriteExist,
-      null,
-      Some(df)).run(sqlContext)
-    // updating relation metadata. This is in case of auto detect high cardinality
-    relation.carbonRelation.metaData =
-      CarbonSparkUtil.createSparkMeta(relation.carbonRelation.tableMeta.carbonTable)
-    load
-  }
-}
-case class LoadTable(
-    databaseNameOp: Option[String],
-    tableName: String,
-    factPathFromUser: String,
-    dimFilesPath: Seq[DataLoadTableFileMapping],
-    options: scala.collection.immutable.Map[String, String],
-    isOverwriteExist: Boolean,
-    var inputSqlString: String = null,
-    dataFrame: Option[DataFrame] = None,
-    updateModel: Option[UpdateTableModel] = None) extends RunnableCommand {
-
-  val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
-  private def checkDefaultValue(value: String, default: String) = if (StringUtils.isEmpty(value)) {
-    default
-  } else {
-    value
-  }
-
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    if (dataFrame.isDefined && !updateModel.isDefined) {
-      val rdd = dataFrame.get.rdd
-      if (rdd.partitions == null || rdd.partitions.length == 0) {
-        LOGGER.warn("DataLoading finished. No data was loaded.")
-        return Seq.empty
-      }
-    }
-
-    val dbName = getDB.getDatabaseName(databaseNameOp, sqlContext)
-    if (null == CarbonMetadata.getInstance.getCarbonTable(dbName + "_" + tableName)) {
-      logError(s"Data loading failed. table not found: $dbName.$tableName")
-      LOGGER.audit(s"Data loading failed. table not found: $dbName.$tableName")
-      sys.error(s"Data loading failed. table not found: $dbName.$tableName")
-    }
-
-    val relation = CarbonEnv.get.carbonMetastore
-      .lookupRelation1(Option(dbName), tableName)(sqlContext)
-      .asInstanceOf[CarbonRelation]
-    if (relation == null) {
-      sys.error(s"Table $dbName.$tableName does not exist")
-    }
-    CarbonProperties.getInstance().addProperty("zookeeper.enable.lock", "false")
-    val carbonLock = CarbonLockFactory
-      .getCarbonLockObj(relation.tableMeta.carbonTable.getAbsoluteTableIdentifier
-        .getCarbonTableIdentifier,
-        LockUsage.METADATA_LOCK
-      )
-    try {
-      // take lock only in case of normal data load.
-      if (!updateModel.isDefined) {
-        if (carbonLock.lockWithRetries()) {
-          logInfo("Successfully able to get the table metadata file lock")
-        } else {
-          sys.error("Table is locked for updation. Please try after some time")
-        }
-      }
-
-      val factPath = if (dataFrame.isDefined) {
-        ""
-      } else {
-        FileUtils.getPaths(
-          CarbonUtil.checkAndAppendHDFSUrl(factPathFromUser))
-      }
-      val carbonLoadModel = new CarbonLoadModel()
-      carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
-      carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
-      carbonLoadModel.setStorePath(relation.tableMeta.storePath)
-
-      val table = relation.tableMeta.carbonTable
-      carbonLoadModel.setTableName(table.getFactTableName)
-      val dataLoadSchema = new CarbonDataLoadSchema(table)
-      // Need to fill dimension relation
-      carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
-
-      val partitionLocation = relation.tableMeta.storePath + "/partition/" +
-                              relation.tableMeta.carbonTableIdentifier.getDatabaseName + "/" +
-                              relation.tableMeta.carbonTableIdentifier.getTableName + "/"
-
-
-      val columnar = sqlContext.getConf("carbon.is.columnar.storage", "true").toBoolean
-
-      val delimiter = options.getOrElse("delimiter", ",")
-      val quoteChar = options.getOrElse("quotechar", "\"")
-      var fileHeader = options.getOrElse("fileheader", "")
-      val escapeChar = options.getOrElse("escapechar", "\\")
-      val commentchar = options.getOrElse("commentchar", "#")
-      val columnDict = options.getOrElse("columndict", null)
-      val serializationNullFormat = options.getOrElse("serialization_null_format", "\\N")
-      val badRecordsLoggerEnable = options.getOrElse("bad_records_logger_enable", "false")
-      val badRecordActionValue = CarbonProperties.getInstance()
-        .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
-          CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT)
-      val badRecordsAction = options.getOrElse("bad_records_action", badRecordActionValue)
-      val isEmptyDataBadRecord = options.getOrElse("is_empty_data_bad_record", "false")
-      val allDictionaryPath = options.getOrElse("all_dictionary_path", "")
-      val complex_delimiter_level_1 = options.getOrElse("complex_delimiter_level_1", "\\$")
-      val complex_delimiter_level_2 = options.getOrElse("complex_delimiter_level_2", "\\:")
-      val dateFormat = options.getOrElse("dateformat", null)
-      ValidateUtil.validateDateFormat(dateFormat, table, tableName)
-      val maxColumns = options.getOrElse("maxcolumns", null)
-      val tableProperties = table.getTableInfo.getFactTable.getTableProperties
-      val sortScopeDefault = CarbonProperties.getInstance().
-        getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
-          CarbonProperties.getInstance().getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
-            CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
-      val sortScope = if (null == tableProperties) {
-        sortScopeDefault
-      } else {
-        tableProperties.getOrDefault("sort_scope", sortScopeDefault)
-      }
-
-      ValidateUtil.validateSortScope(table, sortScope)
-      val carbonProperty: CarbonProperties = CarbonProperties.getInstance()
-      val batchSortSizeInMB = options.getOrElse("batch_sort_size_inmb", carbonProperty
-        .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
-        carbonProperty.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
-          CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT)))
-      val globalSortPartitions = options.getOrElse("global_sort_partitions", carbonProperty
-        .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS, null))
-      ValidateUtil.validateGlobalSortPartitions(globalSortPartitions)
-
-      // if there isn't file header in csv file and load sql doesn't provide FILEHEADER option,
-      // we should use table schema to generate file header.
-      val headerOption = options.get("header")
-      if (headerOption.isDefined) {
-        // whether the csv file has file header
-        // the default value is true
-        val header = try {
-          headerOption.get.toBoolean
-        } catch {
-          case ex: IllegalArgumentException =>
-            throw new MalformedCarbonCommandException(
-              "'header' option should be either 'true' or 'false'. " + ex.getMessage)
-        }
-        header match {
-          case true =>
-            if (fileHeader.nonEmpty) {
-              throw new MalformedCarbonCommandException(
-                "When 'header' option is true, 'fileheader' option is not required.")
-            }
-          case false =>
-            // generate file header
-            if (fileHeader.isEmpty) {
-              fileHeader = table.getCreateOrderColumn(table.getFactTableName)
-                .asScala.map(_.getColName).mkString(",")
-            }
-        }
-      }
-
-      val bad_record_path = options.getOrElse("bad_record_path",
-          CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
-            CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
-      if (badRecordsLoggerEnable.toBoolean ||
-          LoggerAction.REDIRECT.name().equalsIgnoreCase(badRecordsAction)) {
-        if (!CarbonUtil.isValidBadStorePath(bad_record_path)) {
-          sys.error("Invalid bad records location.")
-        }
-      }
-      carbonLoadModel.setBadRecordsLocation(bad_record_path)
-      carbonLoadModel.setEscapeChar(checkDefaultValue(escapeChar, "\\"))
-      carbonLoadModel.setQuoteChar(checkDefaultValue(quoteChar, "\""))
-      carbonLoadModel.setCommentChar(checkDefaultValue(commentchar, "#"))
-      carbonLoadModel.setDateFormat(dateFormat)
-      carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
-        CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
-      carbonLoadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(
-        CarbonCommonConstants.CARBON_DATE_FORMAT,
-        CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
-      carbonLoadModel
-        .setSerializationNullFormat(
-          TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + "," + serializationNullFormat)
-      carbonLoadModel
-        .setBadRecordsLoggerEnable(
-          TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName + "," + badRecordsLoggerEnable)
-      carbonLoadModel
-        .setBadRecordsAction(
-          TableOptionConstant.BAD_RECORDS_ACTION.getName + "," + badRecordsAction)
-      carbonLoadModel
-        .setIsEmptyDataBadRecord(
-          DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," + isEmptyDataBadRecord)
-      carbonLoadModel.setSortScope(sortScope)
-      carbonLoadModel.setBatchSortSizeInMb(batchSortSizeInMB)
-      carbonLoadModel.setGlobalSortPartitions(globalSortPartitions)
-      // when single_pass=true, and not use all dict
-      val useOnePass = options.getOrElse("single_pass", "false").trim.toLowerCase match {
-        case "true" =>
-          true
-        case "false" =>
-          // when single_pass = false  and if either alldictionary
-          // or columnDict is configured the do not allow load
-          if (StringUtils.isNotEmpty(allDictionaryPath) || StringUtils.isNotEmpty(columnDict)) {
-            throw new MalformedCarbonCommandException(
-              "Can not use all_dictionary_path or columndict without single_pass.")
-          } else {
-            false
-          }
-        case illegal =>
-          LOGGER.error(s"Can't use single_pass, because illegal syntax found: [" + illegal + "] " +
-                       "Please set it as 'true' or 'false'")
-          false
-      }
-      carbonLoadModel.setUseOnePass(useOnePass)
-
-      if (delimiter.equalsIgnoreCase(complex_delimiter_level_1) ||
-          complex_delimiter_level_1.equalsIgnoreCase(complex_delimiter_level_2) ||
-          delimiter.equalsIgnoreCase(complex_delimiter_level_2)) {
-        sys.error(s"Field Delimiter & Complex types delimiter are same")
-      }
-      else {
-        carbonLoadModel.setComplexDelimiterLevel1(
-          CarbonUtil.delimiterConverter(complex_delimiter_level_1))
-        carbonLoadModel.setComplexDelimiterLevel2(
-          CarbonUtil.delimiterConverter(complex_delimiter_level_2))
-      }
-      // set local dictionary path, and dictionary file extension
-      carbonLoadModel.setAllDictPath(allDictionaryPath)
-
-      val partitionStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
-
-      try {
-        // First system has to partition the data first and then call the load data
-        LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
-        carbonLoadModel.setFactFilePath(factPath)
-        carbonLoadModel.setCsvDelimiter(CarbonUtil.unescapeChar(delimiter))
-        carbonLoadModel.setCsvHeader(fileHeader)
-        carbonLoadModel.setColDictFilePath(columnDict)
-        carbonLoadModel.setDirectLoad(true)
-        carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
-        val validatedMaxColumns = CommonUtil.validateMaxColumns(carbonLoadModel.getCsvHeaderColumns,
-          maxColumns)
-        carbonLoadModel.setMaxColumns(validatedMaxColumns.toString)
-        GlobalDictionaryUtil.updateTableMetadataFunc = updateTableMetadata
-        val storePath = relation.tableMeta.storePath
-        val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
-        val carbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier
-          .getCarbonTableIdentifier
-        val carbonTablePath = CarbonStorePath
-          .getCarbonTablePath(storePath, carbonTableIdentifier)
-        val dictFolderPath = carbonTablePath.getMetadataDirectoryPath
-        val dimensions = carbonTable.getDimensionByTableName(
-          carbonTable.getFactTableName).asScala.toArray
-        // add the start entry for the new load in the table status file
-        if (!updateModel.isDefined) {
-          CommonUtil.
-            readAndUpdateLoadProgressInTableMeta(carbonLoadModel, storePath, isOverwriteExist)
-        }
-        if (isOverwriteExist) {
-          LOGGER.info(s"Overwrite is in progress for carbon table with $dbName.$tableName")
-        }
-        if (null == carbonLoadModel.getLoadMetadataDetails) {
-          CommonUtil.readLoadMetadataDetails(carbonLoadModel)
-        }
-        if (carbonLoadModel.getLoadMetadataDetails.isEmpty && carbonLoadModel.getUseOnePass &&
-            StringUtils.isEmpty(columnDict) && StringUtils.isEmpty(allDictionaryPath)) {
-          LOGGER.info(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
-          LOGGER.audit(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
-          carbonLoadModel.setUseOnePass(false)
-        }
-        if (carbonLoadModel.getUseOnePass) {
-          val colDictFilePath = carbonLoadModel.getColDictFilePath
-          if (!StringUtils.isEmpty(colDictFilePath)) {
-            carbonLoadModel.initPredefDictMap()
-            // generate predefined dictionary
-            GlobalDictionaryUtil
-              .generatePredefinedColDictionary(colDictFilePath, carbonTableIdentifier,
-                dimensions, carbonLoadModel, sqlContext, storePath, dictFolderPath)
-          }
-          val allDictPath: String = carbonLoadModel.getAllDictPath
-          if(!StringUtils.isEmpty(allDictPath)) {
-            carbonLoadModel.initPredefDictMap()
-            GlobalDictionaryUtil
-              .generateDictionaryFromDictionaryFiles(sqlContext,
-                carbonLoadModel,
-                storePath,
-                carbonTableIdentifier,
-                dictFolderPath,
-                dimensions,
-                allDictionaryPath)
-          }
-          // dictionaryServerClient dictionary generator
-          val dictionaryServerPort = CarbonProperties.getInstance()
-            .getProperty(CarbonCommonConstants.DICTIONARY_SERVER_PORT,
-              CarbonCommonConstants.DICTIONARY_SERVER_PORT_DEFAULT)
-          val sparkDriverHost = sqlContext.sparkContext.getConf.get("spark.driver.host")
-          carbonLoadModel.setDictionaryServerHost(sparkDriverHost)
-          // start dictionary server when use one pass load and dimension with DICTIONARY
-          // encoding is present.
-          val allDimensions = table.getAllDimensions.asScala.toList
-          val createDictionary = allDimensions.exists {
-            carbonDimension => carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
-              !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY)
-          }
-          val server: Option[DictionaryServer] = if (createDictionary) {
-            val dictionaryServer = DictionaryServer
-              .getInstance(dictionaryServerPort.toInt, carbonTable)
-            carbonLoadModel.setDictionaryServerPort(dictionaryServer.getPort)
-            sqlContext.sparkContext.addSparkListener(new SparkListener() {
-              override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd) {
-                dictionaryServer.shutdown()
-              }
-            })
-            Some(dictionaryServer)
-          } else {
-            None
-          }
-          CarbonDataRDDFactory.loadCarbonData(sqlContext,
-            carbonLoadModel,
-            relation.tableMeta.storePath,
-            columnar,
-            partitionStatus,
-            server,
-            isOverwriteExist,
-            dataFrame,
-            updateModel)
-        } else {
-          val (dictionaryDataFrame, loadDataFrame) = if (updateModel.isDefined) {
-            val fields = dataFrame.get.schema.fields
-            import org.apache.spark.sql.functions.udf
-            // extracting only segment from tupleId
-            val getSegIdUDF = udf((tupleId: String) =>
-              CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID))
-            // getting all fields except tupleId field as it is not required in the value
-            var otherFields = fields.toSeq
-              .filter(field => !field.name
-                .equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))
-              .map(field => {
-                if (field.name.endsWith(CarbonCommonConstants.UPDATED_COL_EXTENSION) && false) {
-                  new Column(field.name
-                    .substring(0,
-                      field.name.lastIndexOf(CarbonCommonConstants.UPDATED_COL_EXTENSION)))
-                } else {
-
-                  new Column(field.name)
-                }
-              })
-
-            // extract tupleId field which will be used as a key
-            val segIdColumn = getSegIdUDF(new Column(UnresolvedAttribute
-              .quotedString(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))).
-              as(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_SEGMENTID)
-            // use dataFrameWithoutTupleId as dictionaryDataFrame
-            val dataFrameWithoutTupleId = dataFrame.get.select(otherFields: _*)
-            otherFields = otherFields :+ segIdColumn
-            // use dataFrameWithTupleId as loadDataFrame
-            val dataFrameWithTupleId = dataFrame.get.select(otherFields: _*)
-            (Some(dataFrameWithoutTupleId), Some(dataFrameWithTupleId))
-          } else {
-            (dataFrame, dataFrame)
-          }
-          GlobalDictionaryUtil
-            .generateGlobalDictionary(sqlContext, carbonLoadModel, relation.tableMeta.storePath,
-              dictionaryDataFrame)
-          CarbonDataRDDFactory.loadCarbonData(sqlContext,
-            carbonLoadModel,
-            relation.tableMeta.storePath,
-            columnar,
-            partitionStatus,
-            None,
-            isOverwriteExist,
-            loadDataFrame,
-            updateModel)
-        }
-      } catch {
-        case CausedBy(ex: NoRetryException) =>
-          LOGGER.error(ex, s"Dataload failure for $dbName.$tableName")
-          throw new RuntimeException(s"Dataload failure for $dbName.$tableName, ${ex.getMessage}")
-        case ex: Exception =>
-          LOGGER.error(ex)
-          LOGGER.audit(s"Dataload failure for $dbName.$tableName. Please check the logs")
-          throw ex
-      } finally {
-        // Once the data load is successful delete the unwanted partition files
-        try {
-          val fileType = FileFactory.getFileType(partitionLocation)
-          if (FileFactory.isFileExist(partitionLocation, fileType)) {
-            val file = FileFactory
-              .getCarbonFile(partitionLocation, fileType)
-            CarbonUtil.deleteFoldersAndFiles(file)
-          }
-        } catch {
-          case ex: Exception =>
-            LOGGER.error(ex)
-            LOGGER.audit(s"Dataload failure for $dbName.$tableName. " +
-                         "Problem deleting the partition folder")
-            throw ex
-        }
-
-      }
-    } catch {
-      case dle: DataLoadingException =>
-        LOGGER.audit(s"Dataload failed for $dbName.$tableName. " + dle.getMessage)
-        throw dle
-      case mce: MalformedCarbonCommandException =>
-        LOGGER.audit(s"Dataload failed for $dbName.$tableName. " + mce.getMessage)
-        throw mce
-    } finally {
-      if (carbonLock != null) {
-        if (carbonLock.unlock()) {
-          logInfo("Table MetaData Unlocked Successfully after data load")
-        } else {
-          logError("Unable to unlock Table MetaData")
-        }
-      }
-    }
-    Seq.empty
-  }
-
-  private def updateTableMetadata(carbonLoadModel: CarbonLoadModel,
-      sqlContext: SQLContext,
-      model: DictionaryLoadModel,
-      noDictDimension: Array[CarbonDimension]): Unit = {
-
-    val carbonTablePath = CarbonStorePath.getCarbonTablePath(model.hdfsLocation,
-      model.table)
-    val schemaFilePath = carbonTablePath.getSchemaFilePath
-
-    // read TableInfo
-    val tableInfo = CarbonMetastore.readSchemaFileToThriftTable(schemaFilePath)
-
-    // modify TableInfo
-    val columns = tableInfo.getFact_table.getTable_columns
-    for (i <- 0 until columns.size) {
-      if (noDictDimension.exists(x => columns.get(i).getColumn_id.equals(x.getColumnId))) {
-        columns.get(i).encoders.remove(org.apache.carbondata.format.Encoding.DICTIONARY)
-      }
-    }
-
-    // write TableInfo
-    CarbonMetastore.writeThriftTableToSchemaFile(schemaFilePath, tableInfo)
-
-
-    val catalog = CarbonEnv.get.carbonMetastore
-
-    // upate the schema modified time
-    catalog.updateSchemasUpdatedTime(catalog.touchSchemaFileSystemTime(
-      carbonLoadModel.getDatabaseName,
-      carbonLoadModel.getTableName))
-
-    // update Metadata
-    catalog.updateMetadataByThriftTable(schemaFilePath, tableInfo,
-      model.table.getDatabaseName, model.table.getTableName, carbonLoadModel.getStorePath)
-
-    // update CarbonDataLoadSchema
-    val carbonTable = catalog.lookupRelation1(Option(model.table.getDatabaseName),
-      model.table.getTableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
-    carbonLoadModel.setCarbonDataLoadSchema(new CarbonDataLoadSchema(carbonTable))
-  }
-
-}
-
-private[sql] case class DropTableCommand(ifExistsSet: Boolean, databaseNameOp: Option[String],
-    tableName: String)
-  extends RunnableCommand {
-
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-    val dbName = getDB.getDatabaseName(databaseNameOp, sqlContext)
-    val identifier = TableIdentifier(tableName, Option(dbName))
-    val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
-    val locksToBeAcquired = List(LockUsage.METADATA_LOCK, LockUsage.DROP_TABLE_LOCK)
-    val carbonLocks: scala.collection.mutable.ListBuffer[ICarbonLock] = ListBuffer()
-    val catalog = CarbonEnv.get.carbonMetastore
-    val storePath = catalog.storePath
-    try {
-      locksToBeAcquired foreach {
-        lock => carbonLocks += CarbonLockUtil.getLockObject(carbonTableIdentifier, lock)
-      }
-      LOGGER.audit(s"Deleting table [$tableName] under database [$dbName]")
-      CarbonEnv.get.carbonMetastore.dropTable(storePath, identifier)(sqlContext)
-      LOGGER.audit(s"Deleted table [$tableName] under database [$dbName]")
-    } catch {
-      case ex: Exception =>
-        LOGGER.error(ex, s"Dropping table $dbName.$tableName failed")
-        sys.error(s"Dropping table $dbName.$tableName failed: ${ex.getMessage}")
-    } finally {
-      if (carbonLocks.nonEmpty) {
-        val unlocked = carbonLocks.forall(_.unlock())
-        if (unlocked) {
-          logInfo("Table MetaData Unlocked Successfully")
-          // deleting any remaining files.
-          val metadataFilePath = CarbonStorePath
-            .getCarbonTablePath(storePath, carbonTableIdentifier).getMetadataDirectoryPath
-          val fileType = FileFactory.getFileType(metadataFilePath)
-          if (FileFactory.isFileExist(metadataFilePath, fileType)) {
-            val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
-            CarbonUtil.deleteFoldersAndFiles(file.getParentFile)
-          }
-        }
-      }
-    }
-    Seq.empty
-  }
-}
-
-private[sql] case class ShowLoads(
-    databaseNameOp: Option[String],
-    tableName: String,
-    limit: Option[String],
-    override val output: Seq[Attribute]) extends RunnableCommand {
-
-  override def run(sqlContext: SQLContext): Seq[Row] = {
-    Checker.validateTableExists(databaseNameOp, tableName, sqlContext)
-    val carbonTable = CarbonEnv.get.carbonMetastore.lookupRelation1(databaseNameOp,
-      tableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
-    CarbonStore.showSegments(
-      getDB.getDatabaseName(databaseNameOp, sqlContext),
-      tableName,
-      limit,
-      carbonTable.getMetaDataFilepath
-    )
-  }
-}
-
-private[sql] case class DescribeCommandFormatted(
-    child: SparkPlan,
-    override val output: Seq[Attribute],
-    tblIdentifier: TableIdentifier)
-  extends RunnableCommand {
-
-  override def run(sqlContext: SQLContext): Seq[Row] = {
-    val relation = CarbonEnv.get.carbonMetastore
-      .lookupRelation1(tblIdentifier)(sqlContext).asInstanceOf[CarbonRelation]
-    val mapper = new ObjectMapper()
-    val colProps = StringBuilder.newBuilder
-    var results: Seq[(String, String, String)] = child.schema.fields.map { field =>
-      val comment = if (relation.metaData.dims.contains(field.name)) {
-        val dimension = relation.metaData.carbonTable.getDimensionByName(
-          relation.tableMeta.carbonTableIdentifier.getTableName,
-          field.name)
-        if (null != dimension.getColumnProperties && dimension.getColumnProperties.size() > 0) {
-          colProps.append(field.name).append(".")
-            .append(mapper.writeValueAsString(dimension.getColumnProperties))
-            .append(",")
-        }
-        if (dimension.hasEncoding(Encoding.DICTIONARY)) {
-          "DICTIONARY, KEY COLUMN" + (dimension.hasEncoding(Encoding.INVERTED_INDEX) match {
-                      case false => ",NOINVERTEDINDEX"
-                      case _ => ""
-                    })
-        } else {
-          "KEY COLUMN" + (dimension.hasEncoding(Encoding.INVERTED_INDEX) match {
-            case false => ",NOINVERTEDINDEX"
-            case _ => ""
-          })
-        }
-      } else {
-        "MEASURE"
-      }
-      (field.name, field.dataType.simpleString, comment)
-    }
-    val colPropStr = if (colProps.toString().trim().length() > 0) {
-      // drops additional comma at endpom
-      colProps.toString().dropRight(1)
-    } else {
-      colProps.toString()
-    }
-    results ++= Seq(("", "", ""), ("##Detailed Table Information", "", ""))
-    results ++= Seq(("Database Name: ", relation.tableMeta.carbonTableIdentifier
-      .getDatabaseName, "")
-    )
-    results ++= Seq(("Table Name: ", relation.tableMeta.carbonTableIdentifier.getTableName, ""))
-    results ++= Seq(("CARBON Store Path: ", relation.tableMeta.storePath, ""))
-    val carbonTable = relation.tableMeta.carbonTable
-    results ++= Seq(("Table Block Size : ", carbonTable.getBlockSizeInMB + " MB", ""))
-    results ++= Seq(("SORT_SCOPE", carbonTable.getTableInfo.getFactTable
-      .getTableProperties.getOrDefault("sort_scope", CarbonCommonConstants
-      .LOAD_SORT_SCOPE_DEFAULT), CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
-    results ++= Seq(("", "", ""), ("##Detailed Column property", "", ""))
-    if (colPropStr.length() > 0) {
-      results ++= Seq((colPropStr, "", ""))
-    } else {
-      results ++= Seq(("ADAPTIVE", "", ""))
-    }
-    results ++= Seq(("SORT_COLUMNS", relation.metaData.carbonTable.getSortColumns(
-      relation.tableMeta.carbonTableIdentifier.getTableName).asScala
-      .map(column => column).mkString(","), ""))
-    val dimension = carbonTable
-      .getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
-    results ++= getColumnGroups(dimension.asScala.toList)
-    if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null) {
-      results ++=
-      Seq(("Partition Columns: ", carbonTable.getPartitionInfo(carbonTable.getFactTableName)
-        .getColumnSchemaList.asScala.map(_.getColumnName).mkString(","), ""))
-    }
-    results.map { case (name, dataType, comment) =>
-      Row(f"$name%-36s $dataType%-80s $comment%-72s")
-    }
-  }
-
-  private def getColumnGroups(dimensions: List[CarbonDimension]): Seq[(String, String, String)] = {
-    var results: Seq[(String, String, String)] =
-      Seq(("", "", ""), ("##Column Group Information", "", ""))
-    val groupedDimensions = dimensions.groupBy(x => x.columnGroupId()).filter {
-      case (groupId, _) => groupId != -1
-    }.toSeq.sortBy(_._1)
-    val groups = groupedDimensions.map(colGroups => {
-      colGroups._2.map(dim => dim.getColName).mkString(", ")
-    })
-    var index = 1
-    groups.foreach { x =>
-      results = results :+ (s"Column Group $index", x, "")
-      index = index + 1
-    }
-    results
-  }
-}
-
-private[sql] case class DeleteLoadByDate(
-    databaseNameOp: Option[String],
-    tableName: String,
-    dateField: String,
-    dateValue: String
-) extends RunnableCommand {
-
-  val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    val dbName = getDB.getDatabaseName(databaseNameOp, sqlContext)
-    LOGGER.audit(s"The delete load by date request has been received for $dbName.$tableName")
-    val identifier = TableIdentifier(tableName, Option(dbName))
-    val relation = CarbonEnv.get.carbonMetastore
-      .lookupRelation1(identifier)(sqlContext).asInstanceOf[CarbonRelation]
-    var level: String = ""
-    val carbonTable = CarbonMetadata.getInstance().getCarbonTable(dbName + '_' + tableName)
-    if (relation == null) {
-      LOGGER.audit(s"The delete load by date is failed. Table $dbName.$tableName does not exist")
-      sys.error(s"Table $dbName.$tableName does not exist")
-    }
-    val matches: Seq[AttributeReference] = relation.dimensionsAttr.filter(
-      filter => filter.name.equalsIgnoreCase(dateField) &&
-                filter.dataType.isInstanceOf[TimestampType]).toList
-    if (matches.isEmpty) {
-      LOGGER.audit("The delete load by date is failed. " +
-                   s"Table $dbName.$tableName does not contain date field: $dateField")
-      sys.error(s"Table $dbName.$tableName does not contain date field $dateField")
-    } else {
-      level = matches.asJava.get(0).name
-    }
-    val actualColName = relation.metaData.carbonTable.getDimensionByName(tableName, level)
-      .getColName
-    DataManagementFunc.deleteLoadByDate(
-      sqlContext,
-      new CarbonDataLoadSchema(carbonTable),
-      dbName,
-      tableName,
-      CarbonEnv.get.carbonMetastore.storePath,
-      level,
-      actualColName,
-      dateValue)
-    LOGGER.audit(s"The delete load by date $dateValue is successful for $dbName.$tableName.")
-    Seq.empty
-  }
-
-}
-
-private[sql] case class CleanFiles(
-    databaseNameOp: Option[String],
-    tableName: String) extends RunnableCommand {
-
-  val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    Checker.validateTableExists(databaseNameOp, tableName, sqlContext)
-    val carbonTable = CarbonEnv.get.carbonMetastore.lookupRelation1(databaseNameOp,
-      tableName)(sqlContext).asInstanceOf[CarbonRelation].tableMeta.carbonTable
-    CarbonStore.cleanFiles(
-      getDB.getDatabaseName(databaseNameOp, sqlContext),
-      tableName,
-      sqlContext.asInstanceOf[CarbonContext].storePath,
-      carbonTable,
-      false)
-    Seq.empty
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
deleted file mode 100644
index d23b18f..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit._
-import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedFunction, UnresolvedRelation, UnresolvedStar}
-import org.apache.spark.sql.catalyst.expressions.Alias
-import org.apache.spark.sql.catalyst.plans.Inner
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.execution.command.ProjectForDeleteCommand
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-
-/**
- * Insert into carbon table from other source
- */
-object CarbonPreInsertionCasts extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan.transform {
-    // Wait until children are resolved.
-    case p: LogicalPlan if !p.childrenResolved => p
-
-    case p @ InsertIntoTable(relation: LogicalRelation, _, child, _, _)
-      if relation.relation.isInstanceOf[CarbonDatasourceRelation] =>
-      castChildOutput(p, relation.relation.asInstanceOf[CarbonDatasourceRelation], child)
-  }
-
-  def castChildOutput(p: InsertIntoTable, relation: CarbonDatasourceRelation, child: LogicalPlan)
-  : LogicalPlan = {
-    if (relation.carbonRelation.output.size > CarbonCommonConstants
-      .DEFAULT_MAX_NUMBER_OF_COLUMNS) {
-      sys
-        .error("Maximum supported column by carbon is:" + CarbonCommonConstants
-          .DEFAULT_MAX_NUMBER_OF_COLUMNS
-        )
-    }
-    if (child.output.size >= relation.carbonRelation.output.size ) {
-      InsertIntoCarbonTable(relation, p.partition, p.child, p.overwrite, p.ifNotExists)
-    } else {
-      sys.error("Cannot insert into target table because column number are different")
-    }
-  }
-}
-
-
-object CarbonIUDAnalysisRule extends Rule[LogicalPlan] {
-
-  var sqlContext: SQLContext = _
-
-  def init(sqlContext: SQLContext) {
-    this.sqlContext = sqlContext
-  }
-
-  private def processUpdateQuery(
-                   table: UnresolvedRelation,
-                   columns: List[String],
-                   selectStmt: String,
-                   filter: String): LogicalPlan = {
-    var includedDestColumns = false
-    var includedDestRelation = false
-    var addedTupleId = false
-
-    def prepareTargetReleation(relation: UnresolvedRelation): Subquery = {
-      val tupleId = UnresolvedAlias(Alias(UnresolvedFunction("getTupleId",
-        Seq.empty, isDistinct = false), "tupleId")())
-      val projList = Seq(
-        UnresolvedAlias(UnresolvedStar(table.alias)), tupleId)
-      // include tuple id and rest of the required columns in subqury
-      Subquery(table.alias.getOrElse(""), Project(projList, relation))
-    }
-    // get the un-analyzed logical plan
-    val targetTable = prepareTargetReleation(table)
-    val selectPlan = org.apache.spark.sql.SQLParser.parse(selectStmt, sqlContext) transform {
-      case Project(projectList, child) if (!includedDestColumns) =>
-        includedDestColumns = true
-        if (projectList.size != columns.size) {
-          sys.error("Number of source and destination columns are not matching")
-        }
-        val renamedProjectList = projectList.zip(columns).map{ case(attr, col) =>
-          attr match {
-            case UnresolvedAlias(child) =>
-              UnresolvedAlias(Alias(child, col + "-updatedColumn")())
-            case _ => attr
-          }
-        }
-        val list = Seq(
-          UnresolvedAlias(UnresolvedStar(table.alias))) ++ renamedProjectList
-        Project(list, child)
-      case Filter(cond, child) if (!includedDestRelation) =>
-        includedDestRelation = true
-        Filter(cond, Join(child, targetTable, Inner, None))
-      case r @ UnresolvedRelation(t, a) if (!includedDestRelation &&
-        t != table.tableIdentifier) =>
-        includedDestRelation = true
-        Join(r, targetTable, Inner, None)
-    }
-    val updatedSelectPlan = if (!includedDestRelation) {
-      // special case to handle self join queries
-      // Eg. update tableName  SET (column1) = (column1+1)
-      selectPlan transform {
-        case relation: UnresolvedRelation if (table.tableIdentifier == relation.tableIdentifier &&
-          addedTupleId == false) =>
-          addedTupleId = true
-          targetTable
-      }
-    } else {
-      selectPlan
-    }
-    val finalPlan = if (filter.length > 0) {
-      val alias = table.alias.getOrElse("")
-      var transformed: Boolean = false
-      // Create a dummy projection to include filter conditions
-      SQLParser.parse("select * from  " +
-        table.tableIdentifier.mkString(".") + " " + alias + " " + filter, sqlContext)  transform {
-        case UnresolvedRelation(t, Some(a)) if (
-          !transformed && t == table.tableIdentifier && a == alias) =>
-          transformed = true
-          // Add the filter condition of update statement  on destination table
-          Subquery(alias, updatedSelectPlan)
-      }
-    } else {
-      updatedSelectPlan
-    }
-    val tid = CarbonTableIdentifierImplicit.toTableIdentifier(table.tableIdentifier)
-    val tidSeq = Seq(getDB.getDatabaseName(tid.database, sqlContext), tid.table)
-    val destinationTable = UnresolvedRelation(tidSeq, table.alias)
-    ProjectForUpdate(destinationTable, columns, Seq(finalPlan))
-  }
-
-  def processDeleteRecordsQuery(selectStmt: String, table: UnresolvedRelation): LogicalPlan = {
-    val tid = CarbonTableIdentifierImplicit.toTableIdentifier(table.tableIdentifier)
-    val tidSeq = Seq(getDB.getDatabaseName(tid.database, sqlContext), tid.table)
-    var addedTupleId = false
-    val selectPlan = SQLParser.parse(selectStmt, sqlContext) transform {
-      case relation: UnresolvedRelation if (table.tableIdentifier == relation.tableIdentifier &&
-        addedTupleId == false) =>
-        addedTupleId = true
-        val tupleId = UnresolvedAlias(Alias(UnresolvedFunction("getTupleId",
-          Seq.empty, isDistinct = false), "tupleId")())
-        val projList = Seq(
-          UnresolvedAlias(UnresolvedStar(table.alias)), tupleId)
-        // include tuple id in subqury
-        Project(projList, relation)
-    }
-    ProjectForDeleteCommand(
-      selectPlan,
-      tidSeq,
-      System.currentTimeMillis().toString)
-  }
-
-  override def apply(logicalplan: LogicalPlan): LogicalPlan = {
-
-    logicalplan transform {
-      case UpdateTable(t, cols, sel, where) => processUpdateQuery(t, cols, sel, where)
-      case DeleteRecords(statement, table) => processDeleteRecordsQuery(statement, table)
-    }
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
deleted file mode 100644
index cd42fba..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.sql.catalyst.SqlParser
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-
-
-/**
- * This class contains all carbon hive metadata related utilities
- */
-object CarbonHiveMetadataUtil {
-
-  @transient
-  val LOGGER = LogServiceFactory.getLogService(CarbonHiveMetadataUtil.getClass.getName)
-
-
-  /**
-   * This method invalidates the table from HiveMetastoreCatalog before dropping table
-   *
-   * @param databaseName
-   * @param tableName
-   * @param sqlContext
-   */
-  def invalidateAndDropTable(databaseName: String,
-      tableName: String,
-      sqlContext: SQLContext): Unit = {
-    val hiveContext = sqlContext.asInstanceOf[HiveContext]
-    val tableWithDb = databaseName + "." + tableName
-    val tableIdent = SqlParser.parseTableIdentifier(tableWithDb)
-    try {
-      hiveContext.catalog.invalidateTable(tableIdent)
-      hiveContext.runSqlHive(s"DROP TABLE IF EXISTS $databaseName.$tableName")
-    } catch {
-      case e: Exception =>
-        LOGGER.audit(
-          s"Error While deleting the table $databaseName.$tableName during drop carbon table" +
-          e.getMessage)
-    }
-  }
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
deleted file mode 100644
index 4d5d39a..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ /dev/null
@@ -1,562 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import java.io._
-import java.util.UUID
-
-import scala.Array.canBuildFrom
-import scala.collection.mutable.ArrayBuffer
-import scala.language.implicitConversions
-import scala.util.parsing.combinator.RegexParsers
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.command.Partitioner
-import org.apache.spark.sql.hive.client.ClientInterface
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
-import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.reader.ThriftReader
-import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.writer.ThriftWriter
-import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
-import org.apache.carbondata.processing.merger.TableMeta
-
-case class MetaData(var tablesMeta: ArrayBuffer[TableMeta])
-
-case class CarbonMetaData(dims: Seq[String],
-    msrs: Seq[String],
-    carbonTable: CarbonTable,
-    dictionaryMap: DictionaryMap)
-
-object CarbonMetastore {
-
-  def readSchemaFileToThriftTable(schemaFilePath: String): TableInfo = {
-    val createTBase = new ThriftReader.TBaseCreator() {
-      override def create(): org.apache.thrift.TBase[TableInfo, TableInfo._Fields] = {
-        new TableInfo()
-      }
-    }
-    val thriftReader = new ThriftReader(schemaFilePath, createTBase)
-    var tableInfo: TableInfo = null
-    try {
-      thriftReader.open()
-      tableInfo = thriftReader.read().asInstanceOf[TableInfo]
-    } finally {
-      thriftReader.close()
-    }
-    tableInfo
-  }
-
-  def writeThriftTableToSchemaFile(schemaFilePath: String, tableInfo: TableInfo): Unit = {
-    val thriftWriter = new ThriftWriter(schemaFilePath, false)
-    try {
-      thriftWriter.open()
-      thriftWriter.write(tableInfo);
-    } finally {
-      thriftWriter.close()
-    }
-  }
-
-}
-
-case class DictionaryMap(dictionaryMap: Map[String, Boolean]) {
-  def get(name: String): Option[Boolean] = {
-    dictionaryMap.get(name.toLowerCase)
-  }
-}
-
-class CarbonMetastore(hiveContext: HiveContext, val storePath: String,
-    client: ClientInterface, queryId: String) extends HiveMetastoreCatalog(client, hiveContext) {
-
-  @transient
-  val LOGGER = LogServiceFactory.getLogService("org.apache.spark.sql.CarbonMetastoreCatalog")
-
-  val tableModifiedTimeStore = new java.util.HashMap[String, Long]()
-  tableModifiedTimeStore
-    .put(CarbonCommonConstants.DATABASE_DEFAULT_NAME, System.currentTimeMillis())
-
-  val metadata = loadMetadata(storePath)
-
-  def getTableCreationTime(databaseName: String, tableName: String): Long = {
-    val tableMeta = metadata.tablesMeta.filter(
-      c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(databaseName) &&
-           c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableName))
-    val tableCreationTime = tableMeta.head.carbonTable.getTableLastUpdatedTime
-    tableCreationTime
-  }
-
-  def lookupRelation1(dbName: Option[String],
-      tableName: String)(sqlContext: SQLContext): LogicalPlan = {
-    lookupRelation1(TableIdentifier(tableName, dbName))(sqlContext)
-  }
-
-  def lookupRelation1(tableIdentifier: TableIdentifier,
-      alias: Option[String] = None)(sqlContext: SQLContext): LogicalPlan = {
-    checkSchemasModifiedTimeAndReloadTables()
-    val database = tableIdentifier.database.getOrElse(getDB.getDatabaseName(None, sqlContext))
-    val tables = getTableFromMetadata(database, tableIdentifier.table)
-    tables match {
-      case Some(t) =>
-        CarbonRelation(database, tableIdentifier.table,
-          CarbonSparkUtil.createSparkMeta(tables.head.carbonTable), tables.head, alias)(sqlContext)
-      case None =>
-        LOGGER.audit(s"Table Not Found: ${tableIdentifier.table}")
-        throw new NoSuchTableException
-    }
-  }
-
-  /**
-   * This method will search for a table in the catalog metadata
-   *
-   * @param database
-   * @param tableName
-   * @return
-   */
-  def getTableFromMetadata(database: String,
-      tableName: String): Option[TableMeta] = {
-    metadata.tablesMeta
-      .find(c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
-                 c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableName))
-  }
-
-  def tableExists(identifier: TableIdentifier)(sqlContext: SQLContext): Boolean = {
-    checkSchemasModifiedTimeAndReloadTables()
-    val database = identifier.database.getOrElse(getDB.getDatabaseName(None, sqlContext))
-    val tables = metadata.tablesMeta.filter(
-      c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
-           c.carbonTableIdentifier.getTableName.equalsIgnoreCase(identifier.table))
-    tables.nonEmpty
-  }
-
-  def loadMetadata(metadataPath: String): MetaData = {
-    val recorder = CarbonTimeStatisticsFactory.createDriverRecorder()
-    val statistic = new QueryStatistic()
-    // creating zookeeper instance once.
-    // if zookeeper is configured as carbon lock type.
-    val zookeeperurl = hiveContext.getConf(CarbonCommonConstants.ZOOKEEPER_URL, null)
-    if (null != zookeeperurl) {
-      CarbonProperties.getInstance
-        .addProperty(CarbonCommonConstants.ZOOKEEPER_URL, zookeeperurl)
-    }
-    if (metadataPath == null) {
-      return null
-    }
-    // if no locktype is configured and store type is HDFS set HDFS lock as default
-    if (null == CarbonProperties.getInstance
-      .getProperty(CarbonCommonConstants.LOCK_TYPE) &&
-        FileType.HDFS == FileFactory.getFileType(metadataPath)) {
-      CarbonProperties.getInstance
-        .addProperty(CarbonCommonConstants.LOCK_TYPE,
-          CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS
-        )
-      LOGGER.info("Default lock type HDFSLOCK is configured")
-    }
-    val fileType = FileFactory.getFileType(metadataPath)
-    val metaDataBuffer = new ArrayBuffer[TableMeta]
-    fillMetaData(metadataPath, fileType, metaDataBuffer)
-    updateSchemasUpdatedTime(readSchemaFileSystemTime("", ""))
-    statistic.addStatistics(QueryStatisticsConstants.LOAD_META,
-      System.currentTimeMillis())
-    recorder.recordStatisticsForDriver(statistic, queryId)
-    MetaData(metaDataBuffer)
-  }
-
-  private def fillMetaData(basePath: String, fileType: FileType,
-      metaDataBuffer: ArrayBuffer[TableMeta]): Unit = {
-    val databasePath = basePath // + "/schemas"
-    try {
-      if (FileFactory.isFileExist(databasePath, fileType)) {
-        val file = FileFactory.getCarbonFile(databasePath, fileType)
-        val databaseFolders = file.listFiles()
-
-        databaseFolders.foreach(databaseFolder => {
-          if (databaseFolder.isDirectory) {
-            val dbName = databaseFolder.getName
-            val tableFolders = databaseFolder.listFiles()
-
-            tableFolders.foreach(tableFolder => {
-              if (tableFolder.isDirectory) {
-                val carbonTableIdentifier = new CarbonTableIdentifier(databaseFolder.getName,
-                  tableFolder.getName, UUID.randomUUID().toString)
-                val carbonTablePath = CarbonStorePath.getCarbonTablePath(basePath,
-                  carbonTableIdentifier)
-                val tableMetadataFile = carbonTablePath.getSchemaFilePath
-
-                if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
-                  val tableName = tableFolder.getName
-                  val tableUniqueName = databaseFolder.getName + "_" + tableFolder.getName
-
-
-                  val createTBase = new ThriftReader.TBaseCreator() {
-                    override def create(): org.apache.thrift.TBase[TableInfo, TableInfo._Fields] = {
-                      new TableInfo()
-                    }
-                  }
-                  val thriftReader = new ThriftReader(tableMetadataFile, createTBase)
-                  thriftReader.open()
-                  val tableInfo: TableInfo = thriftReader.read().asInstanceOf[TableInfo]
-                  thriftReader.close()
-
-                  val schemaConverter = new ThriftWrapperSchemaConverterImpl
-                  val wrapperTableInfo = schemaConverter
-                    .fromExternalToWrapperTableInfo(tableInfo, dbName, tableName, basePath)
-                  val schemaFilePath = CarbonStorePath
-                    .getCarbonTablePath(storePath, carbonTableIdentifier).getSchemaFilePath
-                  wrapperTableInfo.setStorePath(storePath)
-                  wrapperTableInfo
-                    .setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
-                  CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
-                  val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
-                  metaDataBuffer += new TableMeta(carbonTable.getCarbonTableIdentifier, storePath,
-                    null, carbonTable)
-                }
-              }
-            })
-          }
-        })
-      } else {
-        // Create folders and files.
-        FileFactory.mkdirs(databasePath, fileType)
-      }
-    } catch {
-      case s: java.io.FileNotFoundException =>
-        // Create folders and files.
-        FileFactory.mkdirs(databasePath, fileType)
-    }
-  }
-
-  /**
-   *
-   * Prepare Thrift Schema from wrapper TableInfo and write to Schema file.
-   * Load CarbonTable from wrapper tableinfo
-   *
-   */
-  def createTableFromThrift(
-      tableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo,
-      dbName: String, tableName: String, partitioner: Partitioner)
-    (sqlContext: SQLContext): String = {
-    if (tableExists(TableIdentifier(tableName, Some(dbName)))(sqlContext)) {
-      sys.error(s"Table [$tableName] already exists under Database [$dbName]")
-    }
-    val schemaConverter = new ThriftWrapperSchemaConverterImpl
-    val thriftTableInfo = schemaConverter
-      .fromWrapperToExternalTableInfo(tableInfo, dbName, tableName)
-    val schemaEvolutionEntry = new SchemaEvolutionEntry(tableInfo.getLastUpdatedTime)
-    thriftTableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history
-      .add(schemaEvolutionEntry)
-
-    val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName,
-      tableInfo.getFactTable.getTableId)
-    val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier)
-    val schemaFilePath = carbonTablePath.getSchemaFilePath
-    val schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath)
-    tableInfo.setMetaDataFilepath(schemaMetadataPath)
-    tableInfo.setStorePath(storePath)
-    CarbonMetadata.getInstance().loadTableMetadata(tableInfo)
-    val tableMeta = new TableMeta(carbonTableIdentifier, storePath, null,
-      CarbonMetadata.getInstance().getCarbonTable(dbName + "_" + tableName))
-
-    val fileType = FileFactory.getFileType(schemaMetadataPath)
-    if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
-      FileFactory.mkdirs(schemaMetadataPath, fileType)
-    }
-
-    val thriftWriter = new ThriftWriter(schemaFilePath, false)
-    thriftWriter.open()
-    thriftWriter.write(thriftTableInfo)
-    thriftWriter.close()
-
-    metadata.tablesMeta += tableMeta
-    logInfo(s"Table $tableName for Database $dbName created successfully.")
-    LOGGER.info(s"Table $tableName for Database $dbName created successfully.")
-    updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
-    carbonTablePath.getPath
-  }
-
-  private def updateMetadataByWrapperTable(
-      wrapperTableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo): Unit = {
-
-    CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
-    val carbonTable = CarbonMetadata.getInstance().getCarbonTable(
-      wrapperTableInfo.getTableUniqueName)
-    for (i <- metadata.tablesMeta.indices) {
-      if (wrapperTableInfo.getTableUniqueName.equals(
-        metadata.tablesMeta(i).carbonTableIdentifier.getTableUniqueName)) {
-        metadata.tablesMeta(i).carbonTable = carbonTable
-      }
-    }
-  }
-
-  def updateMetadataByThriftTable(schemaFilePath: String,
-      tableInfo: TableInfo, dbName: String, tableName: String, storePath: String): Unit = {
-
-    tableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history.get(0)
-      .setTime_stamp(System.currentTimeMillis())
-    val schemaConverter = new ThriftWrapperSchemaConverterImpl
-    val wrapperTableInfo = schemaConverter
-      .fromExternalToWrapperTableInfo(tableInfo, dbName, tableName, storePath)
-    wrapperTableInfo
-      .setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
-    wrapperTableInfo.setStorePath(storePath)
-    updateMetadataByWrapperTable(wrapperTableInfo)
-  }
-
-  /**
-   * Shows all tables for given schema.
-   */
-  def getTables(databaseName: Option[String])(sqlContext: SQLContext): Seq[(String, Boolean)] = {
-
-    val dbName =
-      databaseName.getOrElse(sqlContext.asInstanceOf[HiveContext].catalog.client.currentDatabase)
-    checkSchemasModifiedTimeAndReloadTables()
-    metadata.tablesMeta.filter { c =>
-      c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(dbName)
-    }.map { c => (c.carbonTableIdentifier.getTableName, false) }
-  }
-
-  def isTablePathExists(tableIdentifier: TableIdentifier)(sqlContext: SQLContext): Boolean = {
-    val dbName = tableIdentifier.database.getOrElse(getDB.getDatabaseName(None, sqlContext))
-    val tableName = tableIdentifier.table
-
-    val tablePath = CarbonStorePath.getCarbonTablePath(this.storePath,
-      new CarbonTableIdentifier(dbName, tableName, "")).getPath
-
-    val fileType = FileFactory.getFileType(tablePath)
-    FileFactory.isFileExist(tablePath, fileType)
-  }
-
-  def dropTable(tableStorePath: String, tableIdentifier: TableIdentifier)
-    (sqlContext: SQLContext) {
-    val dbName = tableIdentifier.database.get
-    val tableName = tableIdentifier.table
-
-    val metadataFilePath = CarbonStorePath.getCarbonTablePath(tableStorePath,
-      new CarbonTableIdentifier(dbName, tableName, "")).getMetadataDirectoryPath
-
-    val fileType = FileFactory.getFileType(metadataFilePath)
-
-    if (FileFactory.isFileExist(metadataFilePath, fileType)) {
-      // while drop we should refresh the schema modified time so that if any thing has changed
-      // in the other beeline need to update.
-      checkSchemasModifiedTimeAndReloadTables
-      val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
-      CarbonUtil.deleteFoldersAndFilesSilent(file.getParentFile)
-      val metadataToBeRemoved: Option[TableMeta] = getTableFromMetadata(dbName,
-        tableIdentifier.table)
-      metadataToBeRemoved match {
-        case Some(tableMeta) =>
-          metadata.tablesMeta -= tableMeta
-          CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
-          CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
-          updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
-        case None =>
-          logInfo(s"Metadata does not contain entry for table $tableName in database $dbName")
-      }
-      CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sqlContext)
-      // discard cached table info in cachedDataSourceTables
-      sqlContext.catalog.refreshTable(tableIdentifier)
-      DataMapStoreManager.getInstance().
-        clearDataMap(AbsoluteTableIdentifier.from(storePath, dbName, tableName))
-    }
-  }
-
-  private def getTimestampFileAndType(databaseName: String, tableName: String) = {
-    val timestampFile = storePath + "/" + CarbonCommonConstants.SCHEMAS_MODIFIED_TIME_FILE
-    val timestampFileType = FileFactory.getFileType(timestampFile)
-    (timestampFile, timestampFileType)
-  }
-
-  /**
-   * This method will put the updated timestamp of schema file in the table modified time store map
-   *
-   * @param timeStamp
-   */
-  def updateSchemasUpdatedTime(timeStamp: Long) {
-    tableModifiedTimeStore.put("default", timeStamp)
-  }
-
-  /**
-   * This method will read the timestamp of empty schema file
-   *
-   * @param databaseName
-   * @param tableName
-   * @return
-   */
-  def readSchemaFileSystemTime(databaseName: String, tableName: String): Long = {
-    val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
-    if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
-      FileFactory.getCarbonFile(timestampFile, timestampFileType).getLastModifiedTime
-    } else {
-      System.currentTimeMillis()
-    }
-  }
-
-  /**
-   * This method will check and create an empty schema timestamp file
-   *
-   * @param databaseName
-   * @param tableName
-   * @return
-   */
-  def touchSchemaFileSystemTime(databaseName: String, tableName: String): Long = {
-    val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
-    if (!FileFactory.isFileExist(timestampFile, timestampFileType)) {
-      LOGGER.audit(s"Creating timestamp file for $databaseName.$tableName")
-      FileFactory.createNewFile(timestampFile, timestampFileType)
-    }
-    val systemTime = System.currentTimeMillis()
-    FileFactory.getCarbonFile(timestampFile, timestampFileType)
-      .setLastModifiedTime(systemTime)
-    systemTime
-  }
-
-  def checkSchemasModifiedTimeAndReloadTables() {
-    val (timestampFile, timestampFileType) = getTimestampFileAndType("", "")
-    if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
-      if (!(FileFactory.getCarbonFile(timestampFile, timestampFileType).
-        getLastModifiedTime ==
-            tableModifiedTimeStore.get(CarbonCommonConstants.DATABASE_DEFAULT_NAME))) {
-        refreshCache()
-      }
-    }
-  }
-
-  def refreshCache() {
-    metadata.tablesMeta = loadMetadata(storePath).tablesMeta
-  }
-
-  def getSchemaLastUpdatedTime(databaseName: String, tableName: String): Long = {
-    var schemaLastUpdatedTime = System.currentTimeMillis
-    val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
-    if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
-      schemaLastUpdatedTime = FileFactory.getCarbonFile(timestampFile, timestampFileType)
-        .getLastModifiedTime
-    }
-    schemaLastUpdatedTime
-  }
-
-  def createDatabaseDirectory(dbName: String) {
-    val databasePath = storePath + File.separator + dbName
-    val fileType = FileFactory.getFileType(databasePath)
-    FileFactory.mkdirs(databasePath, fileType)
-  }
-
-  def dropDatabaseDirectory(dbName: String) {
-    val databasePath = storePath + File.separator + dbName
-    val fileType = FileFactory.getFileType(databasePath)
-    if (FileFactory.isFileExist(databasePath, fileType)) {
-      val dbPath = FileFactory.getCarbonFile(databasePath, fileType)
-      CarbonUtil.deleteFoldersAndFiles(dbPath)
-    }
-  }
-
-}
-
-
-object CarbonMetastoreTypes extends RegexParsers {
-  protected lazy val primitiveType: Parser[DataType] =
-    "string" ^^^ StringType |
-    "float" ^^^ FloatType |
-    "int" ^^^ IntegerType |
-    "tinyint" ^^^ ShortType |
-    "short" ^^^ ShortType |
-    "double" ^^^ DoubleType |
-    "long" ^^^ LongType |
-    "binary" ^^^ BinaryType |
-    "boolean" ^^^ BooleanType |
-    fixedDecimalType |
-    "decimal" ^^^ "decimal" ^^^ DecimalType(18, 2) |
-    "varchar\\((\\d+)\\)".r ^^^ StringType |
-    "timestamp" ^^^ TimestampType |
-    "date" ^^^ DateType |
-    "char\\((\\d+)\\)".r ^^^ StringType
-
-  protected lazy val fixedDecimalType: Parser[DataType] =
-    "decimal" ~> "(" ~> "^[1-9]\\d*".r ~ ("," ~> "^[0-9]\\d*".r <~ ")") ^^ {
-      case precision ~ scale =>
-        DecimalType(precision.toInt, scale.toInt)
-    }
-
-  protected lazy val arrayType: Parser[DataType] =
-    "array" ~> "<" ~> dataType <~ ">" ^^ {
-      case tpe => ArrayType(tpe)
-    }
-
-  protected lazy val mapType: Parser[DataType] =
-    "map" ~> "<" ~> dataType ~ "," ~ dataType <~ ">" ^^ {
-      case t1 ~ _ ~ t2 => MapType(t1, t2)
-    }
-
-  protected lazy val structField: Parser[StructField] =
-    "[a-zA-Z0-9_]*".r ~ ":" ~ dataType ^^ {
-      case name ~ _ ~ tpe => StructField(name, tpe, nullable = true)
-    }
-
-  protected lazy val structType: Parser[DataType] =
-    "struct" ~> "<" ~> repsep(structField, ",") <~ ">" ^^ {
-      case fields => StructType(fields)
-    }
-
-  protected lazy val dataType: Parser[DataType] =
-    arrayType |
-    mapType |
-    structType |
-    primitiveType
-
-  def toDataType(metastoreType: String): DataType = {
-    parseAll(dataType, metastoreType) match {
-      case Success(result, _) => result
-      case failure: NoSuccess => sys.error(s"Unsupported dataType: $metastoreType")
-    }
-  }
-
-  def toMetastoreType(dt: DataType): String = {
-    dt match {
-      case ArrayType(elementType, _) => s"array<${ toMetastoreType(elementType) }>"
-      case StructType(fields) =>
-        s"struct<${
-          fields.map(f => s"${ f.name }:${ toMetastoreType(f.dataType) }")
-            .mkString(",")
-        }>"
-      case StringType => "string"
-      case FloatType => "float"
-      case IntegerType => "int"
-      case ShortType => "tinyint"
-      case DoubleType => "double"
-      case LongType => "bigint"
-      case BinaryType => "binary"
-      case BooleanType => "boolean"
-      case DecimalType() => "decimal"
-      case DateType => "date"
-      case TimestampType => "timestamp"
-    }
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala
deleted file mode 100644
index 82c5f7f..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql.CarbonSqlParser
-import org.apache.spark.sql.catalyst.ParserDialect
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
-private[spark] class CarbonSQLDialect(hiveContext: HiveContext) extends ParserDialect {
-
-  @transient
-  protected val sqlParser = new CarbonSqlParser
-
-  override def parse(sqlText: String): LogicalPlan = {
-
-    try {
-      sqlParser.parse(sqlText)
-    } catch {
-      // MalformedCarbonCommandException need to throw directly
-      // because hive can no parse carbon command
-      case ce: MalformedCarbonCommandException =>
-        throw ce
-      case _: Throwable =>
-        HiveQl.parseSql(sqlText)
-    }
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
deleted file mode 100644
index d3d699a..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import java.util
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, AttributeSet, ScalaUDF, _}
-import org.apache.spark.sql.catalyst.planning.{PhysicalOperation, QueryPlanner}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter => LogicalFilter, LogicalPlan}
-import org.apache.spark.sql.execution.{ExecutedCommand, Filter, Project, SparkPlan}
-import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.{DescribeCommand => LogicalDescribeCommand, LogicalRelation}
-import org.apache.spark.sql.hive.execution.{DropTable, HiveNativeCommand}
-import org.apache.spark.sql.hive.execution.command._
-import org.apache.spark.sql.optimizer.CarbonDecoderRelation
-import org.apache.spark.sql.types.{IntegerType, StringType}
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.spark.CarbonAliasDecoderRelation
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
-class CarbonStrategies(sqlContext: SQLContext) extends QueryPlanner[SparkPlan] {
-
-  override def strategies: Seq[Strategy] = getStrategies
-
-  val LOGGER = LogServiceFactory.getLogService("CarbonStrategies")
-
-  def getStrategies: Seq[Strategy] = {
-    val total = sqlContext.planner.strategies :+ CarbonTableScan
-    total
-  }
-
-  /**
-   * Carbon strategies for performing late materizlization (decoding dictionary key
-   * as late as possbile)
-   */
-  private[sql] object CarbonTableScan extends Strategy {
-
-    def apply(plan: LogicalPlan): Seq[SparkPlan] = {
-      plan match {
-        case PhysicalOperation(projectList, predicates, l: LogicalRelation)
-          if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
-          if (isStarQuery(plan)) {
-            carbonRawScanForStarQuery(projectList, predicates, l)(sqlContext) :: Nil
-          } else {
-            carbonRawScan(projectList, predicates, l)(sqlContext) :: Nil
-          }
-        case InsertIntoCarbonTable(relation: CarbonDatasourceRelation,
-        _, child: LogicalPlan, overwrite, _) =>
-          ExecutedCommand(LoadTableByInsert(relation, child, overwrite)) :: Nil
-        case CarbonDictionaryCatalystDecoder(relations, profile, aliasMap, _, child) =>
-          CarbonDictionaryDecoder(relations,
-            profile,
-            aliasMap,
-            planLater(child))(sqlContext) :: Nil
-        case _ =>
-          Nil
-      }
-    }
-
-    /**
-     * Create carbon scan
-     */
-    private def carbonRawScan(projectListRaw: Seq[NamedExpression],
-        predicates: Seq[Expression],
-        logicalRelation: LogicalRelation)(sc: SQLContext): SparkPlan = {
-
-      val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceRelation]
-      val tableName: String =
-        relation.carbonRelation.metaData.carbonTable.getFactTableName.toLowerCase
-      // Check out any expressions are there in project list. if they are present then we need to
-      // decode them as well.
-
-      val projectList = projectListRaw.map {p =>
-        p.transform {
-          case CustomDeterministicExpression(exp) => exp
-        }
-      }.asInstanceOf[Seq[NamedExpression]]
-      val newProjectList = projectList.map {
-        case a@Alias(s: ScalaUDF, name)
-          if name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID) ||
-             name.equalsIgnoreCase(
-               CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID) =>
-          AttributeReference(name, StringType, true)().withExprId(a.exprId)
-        case a@Alias(s: ScalaUDF, name)
-          if name.equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_SEGMENTID) =>
-          val reference =
-            AttributeReference(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID,
-              StringType, true)().withExprId(a.exprId)
-          val alias = a.transform {
-            case s: ScalaUDF =>
-              ScalaUDF(s.function, s.dataType, Seq(reference), s.inputTypes)
-          }.asInstanceOf[Alias]
-          Alias(alias.child, alias.name)(alias.exprId, alias.qualifiers, alias.explicitMetadata)
-        case other => other
-      }
-      val projectSet = AttributeSet(newProjectList.flatMap(_.references))
-      val filterSet = AttributeSet(predicates.flatMap(_.references))
-      val scan = CarbonScan(projectSet.toSeq,
-        relation.carbonRelation,
-        predicates)(sqlContext)
-      newProjectList.map {
-        case attr: AttributeReference =>
-        case Alias(attr: AttributeReference, _) =>
-        case others =>
-          others.references.map { f =>
-            val dictionary = relation.carbonRelation.metaData.dictionaryMap.get(f.name)
-            if (dictionary.isDefined && dictionary.get) {
-              scan.attributesNeedToDecode.add(f.asInstanceOf[AttributeReference])
-            }
-          }
-      }
-      val scanWithDecoder =
-        if (scan.attributesNeedToDecode.size() > 0) {
-          val decoder = getCarbonDecoder(logicalRelation,
-            sc,
-            tableName,
-            scan.attributesNeedToDecode.asScala.toSeq,
-            scan)
-          if (scan.unprocessedExprs.nonEmpty) {
-            val filterCondToAdd = scan.unprocessedExprs.reduceLeftOption(expressions.And)
-            filterCondToAdd.map(Filter(_, decoder)).getOrElse(decoder)
-          } else {
-            decoder
-          }
-        } else {
-          scan
-        }
-
-      if (projectList.map(_.toAttribute) == scan.columnProjection &&
-          projectSet.size == projectList.size &&
-          filterSet.subsetOf(projectSet)) {
-        // copied from spark pruneFilterProjectRaw
-        // When it is possible to just use column pruning to get the right projection and
-        // when the columns of this projection are enough to evaluate all filter conditions,
-        // just do a scan with no extra project.
-        scanWithDecoder
-      } else {
-        Project(newProjectList, scanWithDecoder)
-      }
-    }
-
-    /**
-     * Create carbon scan for star query
-     */
-    private def carbonRawScanForStarQuery(projectList: Seq[NamedExpression],
-        predicates: Seq[Expression],
-        logicalRelation: LogicalRelation)(sc: SQLContext): SparkPlan = {
-      val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceRelation]
-      val tableName: String =
-        relation.carbonRelation.metaData.carbonTable.getFactTableName.toLowerCase
-      // Check out any expressions are there in project list. if they are present then we need to
-      // decode them as well.
-      val projectExprsNeedToDecode = new java.util.HashSet[Attribute]()
-      val scan = CarbonScan(projectList.map(_.toAttribute),
-        relation.carbonRelation,
-        predicates,
-        useUnsafeCoversion = false)(sqlContext)
-      projectExprsNeedToDecode.addAll(scan.attributesNeedToDecode)
-      val updatedAttrs = scan.columnProjection.map(attr =>
-        updateDataType(attr.asInstanceOf[AttributeReference], relation, projectExprsNeedToDecode))
-      scan.columnProjection = updatedAttrs
-      if (projectExprsNeedToDecode.size() > 0
-          && isDictionaryEncoded(projectExprsNeedToDecode.asScala.toSeq, relation)) {
-        val decoder = getCarbonDecoder(logicalRelation,
-          sc,
-          tableName,
-          projectExprsNeedToDecode.asScala.toSeq,
-          scan)
-        if (scan.unprocessedExprs.nonEmpty) {
-          val filterCondToAdd = scan.unprocessedExprs.reduceLeftOption(expressions.And)
-          filterCondToAdd.map(Filter(_, decoder)).getOrElse(decoder)
-        } else {
-          decoder
-        }
-      } else {
-        if (scan.unprocessedExprs.nonEmpty) {
-          val filterCondToAdd = scan.unprocessedExprs.reduceLeftOption(expressions.And)
-          filterCondToAdd.map(Filter(_, scan)).getOrElse(scan)
-        } else {
-          scan
-        }
-      }
-    }
-
-    def getCarbonDecoder(logicalRelation: LogicalRelation,
-        sc: SQLContext,
-        tableName: String,
-        projectExprsNeedToDecode: Seq[Attribute],
-        scan: CarbonScan): CarbonDictionaryDecoder = {
-      val relation = CarbonDecoderRelation(logicalRelation.attributeMap,
-        logicalRelation.relation.asInstanceOf[CarbonDatasourceRelation])
-      val attrs = projectExprsNeedToDecode.map { attr =>
-        val newAttr = AttributeReference(attr.name,
-          attr.dataType,
-          attr.nullable,
-          attr.metadata)(attr.exprId, Seq(tableName))
-        relation.addAttribute(newAttr)
-        newAttr
-      }
-      CarbonDictionaryDecoder(Seq(relation), IncludeProfile(attrs),
-        CarbonAliasDecoderRelation(), scan)(sc)
-    }
-
-    def isDictionaryEncoded(projectExprsNeedToDecode: Seq[Attribute],
-        relation: CarbonDatasourceRelation): Boolean = {
-      var isEncoded = false
-      projectExprsNeedToDecode.foreach { attr =>
-        if (relation.carbonRelation.metaData.dictionaryMap.get(attr.name).getOrElse(false)) {
-          isEncoded = true
-        }
-      }
-      isEncoded
-    }
-
-    def updateDataType(attr: AttributeReference,
-        relation: CarbonDatasourceRelation,
-        allAttrsNotDecode: util.Set[Attribute]): AttributeReference = {
-      if (relation.carbonRelation.metaData.dictionaryMap.get(attr.name).getOrElse(false) &&
-          !allAttrsNotDecode.asScala.exists(p => p.name.equals(attr.name))) {
-        AttributeReference(attr.name,
-          IntegerType,
-          attr.nullable,
-          attr.metadata)(attr.exprId, attr.qualifiers)
-      } else {
-        attr
-      }
-    }
-
-    private def isStarQuery(plan: LogicalPlan) = {
-      plan match {
-        case LogicalFilter(condition, l: LogicalRelation)
-          if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
-          true
-        case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] => true
-        case _ => false
-      }
-    }
-  }
-
-  object DDLStrategies extends Strategy {
-    def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
-      case DropTable(tableName, ifNotExists)
-        if CarbonEnv.get.carbonMetastore
-          .isTablePathExists(toTableIdentifier(tableName.toLowerCase))(sqlContext) =>
-        val identifier = toTableIdentifier(tableName.toLowerCase)
-        ExecutedCommand(DropTableCommand(ifNotExists, identifier.database, identifier.table)) :: Nil
-      case ShowLoadsCommand(databaseName, table, limit) =>
-        ExecutedCommand(ShowLoads(databaseName, table, limit, plan.output)) :: Nil
-      case LoadTable(databaseNameOp, tableName, factPathFromUser, dimFilesPath,
-      options, isOverwriteExist, inputSqlString, dataFrame, _) =>
-        val isCarbonTable = CarbonEnv.get.carbonMetastore
-          .tableExists(TableIdentifier(tableName, databaseNameOp))(sqlContext)
-        if (isCarbonTable || options.nonEmpty) {
-          ExecutedCommand(LoadTable(databaseNameOp, tableName, factPathFromUser, dimFilesPath,
-            options, isOverwriteExist, inputSqlString, dataFrame)) :: Nil
-        } else {
-          ExecutedCommand(HiveNativeCommand(inputSqlString)) :: Nil
-        }
-      case alterTable@AlterTableCompaction(altertablemodel) =>
-        val isCarbonTable = CarbonEnv.get.carbonMetastore
-          .tableExists(TableIdentifier(altertablemodel.tableName,
-            altertablemodel.dbName))(sqlContext)
-        if (isCarbonTable) {
-          if (altertablemodel.compactionType.equalsIgnoreCase("minor") ||
-              altertablemodel.compactionType.equalsIgnoreCase("major")) {
-            ExecutedCommand(alterTable) :: Nil
-          } else {
-            throw new MalformedCarbonCommandException(
-              "Unsupported alter operation on carbon table")
-          }
-        } else {
-          ExecutedCommand(HiveNativeCommand(altertablemodel.alterSql)) :: Nil
-        }
-      case CreateDatabase(dbName, sql) =>
-        ExecutedCommand(CreateDatabaseCommand(dbName, HiveNativeCommand(sql))) :: Nil
-      case DropDatabase(dbName, isCascade, sql) =>
-        if (isCascade) {
-          ExecutedCommand(DropDatabaseCascadeCommand(dbName, HiveNativeCommand(sql))) :: Nil
-        } else {
-          ExecutedCommand(DropDatabaseCommand(dbName, HiveNativeCommand(sql))) :: Nil
-        }
-      case UseDatabase(sql) =>
-        ExecutedCommand(HiveNativeCommand(sql)) :: Nil
-      case d: HiveNativeCommand =>
-        try {
-          val resolvedTable = sqlContext.executePlan(CarbonHiveSyntax.parse(d.sql)).optimizedPlan
-          planLater(resolvedTable) :: Nil
-        } catch {
-          case ce: MalformedCarbonCommandException =>
-            throw ce
-          case ae: AnalysisException =>
-            throw ae
-          case e: Exception => ExecutedCommand(d) :: Nil
-        }
-      case DescribeFormattedCommand(sql, tblIdentifier) =>
-        val isTable = CarbonEnv.get.carbonMetastore
-          .tableExists(tblIdentifier)(sqlContext)
-        if (isTable) {
-          val describe =
-            LogicalDescribeCommand(UnresolvedRelation(tblIdentifier, None), isExtended = false)
-          val resolvedTable = sqlContext.executePlan(describe.table).analyzed
-          val resultPlan = sqlContext.executePlan(resolvedTable).executedPlan
-          ExecutedCommand(DescribeCommandFormatted(resultPlan, plan.output, tblIdentifier)) :: Nil
-        } else {
-          ExecutedCommand(HiveNativeCommand(sql)) :: Nil
-        }
-      case ShowPartitions(t) =>
-        val isCarbonTable = CarbonEnv.get.carbonMetastore
-          .tableExists(t)(sqlContext)
-        if (isCarbonTable) {
-          ExecutedCommand(ShowCarbonPartitionsCommand(t)) :: Nil
-        } else {
-          var tableName = t.table
-          var database = t.database
-          var sql: String = null
-          if (database.isEmpty) {
-            sql = s"show partitions $tableName"
-          } else {
-            sql = s"show partitions $database.$tableName"
-          }
-          ExecutedCommand(HiveNativeCommand(sql)) :: Nil
-        }
-      case _ =>
-        Nil
-    }
-
-    def toTableIdentifier(name: String): TableIdentifier = {
-      val identifier = name.split("\\.")
-      identifier match {
-        case Array(tableName) => TableIdentifier(tableName, None)
-        case Array(dbName, tableName) => TableIdentifier(tableName, Some(dbName))
-      }
-    }
-  }
-
-}
-
-object CarbonHiveSyntax {
-
-  @transient
-  protected val sqlParser = new CarbonSqlParser
-
-  def parse(sqlText: String): LogicalPlan = {
-    sqlParser.parse(sqlText)
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/HiveQlWrapper.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/HiveQlWrapper.scala
deleted file mode 100644
index 6b244f9..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/HiveQlWrapper.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import org.apache.hadoop.hive.ql.parse.ASTNode
-
-/**
- * Wrapper class for using the hiveQl class of hive.
- */
-object HiveQlWrapper {
-
-  def getAst(sql: String): ASTNode = {
-
-    HiveQl.getAst(sql)
-  }
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
deleted file mode 100644
index ad70027..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive.cli
-
-import java.io.File
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.scheduler.StatsReportListener
-import org.apache.spark.sql.CarbonContext
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.sql.hive.thriftserver.{SparkSQLCLIDriver, SparkSQLEnv}
-import org.apache.spark.util.Utils
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-
-object CarbonSQLCLIDriver {
-
-  private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-  var hiveContext: HiveContext = _
-  var sparkContext: SparkContext = _
-
-  def main(args: Array[String]): Unit = {
-    init()
-    SparkSQLEnv.sparkContext = sparkContext
-    SparkSQLEnv.hiveContext = hiveContext
-    SparkSQLCLIDriver.installSignalHandler()
-    SparkSQLCLIDriver.main(args)
-  }
-
-  def init() {
-    if (hiveContext == null) {
-      val sparkConf = new SparkConf(loadDefaults = true)
-      val maybeSerializer = sparkConf.getOption("spark.serializer")
-      val maybeKryoReferenceTracking = sparkConf.getOption("spark.kryo.referenceTracking")
-      // If user doesn't specify the appName, we want to get [SparkSQL::localHostName] instead of
-      // the default appName [CarbonSQLCLIDriver] in cli or beeline.
-      val maybeAppName = sparkConf
-        .getOption("spark.app.name")
-        .filterNot(_ == classOf[SparkSQLCLIDriver].getName)
-      val maybeStorePath = sparkConf.getOption("spark.carbon.storepath")
-
-      sparkConf
-        .setAppName(maybeAppName.getOrElse(s"CarbonSparkSQL::${ Utils.localHostName() }"))
-        .set(
-          "spark.serializer",
-          maybeSerializer.getOrElse("org.apache.spark.serializer.KryoSerializer"))
-        .set(
-          "spark.kryo.referenceTracking",
-          maybeKryoReferenceTracking.getOrElse("false"))
-
-      sparkContext = new SparkContext(sparkConf)
-      sparkContext.addSparkListener(new StatsReportListener())
-      val path = System.getenv("CARBON_HOME") + "/bin/carbonsqlclistore"
-      val store = new File(path)
-      store.mkdirs()
-      hiveContext = new CarbonContext(sparkContext,
-        maybeStorePath.getOrElse(store.getCanonicalPath),
-        store.getCanonicalPath)
-
-      hiveContext.setConf("spark.sql.hive.version", HiveContext.hiveExecutionVersion)
-      hiveContext.hiveconf.getAllProperties.asScala.toSeq.sorted.foreach { case (k, v) =>
-        LOGGER.debug(s"HiveConf var: $k=$v")
-      }
-    }
-  }
-
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
deleted file mode 100644
index 0f42940..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.execution.command
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.execution.RunnableCommand
-import org.apache.spark.sql.execution.command.DropTableCommand
-import org.apache.spark.sql.hive.execution.HiveNativeCommand
-
-private[hive] case class CreateDatabaseCommand(dbName: String,
-    command: HiveNativeCommand) extends RunnableCommand {
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    val rows = command.run(sqlContext)
-    CarbonEnv.get.carbonMetastore.createDatabaseDirectory(dbName)
-    rows
-  }
-}
-
-private[hive] case class DropDatabaseCommand(dbName: String,
-    command: HiveNativeCommand) extends RunnableCommand {
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    val rows = command.run(sqlContext)
-    CarbonEnv.get.carbonMetastore.dropDatabaseDirectory(dbName)
-    rows
-  }
-}
-
-private[hive] case class DropDatabaseCascadeCommand(dbName: String,
-    command: HiveNativeCommand) extends RunnableCommand {
-  def run(sqlContext: SQLContext): Seq[Row] = {
-    val tablesInDB = CarbonEnv.get.carbonMetastore
-        .getTables(Some(dbName))(sqlContext).map(x => x._1)
-    val rows = command.run(sqlContext)
-    tablesInDB.foreach{tableName =>
-      DropTableCommand(true, Some(dbName), tableName).run(sqlContext)
-    }
-    CarbonEnv.get.carbonMetastore.dropDatabaseDirectory(dbName)
-    rows
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
deleted file mode 100644
index 2e45954..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark
-
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, Literal, StartsWith, _}
-import org.apache.spark.sql.optimizer.AttributeReferenceWrapper
-import org.apache.spark.sql.sources
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
-import org.apache.carbondata.core.scan.expression.conditional.{GreaterThanEqualToExpression, LessThanExpression, _}
-import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-/**
- * All filter conversions are done here.
- */
-object CarbonFilters {
-
-  /**
-   * Converts data sources filters to carbon filter predicates.
-   */
-  def createCarbonFilter(schema: StructType,
-      predicate: sources.Filter): Option[CarbonExpression] = {
-    val dataTypeOf = schema.map(f => f.name -> f.dataType).toMap
-
-    def createFilter(predicate: sources.Filter): Option[CarbonExpression] = {
-      predicate match {
-
-        case sources.EqualTo(name, value) =>
-          Some(new EqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.Not(sources.EqualTo(name, value)) =>
-          Some(new NotEqualsExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-
-        case sources.EqualNullSafe(name, value) =>
-          Some(new EqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.Not(sources.EqualNullSafe(name, value)) =>
-          Some(new NotEqualsExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-
-        case sources.GreaterThan(name, value) =>
-          Some(new GreaterThanExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.LessThan(name, value) =>
-          Some(new LessThanExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.GreaterThanOrEqual(name, value) =>
-          Some(new GreaterThanEqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.LessThanOrEqual(name, value) =>
-          Some(new LessThanEqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-
-        case sources.In(name, values) =>
-          Some(new InExpression(getCarbonExpression(name),
-            new ListExpression(
-              convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
-        case sources.Not(sources.In(name, values)) =>
-          Some(new NotInExpression(getCarbonExpression(name),
-            new ListExpression(
-              convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
-
-        case sources.IsNull(name) =>
-          Some(new EqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, null), true))
-        case sources.IsNotNull(name) =>
-          Some(new NotEqualsExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, null), true))
-
-        case sources.And(lhs, rhs) =>
-          (createFilter(lhs) ++ createFilter(rhs)).reduceOption(new AndExpression(_, _))
-
-        case sources.Or(lhs, rhs) =>
-          for {
-            lhsFilter <- createFilter(lhs)
-            rhsFilter <- createFilter(rhs)
-          } yield {
-            new OrExpression(lhsFilter, rhsFilter)
-          }
-        case sources.StringStartsWith(name, value) if value.length > 0 =>
-          val l = new GreaterThanEqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value))
-          val maxValueLimit = value.substring(0, value.length - 1) +
-                              (value.charAt(value.length - 1).toInt + 1).toChar
-          val r = new LessThanExpression(
-            getCarbonExpression(name), getCarbonLiteralExpression(name, maxValueLimit))
-          Some(new AndExpression(l, r))
-        case _ => None
-      }
-    }
-
-    def getCarbonExpression(name: String) = {
-      new CarbonColumnExpression(name,
-        CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name)))
-    }
-
-    def getCarbonLiteralExpression(name: String, value: Any): CarbonExpression = {
-      val dataTypeOfAttribute = CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name))
-      val dataType = if (Option(value).isDefined
-                         && dataTypeOfAttribute == CarbonDataTypes.STRING
-                         && value.isInstanceOf[Double]) {
-        CarbonDataTypes.DOUBLE
-      } else {
-        dataTypeOfAttribute
-      }
-      new CarbonLiteralExpression(value, dataType)
-    }
-
-    createFilter(predicate)
-  }
-
-
-  // Check out which filters can be pushed down to carbon, remaining can be handled in spark layer.
-  // Mostly dimension filters are only pushed down since it is faster in carbon.
-  // TODO - The Filters are first converted Intermediate sources filters expression and then these
-  // expressions are again converted back to CarbonExpression. Instead of two step process of
-  // evaluating the filters it can be merged into a single one.
-  def selectFilters(filters: Seq[Expression],
-      attrList: java.util.HashSet[AttributeReferenceWrapper],
-      aliasMap: CarbonAliasDecoderRelation): Unit = {
-    def translate(expr: Expression, or: Boolean = false): Option[sources.Filter] = {
-      expr match {
-        case or@ Or(left, right) =>
-
-          val leftFilter = translate(left, or = true)
-          val rightFilter = translate(right, or = true)
-          if (leftFilter.isDefined && rightFilter.isDefined) {
-            Some( sources.Or(leftFilter.get, rightFilter.get))
-          } else {
-            or.collect {
-              case attr: AttributeReference =>
-                attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-            None
-          }
-
-        case And(left, right) =>
-          val leftFilter = translate(left, or)
-          val rightFilter = translate(right, or)
-          if (or) {
-            if (leftFilter.isDefined && rightFilter.isDefined) {
-              (leftFilter ++ rightFilter).reduceOption(sources.And)
-            } else {
-              None
-            }
-          } else {
-            (leftFilter ++ rightFilter).reduceOption(sources.And)
-          }
-
-        case EqualTo(a: Attribute, Literal(v, t)) =>
-          Some(sources.EqualTo(a.name, v))
-        case EqualTo(l@Literal(v, t), a: Attribute) =>
-          Some(sources.EqualTo(a.name, v))
-        case Not(EqualTo(a: Attribute, Literal(v, t))) =>
-          Some(sources.Not(sources.EqualTo(a.name, v)))
-        case Not(EqualTo(Literal(v, t), a: Attribute)) =>
-          Some(sources.Not(sources.EqualTo(a.name, v)))
-        case IsNotNull(a: Attribute) =>
-          Some(sources.IsNotNull(a.name))
-        case IsNull(a: Attribute) =>
-          Some(sources.IsNull(a.name))
-        case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
-          val hSet = list.map(e => e.eval(EmptyRow))
-          Some(sources.Not(sources.In(a.name, hSet.toArray)))
-        case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
-          val hSet = list.map(e => e.eval(EmptyRow))
-          Some(sources.In(a.name, hSet.toArray))
-        case GreaterThan(a: Attribute, Literal(v, t)) =>
-          Some(sources.GreaterThan(a.name, v))
-        case GreaterThan(Literal(v, t), a: Attribute) =>
-          Some(sources.LessThan(a.name, v))
-        case LessThan(a: Attribute, Literal(v, t)) =>
-          Some(sources.LessThan(a.name, v))
-        case LessThan(Literal(v, t), a: Attribute) =>
-          Some(sources.GreaterThan(a.name, v))
-        case GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
-          Some(sources.GreaterThanOrEqual(a.name, v))
-        case GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
-          Some(sources.LessThanOrEqual(a.name, v))
-        case LessThanOrEqual(a: Attribute, Literal(v, t)) =>
-          Some(sources.LessThanOrEqual(a.name, v))
-        case LessThanOrEqual(Literal(v, t), a: Attribute) =>
-          Some(sources.GreaterThanOrEqual(a.name, v))
-        case StartsWith(a: Attribute, Literal(v, t)) =>
-          Some(sources.StringStartsWith(a.name, v.toString))
-
-        case others =>
-          if (!or) {
-            others.collect {
-              case attr: AttributeReference =>
-                attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-          }
-          None
-      }
-    }
-    filters.flatMap(translate(_, false)).toArray
-  }
-
-  def isCarbonSupportedDataTypes(expr: Expression): Boolean = {
-    expr.dataType match {
-      case StringType => true
-      case IntegerType => true
-      case LongType => true
-      case DoubleType => true
-      case FloatType => true
-      case BooleanType => true
-      case TimestampType => true
-      case ArrayType(_, _) => true
-      case StructType(_) => true
-      case DecimalType() => true
-      case _ => false
-    }
-  }
-
-  def processExpression(exprs: Seq[Expression],
-      attributesNeedToDecode: java.util.HashSet[AttributeReference],
-      unprocessedExprs: ArrayBuffer[Expression],
-      carbonTable: CarbonTable): Option[CarbonExpression] = {
-    def transformExpression(expr: Expression, or: Boolean = false): Option[CarbonExpression] = {
-      expr match {
-        case orFilter@ Or(left, right)
-          if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
-          val leftFilter = transformExpression(left, or = true)
-          val rightFilter = transformExpression(right, or = true)
-          if (leftFilter.isDefined && rightFilter.isDefined) {
-            Some(new OrExpression(leftFilter.get, rightFilter.get))
-          } else {
-            if (!or) {
-              orFilter.collect {
-                case attr: AttributeReference => attributesNeedToDecode.add(attr)
-              }
-              unprocessedExprs += orFilter
-            }
-            None
-          }
-
-        case And(left, right) if (isCarbonSupportedDataTypes(left) &&
-                                  isCarbonSupportedDataTypes(right)) =>
-          val leftFilter = transformExpression(left, or)
-          val rightFilter = transformExpression(right, or)
-          if (or) {
-            if (leftFilter.isDefined && rightFilter.isDefined) {
-              (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
-            } else {
-              None
-            }
-          } else {
-            (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
-          }
-
-
-        case EqualTo(a: Attribute, l@Literal(v, t)) if (isCarbonSupportedDataTypes(a) &&
-                                                        isCarbonSupportedDataTypes(l)) =>
-          Some(
-            new EqualToExpression(
-              transformExpression(a).get,
-              transformExpression(l).get
-            )
-          )
-        case EqualTo(l@Literal(v, t), a: Attribute) if (isCarbonSupportedDataTypes(l) &&
-                                                        isCarbonSupportedDataTypes(a)) =>
-          Some(
-            new EqualToExpression(
-              transformExpression(a).get,
-              transformExpression(l).get
-            )
-          )
-
-        case Not(EqualTo(a: Attribute, l@Literal(v, t))) if (isCarbonSupportedDataTypes(a) &&
-                                                             isCarbonSupportedDataTypes(l)) =>
-          Some(
-            new NotEqualsExpression(
-              transformExpression(a).get,
-              transformExpression(l).get
-            )
-          )
-        case Not(EqualTo(l@Literal(v, t), a: Attribute)) if (isCarbonSupportedDataTypes(l) &&
-                                                             isCarbonSupportedDataTypes(a)) =>
-          Some(
-            new NotEqualsExpression(
-              transformExpression(a).get,
-              transformExpression(l).get
-            )
-          )
-        case IsNotNull(child: Attribute) if (isCarbonSupportedDataTypes(child)) =>
-          Some(new NotEqualsExpression(transformExpression(child).get,
-            transformExpression(Literal(null)).get, true))
-        case IsNull(child: Attribute) if (isCarbonSupportedDataTypes(child)) =>
-          Some(new EqualToExpression(transformExpression(child).get,
-            transformExpression(Literal(null)).get, true))
-        case Not(In(a: Attribute, list))
-          if !list.exists(!_.isInstanceOf[Literal]) && isCarbonSupportedDataTypes(a) =>
-          if (list.exists(x => isNullLiteral(x.asInstanceOf[Literal]))) {
-            Some(new FalseExpression(transformExpression(a).get))
-          } else {
-            Some(new NotInExpression(transformExpression(a).get,
-              new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
-          }
-        case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) &&
-                                       isCarbonSupportedDataTypes(a) =>
-          Some(new InExpression(transformExpression(a).get,
-            new ListExpression(convertToJavaList(list
-              .map(transformExpression(_).get)))))
-
-        case GreaterThan(a: Attribute, l@Literal(v, t))
-          if (isCarbonSupportedDataTypes(a) && isCarbonSupportedDataTypes(l)) =>
-          Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
-        case GreaterThan(l@Literal(v, t), a: Attribute)
-          if (isCarbonSupportedDataTypes(l) && isCarbonSupportedDataTypes(a)) =>
-          Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
-
-        case LessThan(a: Attribute, l@Literal(v, t))
-          if (isCarbonSupportedDataTypes(a) && isCarbonSupportedDataTypes(l)) =>
-          Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
-        case LessThan(l@Literal(v, t), a: Attribute)
-          if (isCarbonSupportedDataTypes(l) && isCarbonSupportedDataTypes(a)) =>
-          Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
-
-        case GreaterThanOrEqual(a: Attribute, l@Literal(v, t))
-          if (isCarbonSupportedDataTypes(a) && isCarbonSupportedDataTypes(l)) =>
-          Some(new GreaterThanEqualToExpression(transformExpression(a).get,
-            transformExpression(l).get))
-        case GreaterThanOrEqual(l@Literal(v, t), a: Attribute)
-          if (isCarbonSupportedDataTypes(l) && isCarbonSupportedDataTypes(a)) =>
-          Some(new LessThanEqualToExpression(transformExpression(a).get,
-            transformExpression(l).get))
-
-        case LessThanOrEqual(a: Attribute, l@Literal(v, t))
-          if (isCarbonSupportedDataTypes(a) && isCarbonSupportedDataTypes(l)) =>
-          Some(new LessThanEqualToExpression(transformExpression(a).get,
-            transformExpression(l).get))
-        case LessThanOrEqual(l@Literal(v, t), a: Attribute)
-          if (isCarbonSupportedDataTypes(l) && isCarbonSupportedDataTypes(a)) =>
-          Some(new GreaterThanEqualToExpression(transformExpression(a).get,
-            transformExpression(l).get))
-
-        case AttributeReference(name, dataType, _, _) =>
-          Some(new CarbonColumnExpression(name,
-            CarbonScalaUtil.convertSparkToCarbonDataType(
-              getActualCarbonDataType(name, carbonTable))))
-        case Literal(name, dataType) => Some(new
-            CarbonLiteralExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(dataType)))
-        case StartsWith(left : Attribute, right@Literal(pattern, dataType)) if
-        pattern.toString.size > 0 &&
-        isCarbonSupportedDataTypes
-        (left) &&
-        isCarbonSupportedDataTypes
-        (right) =>
-          val l = new GreaterThanEqualToExpression(transformExpression(left).get,
-            transformExpression(right).get)
-          val maxValueLimit = pattern.toString.substring(0, pattern.toString.length - 1) +
-                              (pattern.toString.charAt(pattern.toString.length - 1).toInt + 1)
-                                .toChar
-          val r = new LessThanExpression(
-            transformExpression(left).get,
-            new CarbonLiteralExpression(maxValueLimit,
-              CarbonScalaUtil.convertSparkToCarbonDataType(dataType)))
-          Some(new AndExpression(l, r))
-        case others =>
-          if (!or) {
-            others.collect {
-              case attr: AttributeReference => attributesNeedToDecode.add(attr)
-            }
-            unprocessedExprs += others
-          }
-          None
-      }
-    }
-    exprs.flatMap(transformExpression(_, false)).reduceOption(new AndExpression(_, _))
-  }
-  private def isNullLiteral(exp: Expression): Boolean = {
-    if (null != exp
-        &&  exp.isInstanceOf[Literal]
-        && (exp.asInstanceOf[Literal].dataType == org.apache.spark.sql.types.DataTypes.NullType)
-        || (exp.asInstanceOf[Literal].value == null)) {
-      true
-    } else {
-      false
-    }
-  }
-  private def getActualCarbonDataType(column: String, carbonTable: CarbonTable) = {
-    var carbonColumn: CarbonColumn =
-      carbonTable.getDimensionByName(carbonTable.getFactTableName, column)
-    val dataType = if (carbonColumn != null) {
-      carbonColumn.getDataType
-    } else {
-      carbonColumn = carbonTable.getMeasureByName(carbonTable.getFactTableName, column)
-      carbonColumn.getDataType match {
-        case CarbonDataTypes.INT => CarbonDataTypes.INT
-        case CarbonDataTypes.SHORT => CarbonDataTypes.SHORT
-        case CarbonDataTypes.LONG => CarbonDataTypes.LONG
-        case CarbonDataTypes.DECIMAL => CarbonDataTypes.DECIMAL
-        case _ => CarbonDataTypes.DOUBLE
-      }
-    }
-    CarbonScalaUtil.convertCarbonToSparkDataType(dataType)
-  }
-
-  // Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is
-  // not able find the classes inside scala list and gives ClassNotFoundException.
-  private def convertToJavaList(
-      scalaList: Seq[CarbonExpression]): java.util.List[CarbonExpression] = {
-    val javaList = new java.util.ArrayList[CarbonExpression]()
-    scalaList.foreach(javaList.add)
-    javaList
-  }
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
deleted file mode 100644
index 914203f..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
+++ /dev/null
@@ -1,862 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.optimizer
-
-import java.util
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.CarbonTableIdentifierImplicit._
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate._
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.command.ProjectForUpdateCommand
-import org.apache.spark.sql.execution.RunnableCommand
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.types.{IntegerType, StringType}
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.stats.QueryStatistic
-import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory
-import org.apache.carbondata.spark.{CarbonAliasDecoderRelation, CarbonFilters}
-
-/**
- * Carbon Optimizer to add dictionary decoder.
- */
-object CarbonOptimizer {
-
-  def optimizer(optimizer: Optimizer, conf: CarbonSQLConf, version: String): Optimizer = {
-    CodeGenerateFactory.getInstance().optimizerFactory.createOptimizer(optimizer, conf)
-  }
-
-  def execute(plan: LogicalPlan, optimizer: Optimizer): LogicalPlan = {
-    val executedPlan: LogicalPlan = optimizer.execute(plan)
-    val relations = CarbonOptimizer.collectCarbonRelation(plan)
-    if (relations.nonEmpty) {
-      new ResolveCarbonFunctions(relations).apply(executedPlan)
-    } else {
-      executedPlan
-    }
-  }
-
-  // get the carbon relation from plan.
-  def collectCarbonRelation(plan: LogicalPlan): Seq[CarbonDecoderRelation] = {
-    plan collect {
-      case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
-        CarbonDecoderRelation(l.attributeMap, l.relation.asInstanceOf[CarbonDatasourceRelation])
-    }
-  }
-}
-
-/**
- * It does two jobs. 1. Change the datatype for dictionary encoded column 2. Add the dictionary
- * decoder plan.
- */
-class ResolveCarbonFunctions(relations: Seq[CarbonDecoderRelation])
-  extends Rule[LogicalPlan] with PredicateHelper {
-  val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
-  def apply(logicalPlan: LogicalPlan): LogicalPlan = {
-    if (relations.nonEmpty && !isOptimized(logicalPlan)) {
-      val plan = processPlan(logicalPlan)
-      val udfTransformedPlan = pushDownUDFToJoinLeftRelation(plan)
-      LOGGER.info("Starting to optimize plan")
-      val recorder = CarbonTimeStatisticsFactory.createExecutorRecorder("")
-      val queryStatistic = new QueryStatistic()
-      val result = transformCarbonPlan(udfTransformedPlan, relations)
-      queryStatistic.addStatistics("Time taken for Carbon Optimizer to optimize: ",
-        System.currentTimeMillis)
-      recorder.recordStatistics(queryStatistic)
-      recorder.logStatistics()
-      result
-    } else {
-      LOGGER.info("Skip CarbonOptimizer")
-      logicalPlan
-    }
-  }
-
-  private def processPlan(plan: LogicalPlan): LogicalPlan = {
-    plan transform {
-      case ProjectForUpdate(table, cols, Seq(updatePlan)) =>
-        var isTransformed = false
-        val newPlan = updatePlan transform {
-          case Project(pList, child) if (!isTransformed) =>
-            val (dest: Seq[NamedExpression], source: Seq[NamedExpression]) = pList
-              .splitAt(pList.size - cols.size)
-            val diff = cols.diff(dest.map(_.name))
-            if (diff.size > 0) {
-              sys.error(s"Unknown column(s) ${diff.mkString(",")} in table ${table.tableName}")
-            }
-            isTransformed = true
-            Project(dest.filter(a => !cols.contains(a.name)) ++ source, child)
-        }
-        ProjectForUpdateCommand(newPlan, table.tableIdentifier)
-    }
-  }
-  private def pushDownUDFToJoinLeftRelation(plan: LogicalPlan): LogicalPlan = {
-    val output = plan match {
-      case proj@Project(cols, Join(
-      left, right, jointype: org.apache.spark.sql.catalyst.plans.JoinType, condition)) =>
-        var projectionToBeAdded: Seq[org.apache.spark.sql.catalyst.expressions.Alias] = Seq.empty
-        val newCols = cols.map { col =>
-          col match {
-            case a@Alias(s: ScalaUDF, name)
-              if (name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID) ||
-                  name.equalsIgnoreCase(
-                    CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID)) =>
-              projectionToBeAdded :+= a
-              AttributeReference(name, StringType, true)().withExprId(a.exprId)
-            case other => other
-          }
-        }
-        val newLeft = left match {
-          case Project(columns, logicalPlan) =>
-            Project(columns ++ projectionToBeAdded, logicalPlan)
-          case filter: Filter =>
-            Project(filter.output ++ projectionToBeAdded, filter)
-          case other => other
-        }
-        Project(newCols, Join(newLeft, right, jointype, condition))
-      case other => other
-    }
-    output
-  }
-  def isOptimized(plan: LogicalPlan): Boolean = {
-    plan find {
-      case cd: CarbonDictionaryCatalystDecoder => true
-      case ic: InsertIntoCarbonTable => true
-      case other => false
-    } isDefined
-  }
-
-  case class ExtraNodeInfo(var hasCarbonRelation: Boolean)
-
-  def fillNodeInfo(
-      plan: LogicalPlan,
-      extraNodeInfos: java.util.HashMap[LogicalPlan, ExtraNodeInfo]): ExtraNodeInfo = {
-    plan match {
-      case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
-        val extraNodeInfo = ExtraNodeInfo(true)
-        extraNodeInfo
-      case others =>
-        val extraNodeInfo = ExtraNodeInfo(false)
-        others.children.foreach { childPlan =>
-          val childExtraNodeInfo = fillNodeInfo(childPlan, extraNodeInfos)
-          if (childExtraNodeInfo.hasCarbonRelation) {
-            extraNodeInfo.hasCarbonRelation = true
-          }
-        }
-        // only put no carbon realtion plan
-        if (!extraNodeInfo.hasCarbonRelation) {
-          extraNodeInfos.put(plan, extraNodeInfo)
-        }
-        extraNodeInfo
-    }
-  }
-
-  /**
-   * Steps for changing the plan.
-   * 1. It finds out the join condition columns and dimension aggregate columns which are need to
-   * be decoded just before that plan executes.
-   * 2. Plan starts transform by adding the decoder to the plan where it needs the decoded data
-   * like dimension aggregate columns decoder under aggregator and join condition decoder under
-   * join children.
-   */
-  def transformCarbonPlan(plan: LogicalPlan,
-      relations: Seq[CarbonDecoderRelation]): LogicalPlan = {
-    if (plan.isInstanceOf[RunnableCommand]) {
-      return plan
-    }
-    var decoder = false
-    val mapOfNonCarbonPlanNodes = new java.util.HashMap[LogicalPlan, ExtraNodeInfo]
-    fillNodeInfo(plan, mapOfNonCarbonPlanNodes)
-    val aliasMap = CarbonAliasDecoderRelation()
-    // collect alias information before hand.
-    collectInformationOnAttributes(plan, aliasMap)
-
-    def hasCarbonRelation(currentPlan: LogicalPlan): Boolean = {
-      val extraNodeInfo = mapOfNonCarbonPlanNodes.get(currentPlan)
-      if (extraNodeInfo == null) {
-        true
-      } else {
-        extraNodeInfo.hasCarbonRelation
-      }
-    }
-
-    val attrMap = new util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation]()
-    relations.foreach(_.fillAttributeMap(attrMap))
-
-    def addTempDecoder(currentPlan: LogicalPlan): LogicalPlan = {
-
-      def transformAggregateExpression(agg: Aggregate,
-          aggonGroups: util.HashSet[AttributeReferenceWrapper] = null): LogicalPlan = {
-        val attrsOndimAggs = new util.HashSet[AttributeReferenceWrapper]
-        if (aggonGroups != null) {
-          attrsOndimAggs.addAll(aggonGroups)
-        }
-        agg.aggregateExpressions.map {
-          case attr: AttributeReference =>
-          case a@Alias(attr: AttributeReference, name) =>
-          case aggExp: AggregateExpression =>
-            aggExp.transform {
-              case aggExp: AggregateExpression =>
-                collectDimensionAggregates(aggExp, attrsOndimAggs, aliasMap, attrMap)
-                aggExp
-            }
-          case others =>
-            others.collect {
-              case attr: AttributeReference
-                if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                attrsOndimAggs.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-        }
-        var child = agg.child
-        // Incase if the child also aggregate then push down decoder to child
-        if (attrsOndimAggs.size() > 0 && !child.equals(agg)) {
-          child = CarbonDictionaryTempDecoder(attrsOndimAggs,
-            new util.HashSet[AttributeReferenceWrapper](),
-            agg.child)
-        }
-        if (!decoder && aggonGroups == null) {
-          decoder = true
-          CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-            new util.HashSet[AttributeReferenceWrapper](),
-            Aggregate(agg.groupingExpressions, agg.aggregateExpressions, child),
-            isOuter = true)
-        } else {
-          Aggregate(agg.groupingExpressions, agg.aggregateExpressions, child)
-        }
-      }
-
-      currentPlan match {
-        case limit@Limit(_, child: Sort) =>
-          if (!decoder) {
-            decoder = true
-            CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-              new util.HashSet[AttributeReferenceWrapper](),
-              limit,
-              isOuter = true)
-          } else {
-            limit
-          }
-        case sort: Sort if !sort.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
-          val attrsOnSort = new util.HashSet[AttributeReferenceWrapper]()
-          sort.order.map { s =>
-            s.collect {
-              case attr: AttributeReference
-                if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                attrsOnSort.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-          }
-          var child = sort.child
-          if (attrsOnSort.size() > 0 && !child.isInstanceOf[Sort]) {
-            child = CarbonDictionaryTempDecoder(attrsOnSort,
-              new util.HashSet[AttributeReferenceWrapper](), sort.child)
-          }
-          if (!decoder) {
-            decoder = true
-            CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-              new util.HashSet[AttributeReferenceWrapper](),
-              Sort(sort.order, sort.global, child),
-              isOuter = true)
-          } else {
-            Sort(sort.order, sort.global, child)
-          }
-
-        case union: Union
-          if !(union.left.isInstanceOf[CarbonDictionaryTempDecoder] ||
-               union.right.isInstanceOf[CarbonDictionaryTempDecoder]) =>
-          val leftCondAttrs = new util.HashSet[AttributeReferenceWrapper]
-          val rightCondAttrs = new util.HashSet[AttributeReferenceWrapper]
-          val leftLocalAliasMap = CarbonAliasDecoderRelation()
-          val rightLocalAliasMap = CarbonAliasDecoderRelation()
-          // collect alias information for the child plan again. It is required as global alias
-          // may have duplicated in case of aliases
-          collectInformationOnAttributes(union.left, leftLocalAliasMap)
-          collectInformationOnAttributes(union.right, rightLocalAliasMap)
-          union.left.output.foreach { attr =>
-            if (isDictionaryEncoded(attr, attrMap, leftLocalAliasMap)) {
-              leftCondAttrs.add(AttributeReferenceWrapper(leftLocalAliasMap.getOrElse(attr, attr)))
-            }
-          }
-          union.right.output.foreach { attr =>
-            if (isDictionaryEncoded(attr, attrMap, rightLocalAliasMap)) {
-              rightCondAttrs.add(
-                AttributeReferenceWrapper(rightLocalAliasMap.getOrElse(attr, attr)))
-            }
-          }
-          var leftPlan = union.left
-          var rightPlan = union.right
-          if (hasCarbonRelation(leftPlan) && leftCondAttrs.size() > 0 &&
-              !leftPlan.isInstanceOf[CarbonDictionaryCatalystDecoder]) {
-            leftPlan = CarbonDictionaryTempDecoder(leftCondAttrs,
-              new util.HashSet[AttributeReferenceWrapper](),
-              union.left, isOuter = false, Some(leftLocalAliasMap))
-          }
-          if (hasCarbonRelation(rightPlan) && rightCondAttrs.size() > 0 &&
-              !rightPlan.isInstanceOf[CarbonDictionaryCatalystDecoder]) {
-            rightPlan = CarbonDictionaryTempDecoder(rightCondAttrs,
-              new util.HashSet[AttributeReferenceWrapper](),
-              union.right, isOuter = false, Some(rightLocalAliasMap))
-          }
-          if (!decoder) {
-            decoder = true
-            CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-              new util.HashSet[AttributeReferenceWrapper](),
-              Union(leftPlan, rightPlan),
-              isOuter = true)
-          } else {
-            Union(leftPlan, rightPlan)
-          }
-
-        case agg: Aggregate if !agg.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
-          transformAggregateExpression(agg)
-        case expand: Expand if !expand.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
-          val attrsOnExpand = new util.HashSet[AttributeReferenceWrapper]
-          expand.projections.map {s =>
-            s.map {
-              case attr: AttributeReference =>
-              case a@Alias(attr: AttributeReference, name) =>
-              case others =>
-                others.collect {
-                  case attr: AttributeReference
-                    if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                    attrsOnExpand.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-                }
-            }
-          }
-          var child = expand.child
-          if (attrsOnExpand.size() > 0 && !child.isInstanceOf[Expand]) {
-            child = CarbonDictionaryTempDecoder(attrsOnExpand,
-              new util.HashSet[AttributeReferenceWrapper](),
-              expand.child)
-          }
-          if (!decoder) {
-            decoder = true
-            CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-              new util.HashSet[AttributeReferenceWrapper](),
-              CodeGenerateFactory.getInstance().expandFactory.createExpand(expand, child),
-              isOuter = true)
-          } else {
-            CodeGenerateFactory.getInstance().expandFactory.createExpand(expand, child)
-          }
-        case filter: Filter if !filter.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
-          val attrsOnConds = new util.HashSet[AttributeReferenceWrapper]
-          // In case the child is join then we cannot push down the filters so decode them earlier
-          if (filter.child.isInstanceOf[Join] || filter.child.isInstanceOf[Sort]) {
-            filter.condition.collect {
-              case attr: AttributeReference =>
-                attrsOnConds.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-          } else {
-            CarbonFilters
-              .selectFilters(splitConjunctivePredicates(filter.condition), attrsOnConds, aliasMap)
-          }
-
-          var child = filter.child
-          if (attrsOnConds.size() > 0 && !child.isInstanceOf[Filter]) {
-            child = CarbonDictionaryTempDecoder(attrsOnConds,
-              new util.HashSet[AttributeReferenceWrapper](),
-              filter.child)
-          }
-
-          if (!decoder) {
-            decoder = true
-            CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-              new util.HashSet[AttributeReferenceWrapper](),
-              Filter(filter.condition, child),
-              isOuter = true)
-          } else {
-            Filter(filter.condition, child)
-          }
-
-        case j: Join
-          if !(j.left.isInstanceOf[CarbonDictionaryTempDecoder] ||
-               j.right.isInstanceOf[CarbonDictionaryTempDecoder]) =>
-          val attrsOnJoin = new util.HashSet[Attribute]
-          j.condition match {
-            case Some(expression) =>
-              expression.collect {
-                case attr: AttributeReference
-                  if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                  attrsOnJoin.add(aliasMap.getOrElse(attr, attr))
-              }
-            case _ =>
-          }
-
-          val leftCondAttrs = new util.HashSet[AttributeReferenceWrapper]
-          val rightCondAttrs = new util.HashSet[AttributeReferenceWrapper]
-          if (attrsOnJoin.size() > 0) {
-
-            attrsOnJoin.asScala.map { attr =>
-              if (qualifierPresence(j.left, attr)) {
-                leftCondAttrs.add(AttributeReferenceWrapper(attr))
-              }
-              if (qualifierPresence(j.right, attr)) {
-                rightCondAttrs.add(AttributeReferenceWrapper(attr))
-              }
-            }
-            var leftPlan = j.left
-            var rightPlan = j.right
-            if (leftCondAttrs.size() > 0 &&
-                !leftPlan.isInstanceOf[CarbonDictionaryCatalystDecoder]) {
-              leftPlan = leftPlan match {
-                case agg: Aggregate =>
-                  CarbonDictionaryTempDecoder(leftCondAttrs,
-                    new util.HashSet[AttributeReferenceWrapper](),
-                    transformAggregateExpression(agg, leftCondAttrs))
-                case _ =>
-                  CarbonDictionaryTempDecoder(leftCondAttrs,
-                    new util.HashSet[AttributeReferenceWrapper](),
-                    j.left)
-              }
-            }
-            if (rightCondAttrs.size() > 0 &&
-                !rightPlan.isInstanceOf[CarbonDictionaryCatalystDecoder]) {
-              rightPlan = rightPlan match {
-                case agg: Aggregate =>
-                  CarbonDictionaryTempDecoder(rightCondAttrs,
-                    new util.HashSet[AttributeReferenceWrapper](),
-                    transformAggregateExpression(agg, rightCondAttrs))
-                case _ =>
-                  CarbonDictionaryTempDecoder(rightCondAttrs,
-                    new util.HashSet[AttributeReferenceWrapper](),
-                    j.right)
-              }
-            }
-            if (!decoder) {
-              decoder = true
-              CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-                new util.HashSet[AttributeReferenceWrapper](),
-                Join(leftPlan, rightPlan, j.joinType, j.condition),
-                isOuter = true)
-            } else {
-              Join(leftPlan, rightPlan, j.joinType, j.condition)
-            }
-          } else {
-            j
-          }
-
-        case p: Project
-          if relations.nonEmpty && !p.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
-          val attrsOnProjects = new util.HashSet[AttributeReferenceWrapper]
-          p.projectList.map {
-            case attr: AttributeReference =>
-            case a@Alias(attr: AttributeReference, name) =>
-            case others =>
-              others.collect {
-                case attr: AttributeReference
-                  if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                  attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-              }
-          }
-          var child = p.child
-          if (attrsOnProjects.size() > 0 && !child.isInstanceOf[Project]) {
-            child = CarbonDictionaryTempDecoder(attrsOnProjects,
-              new util.HashSet[AttributeReferenceWrapper](),
-              p.child)
-          }
-          if (!decoder) {
-            decoder = true
-            CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-              new util.HashSet[AttributeReferenceWrapper](),
-              Project(p.projectList, child),
-              isOuter = true)
-          } else {
-            Project(p.projectList, child)
-          }
-
-        case wd: Window if !wd.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
-          val attrsOnProjects = new util.HashSet[AttributeReferenceWrapper]
-          wd.projectList.map {
-            case attr: AttributeReference =>
-            case others =>
-              others.collect {
-                case attr: AttributeReference
-                  if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                  attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-              }
-          }
-          wd.windowExpressions.map { others =>
-            others.collect {
-              case attr: AttributeReference
-                if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-          }
-          wd.partitionSpec.map{
-            case attr: AttributeReference =>
-            case others =>
-              others.collect {
-                case attr: AttributeReference
-                  if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                  attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-              }
-          }
-          wd.orderSpec.map { s =>
-            s.collect {
-              case attr: AttributeReference
-                if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-          }
-          wd.partitionSpec.map { s =>
-            s.collect {
-              case attr: AttributeReference
-                if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-                attrsOnProjects.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-          }
-          var child = wd.child
-          if (attrsOnProjects.size() > 0 && !child.isInstanceOf[Project]) {
-            child = CarbonDictionaryTempDecoder(attrsOnProjects,
-              new util.HashSet[AttributeReferenceWrapper](),
-              wd.child)
-          }
-          if (!decoder) {
-            decoder = true
-            CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-              new util.HashSet[AttributeReferenceWrapper](),
-              Window(wd.projectList, wd.windowExpressions, wd.partitionSpec, wd.orderSpec, child),
-              isOuter = true)
-          } else {
-            Window(wd.projectList, wd.windowExpressions, wd.partitionSpec, wd.orderSpec, child)
-          }
-
-        case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
-          if (!decoder) {
-            decoder = true
-            CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-              new util.HashSet[AttributeReferenceWrapper](), l, isOuter = true)
-          } else {
-            l
-          }
-
-        case others => others
-      }
-
-    }
-
-    val transFormedPlan =
-      plan transformDown {
-        case cd: CarbonDictionaryTempDecoder if cd.isOuter =>
-          decoder = true
-          cd
-        case currentPlan =>
-          if (hasCarbonRelation(currentPlan)) {
-            addTempDecoder(currentPlan)
-          } else {
-            currentPlan
-          }
-      }
-
-    val processor = new CarbonDecoderProcessor
-    processor.updateDecoders(processor.getDecoderList(transFormedPlan))
-    updateProjection(updateTempDecoder(transFormedPlan, aliasMap, attrMap))
-  }
-
-  private def updateTempDecoder(plan: LogicalPlan,
-      aliasMapOriginal: CarbonAliasDecoderRelation,
-      attrMap: java.util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation]):
-  LogicalPlan = {
-    var allAttrsNotDecode: util.Set[AttributeReferenceWrapper] =
-      new util.HashSet[AttributeReferenceWrapper]()
-    val marker = new CarbonPlanMarker
-    var aliasMap = aliasMapOriginal
-    plan transformDown {
-      case cd: CarbonDictionaryTempDecoder if !cd.processed =>
-        cd.processed = true
-        allAttrsNotDecode = cd.attrsNotDecode
-        aliasMap = cd.aliasMap.getOrElse(aliasMap)
-        marker.pushMarker(allAttrsNotDecode)
-        if (cd.isOuter) {
-          CarbonDictionaryCatalystDecoder(relations,
-            ExcludeProfile(cd.getAttrsNotDecode.asScala.toSeq),
-            aliasMap,
-            isOuter = true,
-            cd.child)
-        } else {
-          CarbonDictionaryCatalystDecoder(relations,
-            IncludeProfile(cd.getAttrList.asScala.toSeq),
-            aliasMap,
-            isOuter = false,
-            cd.child)
-        }
-      case cd: CarbonDictionaryCatalystDecoder =>
-        cd
-      case sort: Sort =>
-        val sortExprs = sort.order.map { s =>
-          s.transform {
-            case attr: AttributeReference =>
-              updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-          }.asInstanceOf[SortOrder]
-        }
-        Sort(sortExprs, sort.global, sort.child)
-      case agg: Aggregate if !agg.child.isInstanceOf[CarbonDictionaryCatalystDecoder] =>
-        val aggExps = agg.aggregateExpressions.map { aggExp =>
-          aggExp.transform {
-            case attr: AttributeReference =>
-              updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-          }
-        }.asInstanceOf[Seq[NamedExpression]]
-
-        val grpExps = agg.groupingExpressions.map { gexp =>
-          gexp.transform {
-            case attr: AttributeReference =>
-              updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-          }
-        }
-        Aggregate(grpExps, aggExps, agg.child)
-      case expand: Expand =>
-        expand.transformExpressions {
-          case attr: AttributeReference =>
-            updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-        }
-      case filter: Filter =>
-        val filterExps = filter.condition transform {
-          case attr: AttributeReference =>
-            updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-        }
-        Filter(filterExps, filter.child)
-      case j: Join =>
-        marker.pushBinaryMarker(allAttrsNotDecode)
-        j
-      case u: Union =>
-        marker.pushBinaryMarker(allAttrsNotDecode)
-        u
-      case p: Project if relations.nonEmpty =>
-        val prExps = p.projectList.map { prExp =>
-          prExp.transform {
-            case attr: AttributeReference =>
-              updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-          }
-        }.asInstanceOf[Seq[NamedExpression]]
-        Project(prExps, p.child)
-      case wd: Window if relations.nonEmpty =>
-        val prExps = wd.projectList.map { prExp =>
-          prExp.transform {
-            case attr: AttributeReference =>
-              updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-          }
-        }.asInstanceOf[Seq[Attribute]]
-        val wdExps = wd.windowExpressions.map { gexp =>
-          gexp.transform {
-            case attr: AttributeReference =>
-              updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-          }
-        }.asInstanceOf[Seq[NamedExpression]]
-        val partitionSpec = wd.partitionSpec.map{ exp =>
-          exp.transform {
-            case attr: AttributeReference =>
-              updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-          }
-        }
-        val orderSpec = wd.orderSpec.map { exp =>
-          exp.transform {
-            case attr: AttributeReference =>
-              updateDataType(attr, attrMap, allAttrsNotDecode, aliasMap)
-          }
-        }.asInstanceOf[Seq[SortOrder]]
-        Window(prExps, wdExps, partitionSpec, orderSpec, wd.child)
-
-      case l: LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceRelation] =>
-        allAttrsNotDecode = marker.revokeJoin()
-        l
-      case others => others
-    }
-  }
-
-  private def updateProjection(plan: LogicalPlan): LogicalPlan = {
-    val transFormedPlan = plan transform {
-      case p@Project(projectList: Seq[NamedExpression], cd: CarbonDictionaryCatalystDecoder) =>
-        if (cd.child.isInstanceOf[Filter] || cd.child.isInstanceOf[LogicalRelation]) {
-          Project(projectList: Seq[NamedExpression], cd.child)
-        } else {
-          p
-        }
-      case f@Filter(condition: Expression, cd: CarbonDictionaryCatalystDecoder) =>
-        if (cd.child.isInstanceOf[Project] || cd.child.isInstanceOf[LogicalRelation]) {
-          Filter(condition, cd.child)
-        } else {
-          f
-        }
-    }
-    // Remove unnecessary decoders
-    val finalPlan = transFormedPlan transform {
-      case CarbonDictionaryCatalystDecoder(_, profile, _, false, child)
-        if profile.isInstanceOf[IncludeProfile] && profile.isEmpty =>
-        child
-    }
-    val updateDtrFn = finalPlan transform {
-      case p@Project(projectList: Seq[NamedExpression], cd) =>
-        if (cd.isInstanceOf[Filter] || cd.isInstanceOf[LogicalRelation]) {
-          p.transformAllExpressions {
-            case a@Alias(exp, _)
-              if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
-              Alias(CustomDeterministicExpression(exp), a.name)(a.exprId, a.qualifiers,
-                a.explicitMetadata)
-            case exp: NamedExpression
-              if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
-              CustomDeterministicExpression(exp)
-          }
-        } else {
-          p
-        }
-      case f@Filter(condition: Expression, cd) =>
-        if (cd.isInstanceOf[Project] || cd.isInstanceOf[LogicalRelation]) {
-          f.transformAllExpressions {
-            case a@Alias(exp, _)
-              if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
-              Alias(CustomDeterministicExpression(exp), a.name)(a.exprId, a.qualifiers,
-                a.explicitMetadata)
-            case exp: NamedExpression
-              if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
-              CustomDeterministicExpression(exp)
-          }
-        } else {
-          f
-        }
-    }
-
-    updateDtrFn
-  }
-
-  private def collectInformationOnAttributes(plan: LogicalPlan,
-      aliasMap: CarbonAliasDecoderRelation) {
-    plan transformAllExpressions  {
-      case a@Alias(exp, name) =>
-        exp match {
-          case attr: Attribute => aliasMap.put(a.toAttribute, attr)
-          case _ => aliasMap.put(a.toAttribute, AttributeReference("", StringType)())
-        }
-        a
-    }
-    // collect the output of expand and add projections attributes as alias to it.
-    plan.collect {
-      case expand: Expand =>
-        expand.projections.foreach {s =>
-          s.zipWithIndex.foreach { f =>
-            f._1 match {
-              case attr: AttributeReference =>
-                aliasMap.put(expand.output(f._2).toAttribute, attr)
-              case a@Alias(attr: AttributeReference, name) =>
-                aliasMap.put(expand.output(f._2).toAttribute, attr)
-              case others =>
-            }
-          }
-        }
-    }
-  }
-
-  // Collect aggregates on dimensions so that we can add decoder to it.
-  private def collectDimensionAggregates(aggExp: AggregateExpression,
-      attrsOndimAggs: util.HashSet[AttributeReferenceWrapper],
-      aliasMap: CarbonAliasDecoderRelation,
-      attrMap: java.util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation]) {
-    aggExp collect {
-      case attr: AttributeReference if isDictionaryEncoded(attr, attrMap, aliasMap) =>
-        attrsOndimAggs.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-    }
-  }
-
-  /**
-   * Update the attribute datatype with [IntegerType] if the carbon column is encoded with
-   * dictionary.
-   *
-   */
-  private def updateDataType(attr: Attribute,
-      attrMap: java.util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation],
-      allAttrsNotDecode: java.util.Set[AttributeReferenceWrapper],
-      aliasMap: CarbonAliasDecoderRelation): Attribute = {
-    val uAttr = aliasMap.getOrElse(attr, attr)
-    val relation = Option(attrMap.get(AttributeReferenceWrapper(uAttr)))
-    if (relation.isDefined) {
-      relation.get.dictionaryMap.get(uAttr.name) match {
-        case Some(true)
-          if !allAttrsNotDecode.contains(AttributeReferenceWrapper(uAttr)) =>
-          val newAttr = AttributeReference(attr.name,
-            IntegerType,
-            attr.nullable,
-            attr.metadata)(attr.exprId, attr.qualifiers)
-          newAttr
-        case _ => attr
-      }
-    } else {
-      attr
-    }
-  }
-
-  private def isDictionaryEncoded(attr: Attribute,
-      attrMap: java.util.HashMap[AttributeReferenceWrapper, CarbonDecoderRelation],
-      aliasMap: CarbonAliasDecoderRelation): Boolean = {
-    val uAttr = aliasMap.getOrElse(attr, attr)
-    val relation = Option(attrMap.get(AttributeReferenceWrapper(uAttr)))
-    if (relation.isDefined) {
-      relation.get.dictionaryMap.get(uAttr.name) match {
-        case Some(true) => true
-        case _ => false
-      }
-    } else {
-      false
-    }
-  }
-
-  def qualifierPresence(plan: LogicalPlan, attr: Attribute): Boolean = {
-    var present = false
-    plan collect {
-      case l: LogicalRelation if l.attributeMap.contains(attr) =>
-        present = true
-    }
-    present
-  }
-}
-
-case class CarbonDecoderRelation(
-    attributeMap: AttributeMap[AttributeReference],
-    carbonRelation: CarbonDatasourceRelation) {
-
-  val extraAttrs = new ArrayBuffer[Attribute]()
-
-  def addAttribute(attr: Attribute): Unit = {
-    extraAttrs += attr
-  }
-
-  def contains(attr: Attribute): Boolean = {
-    val exists =
-      attributeMap.exists(entry => entry._1.name.equalsIgnoreCase(attr.name) &&
-                                   entry._1.exprId.equals(attr.exprId)) ||
-      extraAttrs.exists(entry => entry.name.equalsIgnoreCase(attr.name) &&
-                                 entry.exprId.equals(attr.exprId))
-    exists
-  }
-
-  def fillAttributeMap(attrMap: java.util.HashMap[AttributeReferenceWrapper,
-    CarbonDecoderRelation]): Unit = {
-    attributeMap.foreach { attr =>
-      attrMap.put(AttributeReferenceWrapper(attr._1), this)
-    }
-  }
-
-  lazy val dictionaryMap = carbonRelation.carbonRelation.metaData.dictionaryMap
-}
-
-
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala b/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
deleted file mode 100644
index bb00126..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.test
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.{CarbonContext, DataFrame, SQLContext}
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * This class is a sql executor of unit test case for spark version 1.x.
- */
-
-class SparkTestQueryExecutor extends TestQueryExecutorRegister {
-  override def sql(sqlText: String): DataFrame = SparkTestQueryExecutor.cc.sql(sqlText)
-
-  override def sqlContext: SQLContext = SparkTestQueryExecutor.cc
-
-  override def stop(): Unit = SparkTestQueryExecutor.cc.sparkContext.stop()
-}
-
-object SparkTestQueryExecutor {
-  private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-  LOGGER.info("use TestQueryExecutorImplV1")
-  CarbonProperties.getInstance()
-    .addProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
-      System.getProperty("java.io.tmpdir"))
-    .addProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL)
-    .addProperty(CarbonCommonConstants.STORE_LOCATION, TestQueryExecutor.storeLocation)
-
-  val sc = new SparkContext(new SparkConf()
-    .setAppName("CarbonSpark")
-    .setMaster("local[2]")
-    .set("spark.sql.shuffle.partitions", "20"))
-  sc.setLogLevel("ERROR")
-
-  val cc = new CarbonContext(sc, TestQueryExecutor.storeLocation, TestQueryExecutor.metastoredb)
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/util/TaskContextUtil.scala b/integration/spark/src/main/scala/org/apache/spark/util/TaskContextUtil.scala
deleted file mode 100644
index e73f78c..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/util/TaskContextUtil.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.util
-
-import org.apache.spark.TaskContext
-
-
-object TaskContextUtil {
-  def setTaskContext(context: TaskContext): Unit = {
-    val localThreadContext = TaskContext.get()
-    if (localThreadContext == null) {
-      TaskContext.setTaskContext(context)
-    }
-  }
-}
diff --git a/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
deleted file mode 100644
index d09c9b5..0000000
--- a/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++ /dev/null
@@ -1,17 +0,0 @@
-## ------------------------------------------------------------------------
-## Licensed to the Apache Software Foundation (ASF) under one or more
-## contributor license agreements.  See the NOTICE file distributed with
-## this work for additional information regarding copyright ownership.
-## The ASF licenses this file to You under the Apache License, Version 2.0
-## (the "License"); you may not use this file except in compliance with
-## the License.  You may obtain a copy of the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-## ------------------------------------------------------------------------
-org.apache.spark.sql.CarbonSource
\ No newline at end of file
diff --git a/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.test.TestQueryExecutorRegister b/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.test.TestQueryExecutorRegister
deleted file mode 100644
index fc96db4..0000000
--- a/integration/spark/src/resources/META-INF/services/org.apache.spark.sql.test.TestQueryExecutorRegister
+++ /dev/null
@@ -1,17 +0,0 @@
-## ------------------------------------------------------------------------
-## Licensed to the Apache Software Foundation (ASF) under one or more
-## contributor license agreements.  See the NOTICE file distributed with
-## this work for additional information regarding copyright ownership.
-## The ASF licenses this file to You under the Apache License, Version 2.0
-## (the "License"); you may not use this file except in compliance with
-## the License.  You may obtain a copy of the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-## ------------------------------------------------------------------------
-org.apache.spark.sql.test.SparkTestQueryExecutor
\ No newline at end of file
diff --git a/integration/spark/src/test/resources/badrecords/test2.csv b/integration/spark/src/test/resources/badrecords/test2.csv
deleted file mode 100644
index 51d25b2..0000000
--- a/integration/spark/src/test/resources/badrecords/test2.csv
+++ /dev/null
@@ -1,4 +0,0 @@
-0,569,silo
-1,843658743265874365874365874365584376547598375987,hello
-2,87436587349436587568784658743065874376,priyal
-3,56985,simple
\ No newline at end of file
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
deleted file mode 100644
index aaaf66b..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.complexType
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * Test class of creating and loading for carbon table with double
- *
- */
-class TestComplexPrimitiveTimestampDirectDictionary extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll: Unit = {
-    sql("drop table if exists complexcarbontimestamptable")
-    sql("drop table if exists complexhivetimestamptable")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss.SSS")
-    sql("CREATE TABLE complexcarbontimestamptable (empno string,workdate Timestamp,punchinout array<Timestamp>, worktime struct<begintime:Timestamp, endtime:Timestamp>, salary double) STORED BY 'org.apache.carbondata.format'")
-    sql(s"LOAD DATA local inpath '$resourcesPath/datasamplecomplex.csv' INTO TABLE complexcarbontimestamptable OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='empno,workdate,punchinout,worktime,salary')");
-    sql("CREATE TABLE complexhivetimestamptable (empno string,workdate Timestamp,punchinout array<Timestamp>, worktime struct<begintime:Timestamp, endtime:Timestamp>, salary double)row format delimited fields terminated by ',' collection items terminated by '$'")
-    sql(s"LOAD DATA local inpath '$resourcesPath/datasamplecomplex.csv' INTO TABLE complexhivetimestamptable")
-  }
-
-  test("select * query") {
-     checkAnswer(sql("select * from complexcarbontimestamptable"),
-     sql("select * from complexhivetimestamptable"))
-  }
-  
-  test("timestamp complex type in the middle of complex types") {
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss.SSS")
-    sql("CREATE TABLE testtimestampcarbon(imei string,rat array<string>, sid array<int>, end_time array<Timestamp>, probeid array<double>, contact struct<name:string, id:string>)STORED BY 'org.apache.carbondata.format'")
-    sql("LOAD DATA local inpath '" + resourcesPath + "/timestampdata.csv' INTO TABLE testtimestampcarbon options('DELIMITER'=',', 'QUOTECHAR'='\"','COMPLEX_DELIMITER_LEVEL_1'='$', 'FILEHEADER'='imei,rat,sid,end_time,probeid,contact')")
-    sql("CREATE TABLE testtimestamphive(imei string,rat array<string>, sid array<int>, end_time array<Timestamp>, probeid array<double>, contact struct<name:string, id:string>)row format delimited fields terminated by ',' collection items terminated by '$'")
-    sql(s"LOAD DATA local inpath '$resourcesPath/timestampdata.csv' INTO TABLE testtimestamphive")
-    checkAnswer(sql("select * from testtimestampcarbon"), sql("select * from testtimestamphive"))
-    sql("drop table if exists testtimestampcarbon")
-    sql("drop table if exists testtimestamphive")
-  }
-  
-  override def afterAll {
-	  sql("drop table if exists complexcarbontimestamptable")
-    sql("drop table if exists complexhivetimestamptable")
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala
deleted file mode 100644
index 98e4f18..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.dataload
-
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode}
-import org.scalatest.BeforeAndAfterAll
-
-class SparkDatasourceSuite extends QueryTest with BeforeAndAfterAll {
-
-  var df: DataFrame = _
-
-  override def beforeAll {
-    sql("DROP TABLE IF EXISTS carbon1")
-
-    import sqlContext.implicits._
-    df = sqlContext.sparkContext.parallelize(1 to 1000)
-        .map(x => ("a", "b", x))
-        .toDF("c1", "c2", "c3")
-
-    // save dataframe to carbon file
-    df.write
-        .format("carbondata")
-        .option("tableName", "carbon1")
-        .mode(SaveMode.Overwrite)
-        .save()
-  }
-
-  test("read and write using CarbonContext") {
-    val in = sqlContext.read
-        .format("carbondata")
-        .option("tableName", "carbon1")
-        .load()
-
-    assert(in.where("c3 > 500").count() == 500)
-  }
-
-  test("read and write using CarbonContext with compression") {
-    val in = sqlContext.read
-        .format("carbondata")
-        .option("tableName", "carbon1")
-        .option("compress", "true")
-        .load()
-
-    assert(in.where("c3 > 500").count() == 500)
-  }
-
-  test("test overwrite") {
-    sql("DROP TABLE IF EXISTS carbon4")
-    df.write
-        .format("carbondata")
-        .option("tableName", "carbon4")
-        .mode(SaveMode.Overwrite)
-        .save()
-    df.write
-        .format("carbondata")
-        .option("tableName", "carbon4")
-        .mode(SaveMode.Overwrite)
-        .save()
-    val in = sqlContext.read
-        .format("carbondata")
-        .option("tableName", "carbon4")
-        .load()
-    assert(in.where("c3 > 500").count() == 500)
-    sql("DROP TABLE IF EXISTS carbon4")
-  }
-
-  test("read and write using CarbonContext, multiple load") {
-    sql("DROP TABLE IF EXISTS carbon4")
-    df.write
-        .format("carbondata")
-        .option("tableName", "carbon4")
-        .mode(SaveMode.Overwrite)
-        .save()
-    df.write
-        .format("carbondata")
-        .option("tableName", "carbon4")
-        .mode(SaveMode.Append)
-        .save()
-    val in = sqlContext.read
-        .format("carbondata")
-        .option("tableName", "carbon4")
-        .load()
-    assert(in.where("c3 > 500").count() == 1000)
-    sql("DROP TABLE IF EXISTS carbon4")
-  }
-  
-  test("query using SQLContext") {
-    val newSQLContext = new SQLContext(sqlContext.sparkContext)
-    newSQLContext.sql(
-      s"""
-         | CREATE TEMPORARY TABLE temp
-         | (c1 string, c2 string, c3 int)
-         | USING org.apache.spark.sql.CarbonSource
-         | OPTIONS (path '$storeLocation/default/carbon1')
-      """.stripMargin)
-    checkAnswer(newSQLContext.sql(
-      """
-        | SELECT c1, c2, count(*)
-        | FROM temp
-        | WHERE c3 > 100
-        | GROUP BY c1, c2
-      """.stripMargin), Seq(Row("a", "b", 900)))
-    newSQLContext.dropTempTable("temp")
-  }
-
-  test("query using SQLContext without providing schema") {
-    val newSQLContext = new SQLContext(sqlContext.sparkContext)
-    newSQLContext.sql(
-      s"""
-         | CREATE TEMPORARY TABLE temp
-         | USING org.apache.spark.sql.CarbonSource
-         | OPTIONS (path '$storeLocation/default/carbon1')
-      """.stripMargin)
-    checkAnswer(newSQLContext.sql(
-      """
-        | SELECT c1, c2, count(*)
-        | FROM temp
-        | WHERE c3 > 100
-        | GROUP BY c1, c2
-      """.stripMargin), Seq(Row("a", "b", 900)))
-    newSQLContext.dropTempTable("temp")
-  }
-
-  test("query using SQLContext, multiple load") {
-    sql("DROP TABLE IF EXISTS test")
-    sql(
-      """
-        | CREATE TABLE test(id int, name string, city string, age int)
-        | STORED BY 'org.apache.carbondata.format'
-      """.stripMargin)
-    val testData = s"${resourcesPath}/sample.csv"
-    sql(s"LOAD DATA LOCAL INPATH '$testData' into table test")
-    sql(s"LOAD DATA LOCAL INPATH '$testData' into table test")
-
-    val newSQLContext = new SQLContext(sqlContext.sparkContext)
-    newSQLContext.sql(
-      s"""
-         | CREATE TEMPORARY TABLE temp
-         | (id int, name string, city string, age int)
-         | USING org.apache.spark.sql.CarbonSource
-         | OPTIONS (path '$storeLocation/default/test')
-      """.stripMargin)
-    checkAnswer(newSQLContext.sql(
-      """
-        | SELECT count(id)
-        | FROM temp
-      """.stripMargin), Seq(Row(8)))
-    newSQLContext.dropTempTable("temp")
-    sql("DROP TABLE test")
-  }
-
-  test("json data with long datatype issue CARBONDATA-405") {
-    val jsonDF = sqlContext.read.format("json").load(s"$resourcesPath/test.json")
-    jsonDF.write
-      .format("carbondata")
-      .option("tableName", "dftesttable")
-      .option("compress", "true")
-      .mode(SaveMode.Overwrite)
-      .save()
-    val carbonDF = sqlContext
-      .read
-      .format("carbondata")
-      .option("tableName", "dftesttable")
-      .load()
-    checkAnswer(
-      carbonDF.select("age", "name"),
-      jsonDF.select("age", "name"))
-    sql("drop table dftesttable")
-  }
-
-  override def afterAll {
-    sql("DROP TABLE IF EXISTS carbon1")
-    sql("DROP TABLE IF EXISTS carbon2")
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
deleted file mode 100644
index b61ecce..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.dataload
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-/**
- * Test Class for data loading when there is single quote in fact data
- *
- */
-class TestLoadDataWithSingleQuotechar extends QueryTest with BeforeAndAfterAll {
-  override def beforeAll {
-    sql("DROP TABLE IF EXISTS carbontable")
-    sql(
-      "CREATE TABLE carbontable (id Int, name String) STORED BY 'carbondata'")
-  }
-
-  test("test data loading with single quote char") {
-    try {
-      sql(
-        s"LOAD DATA LOCAL INPATH '$resourcesPath/dataWithSingleQuote.csv' INTO TABLE " +
-          "carbontable OPTIONS('DELIMITER'= ',')")
-      sql("SELECT * from carbontable")
-      checkAnswer(
-        sql("SELECT * from carbontable"),
-        Seq(Row(1,"Tom"),
-          Row(2,"Tony\n3,Lily"),
-          Row(4,"Games\""),
-          Row(5,"prival\"\n6,\"hello\"")
-        )
-      )
-    } catch {
-      case e: Throwable =>
-        assert(false)
-    }
-  }
-
-  override def afterAll {
-    sql("DROP TABLE carbontable")
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllQueriesSpark1TestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllQueriesSpark1TestCase.scala
deleted file mode 100644
index 1c003c7..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllQueriesSpark1TestCase.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.allqueries
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{Row, SaveMode}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
-  * Test Class for all query on multiple datatypes
-  *
-  */
-class AllQueriesSpark1TestCase extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll {
-    clean
-
-    sql("create table if not exists Carbon_automation_test (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY int, Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string, gamePointId int,contractNumber int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='Latest_MONTH,Latest_DAY,deviceInformationId')")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
-    sql("LOAD DATA LOCAL INPATH '" + resourcesPath + "/100_olap.csv' INTO table Carbon_automation_test options('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'= 'imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')")
-  }
-
-  def clean {
-    sql("drop table if exists Carbon_automation_test")
-  }
-  
-  override def afterAll {
-    clean
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
-  }
-
-
-  //TC_113
-  test("select percentile_approx(deviceInformationId,0.2) as a  from Carbon_automation_test")({
-    checkAnswer(
-      sql("select percentile_approx(deviceInformationId,0.2) as a  from Carbon_automation_test"),
-      Seq(Row(100005.8)))
-  })
-
-}
\ No newline at end of file
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala
deleted file mode 100644
index d762ec6..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.allqueries
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-class InsertIntoCarbonTableSpark1TestCase extends QueryTest with BeforeAndAfterAll {
-  override def beforeAll {
-    sql("drop table if exists THive")
-    sql("create table THive (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
-    sql(s"LOAD DATA local INPATH '$resourcesPath/100_olap.csv' INTO TABLE THive")
-  }
-
-
-  test("insert from carbon-select columns-source table has more column then target column") {
-    val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT)
-     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
-     
-     sql("drop table if exists load")
-     sql("drop table if exists inser")
-     sql("CREATE TABLE load(imei string,age int,task bigint,num double,level decimal(10,3),productdate timestamp,name string,point int)STORED BY 'org.apache.carbondata.format'")
-     sql("LOAD DATA INPATH '" + resourcesPath + "/shortolap.csv' INTO TABLE load options ('DELIMITER'=',', 'QUOTECHAR'='\"','FILEHEADER' = 'imei,age,task,num,level,productdate,name,point')")
-     sql("CREATE TABLE inser(imei string,age int,task bigint,num double,level decimal(10,3),productdate timestamp)STORED BY 'org.apache.carbondata.format'")
-     sql("insert into inser select * from load")
-     checkAnswer(
-         sql("select * from inser"),
-         sql("select imei,age,task,num,level,productdate from load")
-     ) 
-     sql("drop table if exists load")
-     sql("drop table if exists inser")
-     CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig)
-  }
-
-  test("insert->hive column more than carbon column->success") {
-     sql("drop table if exists TCarbon")
-     sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
-  
-     sql("insert into TCarbon select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber,device_backColor,modelId,CUPAudit,CPIClocked from THive")
-     checkAnswer(
-         sql("select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber from THive"),
-         sql("select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber from TCarbon")
-     )
-  }
-
-//  test("insert->insert empty data -pass") {
-//     sql("drop table if exists TCarbon")
-//     sql("create table TCarbon (imei string,deviceInformationId int,MAC string) STORED BY 'org.apache.carbondata.format'")
-//     sql("insert into TCarbon select imei,deviceInformationId,MAC from THive where MAC='wrongdata'")
-//     val result = sql("select imei,deviceInformationId,MAC from TCarbon where MAC='wrongdata'").collect()
-//     checkAnswer(
-//         sql("select imei,deviceInformationId,MAC from THive where MAC='wrongdata'"),
-//         sql("select imei,deviceInformationId,MAC from TCarbon where MAC='wrongdata'")
-//     )
-//  }
-
-  override def afterAll {
-    sql("drop table if exists load")
-    sql("drop table if exists inser")
-    sql("DROP TABLE IF EXISTS THive")
-    sql("DROP TABLE IF EXISTS TCarbon")
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
deleted file mode 100644
index 7aee00d..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.badrecordloger
-
-import java.io.File
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.hive.HiveContext
-import org.scalatest.BeforeAndAfterAll
-
-
-/**
- * Test Class for detailed query on timestamp datatypes
- *
- *
- */
-class BadRecordLoggerSharedDictionaryTest extends QueryTest with BeforeAndAfterAll {
-  var hiveContext: HiveContext = _
-  var csvFilePath : String = null
-  var timestamp_format: String = null
-
-  override def beforeAll {
-      sql("drop table IF EXISTS testdrive")
-    sql(
-      """create table testdrive (ID int,CUST_ID int,cust_name string)
-          STORED BY 'org.apache.carbondata.format'
-            TBLPROPERTIES("columnproperties.cust_name.shared_column"="shared.cust_name",
-            "columnproperties.ID.shared_column"="shared.ID",
-            "columnproperties.CUST_ID.shared_column"="shared.CUST_ID",
-            'DICTIONARY_INCLUDE'='ID,CUST_ID')"""
-    )
-
-      CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
-          new File("./target/test/badRecords")
-            .getCanonicalPath
-        )
-
-    val carbonProp = CarbonProperties.getInstance()
-    timestamp_format = carbonProp.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-      CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
-    carbonProp.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/mm/dd")
-    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-    csvFilePath = currentDirectory + "/src/test/resources/badrecords/test2.csv"
-
-  }
-  test("dataload with bad record test") {
-    try {
-      sql(
-        s"""LOAD DATA INPATH '$csvFilePath' INTO TABLE testdrive OPTIONS('DELIMITER'=',',
-            |'QUOTECHAR'= '"', 'BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='FAIL',
-            |'FILEHEADER'= 'ID,CUST_ID,cust_name')""".stripMargin)
-    } catch {
-      case e: Throwable =>
-        assert(e.getMessage.contains("Data load failed due to bad record"))
-    }
-  }
-
-  override def afterAll {
-    sql("drop table IF EXISTS testdrive")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timestamp_format)
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
deleted file mode 100644
index 7400839..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.createtable
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
-/**
- * Test Class for validating create table syntax for carbontable
- *
- */
-class TestCreateTableSyntax extends QueryTest with BeforeAndAfterAll {
-  
-  override def beforeAll {
-  }
-
-  test("Struct field with underscore and struct<struct> syntax check") {
-    sql("drop table if exists carbontable")
-    sql("create table carbontable(id int, username struct<sur_name:string," +
-        "actual_name:struct<first_name:string,last_name:string>>, country string, salary double)" +
-        "STORED BY 'org.apache.carbondata.format'")
-    sql("describe carbontable")
-  }
-  
-  test("Test table rename operation on carbon table and on hive table") {
-    sql("drop table if exists hivetable")
-    sql("drop table if exists carbontable")
-    sql("drop table if exists hiveRenamedTable")
-    sql("drop table if exists carbonRenamedTable")
-    sql("create table hivetable(test1 int, test2 array<String>,test3 array<bigint>,"+
-        "test4 array<int>,test5 array<decimal>,test6 array<timestamp>,test7 array<double>)"+
-        "row format delimited fields terminated by ',' collection items terminated by '$' map keys terminated by ':'")
-    sql("alter table hivetable rename To hiveRenamedTable")
-    sql("create table carbontable(test1 int, test2 array<String>,test3 array<bigint>,"+
-        "test4 array<int>,test5 array<decimal>,test6 array<timestamp>,test7 array<double>)"+
-        "STORED BY 'org.apache.carbondata.format'")
-    sql("alter table carbontable compact 'minor'")
-    try {
-      sql("alter table carbontable rename To carbonRenamedTable")
-      assert(false)
-    } catch {
-      case e : MalformedCarbonCommandException => {
-        assert(e.getMessage.equals("Unsupported alter operation on carbon table"))
-      }
-    }
-  }
-
-  
-  test("test carbon table create with complex datatype as dictionary exclude") {
-    try {
-      sql("drop table if exists carbontable")
-      sql("create table carbontable(id int, name string, dept string, mobile array<string>, "+
-          "country string, salary double) STORED BY 'org.apache.carbondata.format' " +
-          "TBLPROPERTIES('DICTIONARY_EXCLUDE'='dept,mobile')")
-      assert(false)
-    } catch {
-      case e : MalformedCarbonCommandException => {
-        assert(e.getMessage.equals("DICTIONARY_EXCLUDE is unsupported for complex datatype column: mobile"))
-      }
-    }
-  }
-
-  test("test carbon table create with double datatype as dictionary exclude") {
-    try {
-      sql("drop table if exists carbontable")
-      sql("create table carbontable(id int, name string, dept string, mobile array<string>, "+
-        "country string, salary double) STORED BY 'org.apache.carbondata.format' " +
-        "TBLPROPERTIES('DICTIONARY_EXCLUDE'='salary')")
-      assert(false)
-    } catch {
-      case e : MalformedCarbonCommandException => {
-        assert(e.getMessage.equals("DICTIONARY_EXCLUDE is unsupported for double " +
-          "data type column: salary"))
-      }
-    }
-  }
-  test("test carbon table create with int datatype as dictionary exclude") {
-    sql("drop table if exists carbontable")
-    sql("create table carbontable(id int, name string, dept string, mobile array<string>, " +
-        "country string, salary double) STORED BY 'org.apache.carbondata.format' " +
-        "TBLPROPERTIES('DICTIONARY_EXCLUDE'='id')")
-    assert(true)
-  }
-
-  test("test carbon table create with decimal datatype as dictionary exclude") {
-    try {
-      sql("drop table if exists carbontable")
-      sql("create table carbontable(id int, name string, dept string, mobile array<string>, "+
-        "country string, salary decimal) STORED BY 'org.apache.carbondata.format' " +
-        "TBLPROPERTIES('DICTIONARY_EXCLUDE'='salary')")
-      assert(false)
-    } catch {
-      case e : MalformedCarbonCommandException => {
-        assert(e.getMessage.equals("DICTIONARY_EXCLUDE is unsupported for decimal " +
-          "data type column: salary"))
-      }
-    }
-  }
-  
-  test("describe formatted on hive table and carbon table") {
-    sql("drop table if exists hivetable")
-    sql("drop table if exists carbontable")
-    sql("create table carbontable(id int, username struct<sur_name:string," +
-        "actual_name:struct<first_name:string,last_name:string>>, country string, salary double)" +
-        "STORED BY 'org.apache.carbondata.format'")
-    sql("describe formatted carbontable")
-    sql("create table hivetable(id int, username struct<sur_name:string," +
-        "actual_name:struct<first_name:string,last_name:string>>, country string, salary double)")
-    sql("describe formatted hivetable")
-  }
-
-  test("describe command carbon table for decimal scale and precision test") {
-    sql("drop table if exists carbontablePrecision")
-    sql("create table carbontablePrecision(id int, name string, dept string, mobile array<string>, "+
-        "country string, salary decimal(10,6)) STORED BY 'org.apache.carbondata.format' " +
-        "TBLPROPERTIES('DICTIONARY_INCLUDE'='salary,id')")
-    checkAnswer(
-      sql("describe carbontablePrecision"),
-      Seq(Row("country","string",""),
-        Row("dept","string",""),Row("id","int",""),Row("mobile","array<string>",""),Row("name","string",""),
-        Row("salary","decimal(10,6)","")
-      )
-    )
-  }
-  
-  test("create carbon table without dimensions") {
-    try {
-      sql("drop table if exists carbontable")
-      sql("create table carbontable(msr1 int, msr2 double, msr3 bigint, msr4 decimal)" +
-        " stored by 'org.apache.carbondata.format'")
-      assert(true)
-    } catch {
-      case e : MalformedCarbonCommandException => {
-        assert(e.getMessage.equals("Table default.carbontable can not be created without " +
-          "key columns. Please use DICTIONARY_INCLUDE or DICTIONARY_EXCLUDE to " +
-          "set at least one key column if all specified columns are numeric types"))
-      }
-    }
-  }
-
-  test("create carbon table with repeated table properties") {
-    try {
-      sql("drop table if exists carbontable")
-      sql(
-        """
-          CREATE TABLE IF NOT EXISTS carbontable
-          (ID Int, date Timestamp, country String,
-          name String, phonetype String, serialname String, salary Int)
-          STORED BY 'carbondata'
-          TBLPROPERTIES('DICTIONARY_EXCLUDE'='country','DICTIONARY_INCLUDE'='ID',
-          'DICTIONARY_EXCLUDE'='phonetype', 'DICTIONARY_INCLUDE'='salary')
-        """)
-      assert(false)
-    } catch {
-      case e : MalformedCarbonCommandException => {
-        assert(e.getMessage.equals("Table properties is repeated: dictionary_include,dictionary_exclude"))
-      }
-    }
-  }
-
-  override def afterAll {
-    sql("drop table if exists hivetable")
-    sql("drop table if exists carbontable")
-    sql("drop table if exists hiveRenamedTable")
-    sql("drop table if exists carbonRenamedTable")
-    sql("drop table if exists carbontablePrecision")
-  }
-}
\ No newline at end of file
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
deleted file mode 100644
index ab75d6e..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datacompaction
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
-  * FT for compaction scenario where major segment should not be included in minor.
-  */
-class CompactionSystemLockFeatureTest extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll {
-    CarbonProperties.getInstance().addProperty("carbon.compaction.level.threshold", "2,2")
-    sql("drop table if exists  table1")
-    sql("drop table if exists  table2")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.ENABLE_CONCURRENT_COMPACTION, "false")
-    sql(
-      "CREATE TABLE IF NOT EXISTS table1 (country String, ID Int, date Timestamp, name " +
-        "String, " +
-        "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
-        ".format'"
-    )
-    sql(
-      "CREATE TABLE IF NOT EXISTS table2 (country String, ID Int, date Timestamp, name " +
-        "String, " +
-        "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
-        ".format'"
-    )
-
-
-    val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
-
-    val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
-    val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
-    // load table1
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE table1 OPTIONS" +
-      "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE table1  OPTIONS" +
-      "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-
-    // load table2
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE table2 OPTIONS" +
-      "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE table2  OPTIONS" +
-      "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-
-    // create  a file in table 2 so that it will also be compacted.
-    val absoluteTableIdentifier = new
-        AbsoluteTableIdentifier(
-          CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
-          new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "table2", "rrr")
-        )
-    val carbonTablePath: CarbonTablePath = CarbonStorePath
-      .getCarbonTablePath(absoluteTableIdentifier.getStorePath,
-        absoluteTableIdentifier.getCarbonTableIdentifier
-      )
-
-    val file = carbonTablePath.getMetadataDirectoryPath + CarbonCommonConstants
-      .FILE_SEPARATOR + CarbonCommonConstants.majorCompactionRequiredFile
-
-    FileFactory.createNewFile(file, FileFactory.getFileType(file))
-
-    // compaction will happen here.
-    sql("alter table table1 compact 'major'"
-    )
-
-  }
-
-  /**
-    * Test whether major compaction is done for both.
-    */
-  test("check for compaction in both tables") {
-    // delete merged segments
-    sql("clean files for table table1")
-    sql("clean files for table table2")
-
-    // check for table 1.
-    val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
-        AbsoluteTableIdentifier(
-          CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
-          new CarbonTableIdentifier("default", "table1", "rrr")
-        )
-    )
-    // merged segment should not be there
-    val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
-    assert(segments.contains("0.1"))
-    assert(!segments.contains("0"))
-    assert(!segments.contains("1"))
-    // check for table 2.
-    val segmentStatusManager2: SegmentStatusManager = new SegmentStatusManager(new
-        AbsoluteTableIdentifier(
-          CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
-          new CarbonTableIdentifier("default", "table2", "rrr1")
-        )
-    )
-    // merged segment should not be there
-    val segments2 = segmentStatusManager2.getValidAndInvalidSegments.getValidSegments.asScala.toList
-    assert(segments2.contains("0.1"))
-    assert(!segments2.contains("0"))
-    assert(!segments2.contains("1"))
-
-  }
-
-  override def afterAll {
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
-    sql("drop table if exists  table1")
-    sql("drop table if exists  table2")
-  }
-
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
deleted file mode 100644
index 827b4f6..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datacompaction
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * FT for data compaction Minor threshold verification.
- */
-class DataCompactionMinorThresholdTest extends QueryTest with BeforeAndAfterAll {
-  val carbonTableIdentifier: CarbonTableIdentifier =
-    new CarbonTableIdentifier("default", "minorthreshold".toLowerCase(), "1")
-
-  val identifier = new AbsoluteTableIdentifier(storeLocation, carbonTableIdentifier)
-
-  override def beforeAll {
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "2,2")
-    sql("drop table if exists  minorthreshold")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
-    sql(
-      "CREATE TABLE IF NOT EXISTS minorthreshold (country String, ID Int, date " +
-      "Timestamp, name " +
-      "String, " +
-      "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
-      ".format'"
-    )
-
-    val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
-    val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
-    val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE minorthreshold " +
-        "OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE minorthreshold  " +
-        "OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE minorthreshold  " +
-        "OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE minorthreshold  " +
-        "OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    // compaction should happen here.
-    sql("alter table minorthreshold compact 'minor'")
-  }
-
-  /**
-   * Compaction should be completed correctly for minor compaction.
-   */
-  test("check if compaction is completed correctly for minor.") {
-
-    sql("clean files for table minorthreshold")
-
-    val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(identifier)
-    val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
-
-    assert(segments.contains("0.2"))
-    assert(!segments.contains("0.1"))
-    assert(!segments.contains("0"))
-    assert(!segments.contains("1"))
-    assert(!segments.contains("2"))
-    assert(!segments.contains("3"))
-  }
-
-  override def afterAll {
-    sql("drop table if exists  minorthreshold")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
-        CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
-  }
-
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
deleted file mode 100644
index 053ad44..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datacompaction
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
-  * FT for data compaction scenario.
-  */
-class DataCompactionNoDictionaryTest extends QueryTest with BeforeAndAfterAll {
-
-  // return segment details
-  def getSegments(databaseName : String, tableName : String, tableId : String): List[String] = {
-    val identifier = new AbsoluteTableIdentifier(
-          CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
-          new CarbonTableIdentifier(databaseName, tableName.toLowerCase , tableId))
-
-    val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(identifier)
-    segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
-  }
-
-  var csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
-  var csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
-  var csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
-  override def beforeAll {
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
-    sql("DROP TABLE IF EXISTS nodictionaryCompaction")
-    sql(
-      "CREATE TABLE nodictionaryCompaction (country String, ID Int, date Timestamp, name " +
-        "String, " +
-        "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
-        ".format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='country')"
-    )
-
-
-
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE nodictionaryCompaction " +
-        "OPTIONS('DELIMITER' = ',')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE nodictionaryCompaction " +
-        "OPTIONS('DELIMITER' = ',')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE nodictionaryCompaction " +
-        "OPTIONS('DELIMITER' = ',')"
-    )
-    // compaction will happen here.
-    sql("alter table nodictionaryCompaction compact 'major'"
-    )
-
-    // wait for compaction to finish.
-    Thread.sleep(1000)
-  }
-
-  // check for 15 seconds if the compacted segments has come or not .
-  // if not created after 15 seconds then testcase will fail.
-
-  test("check if compaction is completed or not.") {
-    var status = true
-    var noOfRetries = 0
-    while (status && noOfRetries < 10) {
-
-      val segments: List[String] = getSegments(
-        CarbonCommonConstants.DATABASE_DEFAULT_NAME, "nodictionaryCompaction", "uni21")
-
-      if (!segments.contains("0.1")) {
-        // wait for 2 seconds for compaction to complete.
-        Thread.sleep(2000)
-        noOfRetries += 1
-      }
-      else {
-        status = false
-      }
-    }
-  }
-
-  test("select country from nodictionaryCompaction") {
-    // check answers after compaction.
-    checkAnswer(
-      sql("select country from nodictionaryCompaction"),
-      Seq(Row("america"),
-        Row("canada"),
-        Row("chile"),
-        Row("china"),
-        Row("england"),
-        Row("burma"),
-        Row("butan"),
-        Row("mexico"),
-        Row("newzealand"),
-        Row("westindies"),
-        Row("china"),
-        Row("india"),
-        Row("iran"),
-        Row("iraq"),
-        Row("ireland")
-      )
-    )
-  }
-
-  test("delete merged folder and execute query") {
-    // delete merged segments
-   sql("clean files for table nodictionaryCompaction")
-
-    // merged segment should not be there
-    val segments =
-      getSegments(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "nodictionaryCompaction", "uni21")
-    assert(!segments.contains("0"))
-    assert(!segments.contains("1"))
-    assert(!segments.contains("2"))
-    assert(segments.contains("0.1"))
-
-    // now check the answers it should be same.
-    checkAnswer(
-      sql("select country from nodictionaryCompaction"),
-      Seq(Row("america"),
-        Row("canada"),
-        Row("chile"),
-        Row("china"),
-        Row("england"),
-        Row("burma"),
-        Row("butan"),
-        Row("mexico"),
-        Row("newzealand"),
-        Row("westindies"),
-        Row("china"),
-        Row("india"),
-        Row("iran"),
-        Row("iraq"),
-        Row("ireland")
-      )
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE nodictionaryCompaction " +
-        "OPTIONS('DELIMITER' = ',')"
-    )
-    sql("delete from table nodictionaryCompaction where segment.id in (0.1,3)")
-    checkAnswer(
-      sql("select country from nodictionaryCompaction"),
-      Seq()
-    )
-  }
-
-  override def afterAll {
-    sql("DROP TABLE IF EXISTS nodictionaryCompaction")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
-    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
-  }
-
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
deleted file mode 100644
index 8a0279f..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datacompaction
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
-  * FT for data compaction scenario.
-  */
-class DataCompactionTest extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll {
-    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
-    sql("drop table if exists  normalcompaction")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
-    sql(
-      "CREATE TABLE IF NOT EXISTS normalcompaction (country String, ID Int, date Timestamp, name " +
-        "String, " +
-        "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
-        ".format'"
-    )
-
-    val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
-    val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
-    val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE normalcompaction OPTIONS" +
-      "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE normalcompaction  OPTIONS" +
-      "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    // compaction will happen here.
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE normalcompaction  OPTIONS" +
-      "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    // compaction will happen here.
-    sql("alter table normalcompaction compact 'major'"
-    )
-
-  }
-
-  test("check if compaction is completed or not.") {
-    var status = true
-    var noOfRetries = 0
-    while (status && noOfRetries < 10) {
-
-      val identifier = new AbsoluteTableIdentifier(
-            CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
-            new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "normalcompaction", "1")
-          )
-
-      val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(identifier)
-
-      val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
-
-      if (!segments.contains("0.1")) {
-        // wait for 2 seconds for compaction to complete.
-        Thread.sleep(2000)
-        noOfRetries += 1
-      }
-      else {
-        status = false
-      }
-    }
-  }
-
-
-  test("select country from normalcompaction") {
-    // check answers after compaction.
-    checkAnswer(
-      sql("select country from normalcompaction"),
-      Seq(Row("america"),
-        Row("canada"),
-        Row("chile"),
-        Row("china"),
-        Row("england"),
-        Row("burma"),
-        Row("butan"),
-        Row("mexico"),
-        Row("newzealand"),
-        Row("westindies"),
-        Row("china"),
-        Row("india"),
-        Row("iran"),
-        Row("iraq"),
-        Row("ireland")
-      )
-    )
-  }
-
-  test("delete merged folder and execute query") {
-    // delete merged segments
-    sql("clean files for table normalcompaction")
-
-    val identifier = new AbsoluteTableIdentifier(
-          CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
-          new CarbonTableIdentifier(
-            CarbonCommonConstants.DATABASE_DEFAULT_NAME, "normalcompaction", "uniqueid")
-        )
-
-    val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(identifier)
-
-    // merged segment should not be there
-    val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
-    assert(!segments.contains("0"))
-    assert(!segments.contains("1"))
-    assert(!segments.contains("2"))
-    assert(segments.contains("0.1"))
-
-    // now check the answers it should be same.
-    checkAnswer(
-      sql("select country from normalcompaction"),
-      Seq(Row("america"),
-        Row("canada"),
-        Row("chile"),
-        Row("china"),
-        Row("england"),
-        Row("burma"),
-        Row("butan"),
-        Row("mexico"),
-        Row("newzealand"),
-        Row("westindies"),
-        Row("china"),
-        Row("india"),
-        Row("iran"),
-        Row("iraq"),
-        Row("ireland")
-      )
-    )
-  }
-
-
-  test("check if compaction with Updates") {
-
-    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
-    sql("drop table if exists  cardinalityUpdatetest")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
-
-    sql(
-      "CREATE TABLE IF NOT EXISTS cardinalityUpdateTest (FirstName String, LastName String, date Timestamp," +
-      "phonetype String, serialname String, ID int, salary Int) STORED BY 'org.apache.carbondata" +
-      ".format'"
-    )
-
-    val csvFilePath1 = s"$resourcesPath/compaction/compactionIUD1.csv"
-    val csvFilePath2 = s"$resourcesPath/compaction/compactionIUD2.csv"
-    val csvFilePath3 = s"$resourcesPath/compaction/compactionIUD3.csv"
-    val csvFilePath4 = s"$resourcesPath/compaction/compactionIUD4.csv"
-
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE cardinalityUpdateTest OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE cardinalityUpdateTest OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE cardinalityUpdateTest OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql("LOAD DATA LOCAL INPATH '" + csvFilePath4 + "' INTO TABLE cardinalityUpdateTest OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-
-    // update the first segment
-    sql("update cardinalityUpdateTest set (FirstName) = ('FirstTwentyOne') where ID = 2").show()
-
-    // alter table.
-    sql("alter table cardinalityUpdateTest compact 'major'")
-
-    // Verify the new updated value in compacted segment.
-    // now check the answers it should be same.
-    checkAnswer(
-      sql("select FirstName from cardinalityUpdateTest where FirstName = ('FirstTwentyOne')"),
-      Seq(Row("FirstTwentyOne")
-      )
-    )
-
-    checkAnswer(
-      sql("select count(*) from cardinalityUpdateTest where FirstName = ('FirstTwentyOne')"),
-      Seq(Row(1)
-      )
-    )
-
-    checkAnswer(
-      sql("select count(*) from cardinalityUpdateTest"),
-      Seq(Row(20)
-      )
-    )
-  }
-
-  override def afterAll {
-    sql("drop table if exists normalcompaction")
-    sql("drop table if exists cardinalityUpdatetest")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
-    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
-  }
-
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala
deleted file mode 100644
index 250ec06..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.filterexpr
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
-  * Test Class for filter expression query on String datatypes
-  *
-  */
-class GrtLtFilterProcessorTestCase extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll {
-    sql("drop table if exists a12_allnull")
-
-    sql(
-      "create table a12_allnull(empid String,ename String,sal double,deptno int,mgr string,gender" +
-        " string," +
-        "dob timestamp,comm decimal(4,2),desc string) stored by 'org.apache.carbondata.format'"
-    )
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
-    val testData = s"$resourcesPath/filter/emp2allnull.csv"
-
-    sql(
-      s"""LOAD DATA LOCAL INPATH '$testData' into table a12_allnull OPTIONS('DELIMITER'=',',
-         'QUOTECHAR'='"','FILEHEADER'='empid,ename,sal,deptno,mgr,gender,dob,comm,desc')"""
-        .stripMargin
-    )
-  }
-
-  test("In condition With improper format query regarding Null filter") {
-    checkAnswer(
-      sql("select empid from a12_allnull " + "where empid not in ('china',NULL)"),
-      Seq()
-    )
-  }
-
-  override def afterAll {
-    sql("drop table if exists a12_allnull")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala
deleted file mode 100644
index 1649875..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.hadooprelation
-
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-/**
- * Test Class for hadoop fs relation
- *
- */
-class HadoopFSRelationTestCase extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll {
-    sql("drop table if exists hadoopfsrelation")
-    sql("drop table if exists hadoopfsrelation_hive")
-    sql(
-      "CREATE TABLE hadoopfsrelation (empno int, empname String, designation " +
-      "String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno " +
-      "int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate " +
-      "Timestamp,attendance int,utilization int,salary int)" +
-      "STORED BY 'org.apache.carbondata.format'")
-    sql(
-      s"LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE hadoopfsrelation " +
-      "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')")
-    
-    sql("CREATE TABLE hadoopfsrelation_hive (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int)row format delimited fields terminated by ','")
-    
-    sql(
-      s"LOAD DATA local inpath '$resourcesPath/datawithoutheader.csv' INTO TABLE hadoopfsrelation_hive ")
-  }
-
-  test("hadoopfsrelation select all test") {
-    val rdd = sqlContext.read.format("org.apache.spark.sql.CarbonSource")
-      .option("tableName", "hadoopfsrelation").load()
-    assert(rdd.collect().length > 0)
-  }
-
-  test("hadoopfsrelation filters test") {
-    val rdd: DataFrame = sqlContext.read.format("org.apache.spark.sql.CarbonSource")
-      .option("tableName", "hadoopfsrelation").load()
-      .select("empno", "empname", "utilization").where("empname in ('arvind','ayushi')")
-    checkAnswer(
-      rdd,
-      sql("select empno,empname,utilization from hadoopfsrelation_hive where empname in ('arvind','ayushi')"))
-  }
-
-  override def afterAll {
-    sql("drop table hadoopfsrelation")
-    sql("drop table hadoopfsrelation_hive")
-  }
-}
\ No newline at end of file
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
deleted file mode 100644
index b5b8261..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.util
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
-import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-
-/**
-  * Test Case for org.apache.carbondata.integration.spark.util.GlobalDictionaryUtil
-  */
-class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
-
-  var pwd: String = _
-  var sampleRelation: CarbonRelation = _
-  var complexRelation: CarbonRelation = _
-  var sampleAllDictionaryFile: String = _
-  var complexAllDictionaryFile: String = _
-
-  def buildCarbonLoadModel(relation: CarbonRelation,
-    filePath: String,
-    header: String,
-    allDictFilePath: String): CarbonLoadModel = {
-    val carbonLoadModel = new CarbonLoadModel
-    carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
-    carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
-    val table = relation.tableMeta.carbonTable
-    val carbonSchema = new CarbonDataLoadSchema(table)
-    carbonLoadModel.setDatabaseName(table.getDatabaseName)
-    carbonLoadModel.setTableName(table.getFactTableName)
-    carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
-    carbonLoadModel.setFactFilePath(filePath)
-    carbonLoadModel.setCsvHeader(header)
-    carbonLoadModel.setCsvDelimiter(",")
-    carbonLoadModel.setComplexDelimiterLevel1("\\$")
-    carbonLoadModel.setComplexDelimiterLevel2("\\:")
-    carbonLoadModel.setAllDictPath(allDictFilePath)
-    carbonLoadModel.setSerializationNullFormat(
-          TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
-    carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
-      CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-      CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
-    carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
-    carbonLoadModel
-  }
-
-  override def beforeAll {
-    sql("drop table if exists sample")
-    sql("drop table if exists complextypes")
-    buildTestData
-    // second time comment this line
-    buildTable
-    buildRelation
-  }
-
-  def buildTestData() = {
-    sampleAllDictionaryFile = s"${resourcesPath}/alldictionary/sample/20160423/1400_1405/*.dictionary"
-    complexAllDictionaryFile = s"${resourcesPath}/alldictionary/complex/20160423/1400_1405/*.dictionary"
-  }
-
-  def buildTable() = {
-    try {
-      sql(
-        "CREATE TABLE IF NOT EXISTS sample (id STRING, name STRING, city STRING, " +
-          "age INT) STORED BY 'org.apache.carbondata.format' " +
-        "tblproperties('dictionary_include'='city')"
-      )
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-    try {
-      sql(
-        "create table complextypes (deviceInformationId string, channelsId string, " +
-          "ROMSize string, purchasedate string, mobile struct<imei: string, imsi: string>, MAC " +
-          "array<string>, locationinfo array<struct<ActiveAreaId: INT, ActiveCountry: string, " +
-          "ActiveProvince: string, Activecity: string, ActiveDistrict: string, ActiveStreet: " +
-          "string>>, proddate struct<productionDate: string,activeDeactivedate: array<string>>, " +
-          "gamePointId INT,contractNumber INT) STORED BY 'org.apache.carbondata.format'" +
-          "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ROMSize', 'dictionary_include'='channelsId')"
-      )
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-  }
-
-  def buildRelation() = {
-    val catalog = CarbonEnv.get.carbonMetastore
-    sampleRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
-      "sample")(sqlContext).asInstanceOf[CarbonRelation]
-    complexRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
-      "complextypes")(sqlContext).asInstanceOf[CarbonRelation]
-  }
-
-  test("Support generate global dictionary from all dictionary files") {
-    val header = "id,name,city,age"
-    val carbonLoadModel = buildCarbonLoadModel(sampleRelation, null, header, sampleAllDictionaryFile)
-    GlobalDictionaryUtil
-      .generateGlobalDictionary(sqlContext,
-        carbonLoadModel,
-        sampleRelation.tableMeta.storePath)
-
-    DictionaryTestCaseUtil.
-      checkDictionary(sampleRelation, "city", "shenzhen")
-  }
-
-  test("Support generate global dictionary from all dictionary files for complex type") {
-    val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber"
-    val carbonLoadModel = buildCarbonLoadModel(complexRelation, null, header, complexAllDictionaryFile)
-    GlobalDictionaryUtil
-      .generateGlobalDictionary(sqlContext,
-      carbonLoadModel,
-      complexRelation.tableMeta.storePath)
-
-    DictionaryTestCaseUtil.
-      checkDictionary(complexRelation, "channelsId", "1650")
-  }
-  
-  override def afterAll {
-    sql("drop table sample")
-    sql("drop table complextypes")
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
deleted file mode 100644
index fd2ab2a..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.util
-
-import org.apache.spark.sql.CarbonRelation
-import org.apache.spark.sql.test.TestQueryExecutor
-
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.util.CarbonLoaderUtil
-
-/**
- * Utility for global dictionary test cases
- */
-object DictionaryTestCaseUtil {
-
-  /**
-   *  check whether the dictionary of specified column generated
-   * @param relation  carbon table relation
-   * @param columnName  name of specified column
-   * @param value  a value of column
-   */
-  def checkDictionary(relation: CarbonRelation, columnName: String, value: String) {
-    val table = relation.tableMeta.carbonTable
-    val dimension = table.getDimensionByName(table.getFactTableName, columnName)
-    val tableIdentifier = new CarbonTableIdentifier(table.getDatabaseName, table.getFactTableName, "uniqueid")
-    val columnIdentifier = new DictionaryColumnUniqueIdentifier(tableIdentifier,
-      dimension.getColumnIdentifier, dimension.getDataType,
-      CarbonStorePath.getCarbonTablePath(table.getStorePath, table.getCarbonTableIdentifier)
-    )
-    val dict = CarbonLoaderUtil.getDictionary(columnIdentifier, TestQueryExecutor.storeLocation)
-    assert(dict.getSurrogateKey(value) != CarbonCommonConstants.INVALID_SURROGATE_KEY)
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
deleted file mode 100644
index f0de47d..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
-  * Licensed to the Apache Software Foundation (ASF) under one
-  * or more contributor license agreements.  See the NOTICE file
-  * distributed with this work for additional information
-  * regarding copyright ownership.  The ASF licenses this file
-  * to you under the Apache License, Version 2.0 (the
-  * "License"); you may not use this file except in compliance
-  * with the License.  You may obtain a copy of the License at
-  *
-  *    http://www.apache.org/licenses/LICENSE-2.0
-  *
-  * Unless required by applicable law or agreed to in writing,
-  * software distributed under the License is distributed on an
-  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  * KIND, either express or implied.  See the License for the
-  * specific language governing permissions and limitations
-  * under the License.
-  */
-package org.apache.carbondata.spark.util
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.processing.exception.DataLoadingException
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
-  /**
- * test case for external column dictionary generation
-  * also support complicated type
-  */
-class ExternalColumnDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
-
-  var extComplexRelation: CarbonRelation = _
-  var verticalDelimiteRelation: CarbonRelation = _
-  var loadSqlRelation: CarbonRelation = _
-  var filePath: String = _
-  var pwd: String = _
-  var complexFilePath1: String = _
-  var complexFilePath2: String = _
-  var extColDictFilePath1: String = _
-  var extColDictFilePath2: String = _
-  var extColDictFilePath3: String = _
-  var header: String = _
-  var header2: String = _
-
-  def buildTestData() = {
-
-    filePath = s"${resourcesPath}/sample.csv"
-    complexFilePath1 = s"${resourcesPath}/complexdata2.csv"
-    complexFilePath2 = s"${resourcesPath}/verticalDelimitedData.csv"
-    extColDictFilePath1 = s"deviceInformationId:${resourcesPath}/deviceInformationId.csv," +
-      s"mobile.imei:${resourcesPath}/mobileimei.csv," +
-      s"mac:${resourcesPath}/mac.csv," +
-      s"locationInfo.ActiveCountry:${resourcesPath}/locationInfoActiveCountry.csv"
-    extColDictFilePath2 = s"deviceInformationId:${resourcesPath}/deviceInformationId2.csv"
-    extColDictFilePath3 = s"channelsId:${resourcesPath}/channelsId.csv"
-    header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber"
-    header2 = "deviceInformationId,channelsId,contractNumber"
-  }
-
-  def buildTable() = {
-    try {
-      sql("""CREATE TABLE extComplextypes (deviceInformationId int,
-     channelsId string, ROMSize string, purchasedate string,
-     mobile struct<imei:string, imsi:string>, MAC array<string>,
-     locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
-     ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
-     proddate struct<productionDate:string,activeDeactivedate:array<string>>,
-     gamePointId double,contractNumber double)
-     STORED BY 'org.apache.carbondata.format'
-     TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId, channelsId')
-      """)
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-
-    try {
-      sql("""CREATE TABLE verticalDelimitedTable (deviceInformationId int,
-     channelsId string,contractNumber double)
-     STORED BY 'org.apache.carbondata.format'
-     TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId, channelsId')
-      """)
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-
-    try {
-      sql("""CREATE TABLE loadSqlTest (deviceInformationId int,
-     channelsId string, ROMSize string, purchasedate string,
-     mobile struct<imei:string, imsi:string>, MAC array<string>,
-     locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
-     ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
-     proddate struct<productionDate:string,activeDeactivedate:array<string>>,
-     gamePointId double,contractNumber double)
-     STORED BY 'org.apache.carbondata.format'
-     TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId')
-      """)
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-  }
-
-  def buildRelation() = {
-    val catalog = CarbonEnv.get.carbonMetastore
-    extComplexRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
-      "extComplextypes")(sqlContext)
-      .asInstanceOf[CarbonRelation]
-    verticalDelimiteRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
-      "verticalDelimitedTable")(sqlContext)
-      .asInstanceOf[CarbonRelation]
-    loadSqlRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
-      "loadSqlTest")(sqlContext)
-      .asInstanceOf[CarbonRelation]
-  }
-
-  def buildCarbonLoadModel(relation: CarbonRelation,
-      filePath:String,
-      header: String,
-      extColFilePath: String,
-      csvDelimiter: String = ","): CarbonLoadModel = {
-    val carbonLoadModel = new CarbonLoadModel
-    carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
-    carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
-    val table = relation.tableMeta.carbonTable
-    val carbonSchema = new CarbonDataLoadSchema(table)
-    carbonLoadModel.setDatabaseName(table.getDatabaseName)
-    carbonLoadModel.setTableName(table.getFactTableName)
-    carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
-    carbonLoadModel.setFactFilePath(filePath)
-    carbonLoadModel.setCsvHeader(header)
-    carbonLoadModel.setCsvDelimiter(csvDelimiter)
-    carbonLoadModel.setComplexDelimiterLevel1("\\$")
-    carbonLoadModel.setComplexDelimiterLevel2("\\:")
-    carbonLoadModel.setColDictFilePath(extColFilePath)
-    carbonLoadModel.setQuoteChar("\"");
-    carbonLoadModel.setSerializationNullFormat(
-      TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
-    carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
-      CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-      CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
-    carbonLoadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(
-      CarbonCommonConstants.CARBON_DATE_FORMAT,
-      CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))  
-    carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
-    carbonLoadModel.setMaxColumns("100")
-    carbonLoadModel
-  }
-
-  override def beforeAll {
-    buildTestData
-    buildTable
-    buildRelation
-  }
-
-  test("Generate global dictionary from external column file") {
-    // load the first time
-    var carbonLoadModel = buildCarbonLoadModel(extComplexRelation, complexFilePath1,
-      header, extColDictFilePath1)
-    GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
-      extComplexRelation.tableMeta.storePath)
-    // check whether the dictionary is generated
-    DictionaryTestCaseUtil.checkDictionary(
-      extComplexRelation, "deviceInformationId", "10086")
-
-    // load the second time
-    carbonLoadModel = buildCarbonLoadModel(extComplexRelation, complexFilePath1,
-      header, extColDictFilePath2)
-    GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
-      extComplexRelation.tableMeta.storePath)
-    // check the old dictionary and whether the new distinct value is generated
-    DictionaryTestCaseUtil.checkDictionary(
-      extComplexRelation, "deviceInformationId", "10086")
-    DictionaryTestCaseUtil.checkDictionary(
-      extComplexRelation, "deviceInformationId", "10011")
-  }
-
-  test("When csv delimiter is not comma") {
-    //  when csv delimiter is comma
-    var carbonLoadModel = buildCarbonLoadModel(extComplexRelation, complexFilePath1,
-      header, extColDictFilePath3)
-    GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
-      extComplexRelation.tableMeta.storePath)
-    // check whether the dictionary is generated
-    DictionaryTestCaseUtil.checkDictionary(
-      extComplexRelation, "channelsId", "1421|")
-
-    //  when csv delimiter is not comma
-    carbonLoadModel = buildCarbonLoadModel(verticalDelimiteRelation, complexFilePath2,
-      header2, extColDictFilePath3, "|")
-    GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
-      verticalDelimiteRelation.tableMeta.storePath)
-    // check whether the dictionary is generated
-    DictionaryTestCaseUtil.checkDictionary(
-      verticalDelimiteRelation, "channelsId", "1431,")
-  }
-
-  test("LOAD DML with COLUMNDICT option") {
-    try {
-      sql(s"""
-      LOAD DATA LOCAL INPATH "$complexFilePath1" INTO TABLE loadSqlTest
-      OPTIONS('single_pass'='true','FILEHEADER'='$header', 'COLUMNDICT'='$extColDictFilePath1')
-        """)
-    } catch {
-      case ex: Exception =>
-        LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-        assert(false)
-    }
-    DictionaryTestCaseUtil.checkDictionary(
-      loadSqlRelation, "deviceInformationId", "10086")
-  }
-
-  test("COLUMNDICT and ALL_DICTIONARY_PATH can not be used together") {
-    try {
-      sql(s"""
-        LOAD DATA LOCAL INPATH "$complexFilePath1" INTO TABLE loadSqlTest
-        OPTIONS('COLUMNDICT'='$extColDictFilePath1',"ALL_DICTIONARY_PATH"='$extColDictFilePath1')
-        """)
-      assert(false)
-    } catch {
-      case ex: MalformedCarbonCommandException =>
-        assertResult(ex.getMessage)("Error: COLUMNDICT and ALL_DICTIONARY_PATH can not be used together " +
-          "in options")
-      case _: Throwable => assert(false)
-    }
-  }
-
-  test("Measure can not use COLUMNDICT") {
-    try {
-      sql(s"""
-      LOAD DATA LOCAL INPATH "$complexFilePath1" INTO TABLE loadSqlTest
-      OPTIONS('single_pass'='true','FILEHEADER'='$header', 'COLUMNDICT'='gamePointId:$filePath')
-      """)
-      assert(false)
-    } catch {
-      case ex: DataLoadingException =>
-        assertResult(ex.getMessage)("Column gamePointId is not a key column. Only key column can be part " +
-          "of dictionary and used in COLUMNDICT option.")
-      case _: Throwable => assert(false)
-    }
-  }
-
-  override def afterAll: Unit = {
-    sql("DROP TABLE extComplextypes")
-    sql("DROP TABLE verticalDelimitedTable")
-    sql("DROP TABLE loadSqlTest")
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
deleted file mode 100644
index d1d31c1..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.util
-
-import java.util.concurrent.{Callable, Executors}
-
-import scala.collection.mutable.ListBuffer
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
-import org.apache.carbondata.core.service.impl.PathFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-
-class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAfterAll {
-
-  var sampleRelation: CarbonRelation = _
-
-  def buildCarbonLoadModel(relation: CarbonRelation,
-                           filePath: String,
-                           header: String): CarbonLoadModel = {
-    val carbonLoadModel = new CarbonLoadModel
-    carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
-    carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
-    // carbonLoadModel.setSchema(relation.cubeMeta.schema)
-    val table = relation.tableMeta.carbonTable
-    val carbonSchema = new CarbonDataLoadSchema(table)
-    carbonLoadModel.setDatabaseName(table.getDatabaseName)
-    carbonLoadModel.setTableName(table.getFactTableName)
-    carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
-    carbonLoadModel.setFactFilePath(filePath)
-    carbonLoadModel.setCsvHeader(header)
-    carbonLoadModel.setCsvDelimiter(",")
-    carbonLoadModel.setComplexDelimiterLevel1("\\$")
-    carbonLoadModel.setComplexDelimiterLevel2("\\:")
-    carbonLoadModel.setStorePath(relation.tableMeta.storePath)
-    carbonLoadModel.setQuoteChar("\"")
-    carbonLoadModel.setSerializationNullFormat(
-      TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
-    carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
-      CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-      CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
-    carbonLoadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(
-      CarbonCommonConstants.CARBON_DATE_FORMAT,
-      CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))  
-    carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
-    carbonLoadModel.setMaxColumns("2000")
-    carbonLoadModel
-  }
-
-  override def beforeAll {
-    // second time comment this line
-    buildTable
-    buildRelation
-  }
-
-  def buildTable() = {
-    try {
-      sql(
-        "CREATE TABLE IF NOT EXISTS employee (empid STRING) STORED BY 'org.apache.carbondata.format' " +
-          "tblproperties('dictionary_include'='empid')")
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-  }
-
-  def buildRelation() = {
-    val catalog = CarbonEnv.get.carbonMetastore
-    sampleRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
-      "employee")(sqlContext)
-      .asInstanceOf[CarbonRelation]
-  }
-
-  def writedummydata(filePath: String, recCount: Int) = {
-    var a: Int = 0
-    var records: StringBuilder = StringBuilder.newBuilder
-    for (a <- 0 to recCount) {
-      records.append(a).append("\n")
-    }
-    val dis = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath))
-    dis.writeBytes(records.toString())
-    dis.close()
-  }
-
-  test("concurrent dictionary generation") {
-    CarbonProperties.getInstance.addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "-1")
-    val noOfFiles = 5
-    val files = new ListBuffer[String]()
-    val loadModels = new ListBuffer[CarbonLoadModel]()
-    for (i <- 0 until noOfFiles) {
-      val filePath: String = s"${integrationPath}/spark/target/singlecolumn_${10 * (i + 1)}.csv"
-      files += filePath
-      loadModels += buildCarbonLoadModel(sampleRelation, filePath, "empid")
-      writedummydata(filePath, 10 * (i + 1))
-    }
-    try {
-      val dictGenerators = new java.util.ArrayList[Callable[String]](noOfFiles)
-      for (i <- 0 until noOfFiles) {
-        dictGenerators.add(new DictGenerator(loadModels(i)))
-      }
-      val executorService = Executors.newFixedThreadPool(10)
-      val results = executorService.invokeAll(dictGenerators)
-      for (i <- 0 until noOfFiles) {
-        val res = results.get(i).get
-        assert("Pass".equals(res))
-      }
-    } catch {
-      case ex: Exception =>
-        ex.printStackTrace()
-        assert(false)
-    }
-    val carbonTableIdentifier = sampleRelation.tableMeta.carbonTable.getCarbonTableIdentifier
-    val columnIdentifier = sampleRelation.tableMeta.carbonTable.getDimensionByName("employee", "empid").getColumnIdentifier
-    val dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(
-      carbonTableIdentifier,
-      columnIdentifier,
-      columnIdentifier.getDataType,
-      CarbonStorePath.getCarbonTablePath(storeLocation, carbonTableIdentifier))
-    val carbonTablePath = PathFactory.getInstance()
-        .getCarbonTablePath(sampleRelation.tableMeta.storePath, carbonTableIdentifier, dictionaryColumnUniqueIdentifier)
-    val dictPath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId)
-    val dictFile = FileFactory.getCarbonFile(dictPath, FileFactory.getFileType(dictPath))
-    val offSet = dictFile.getSize
-    val sortIndexPath = carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId, offSet)
-    val sortIndexFile = FileFactory.getCarbonFile(sortIndexPath, FileFactory.getFileType(sortIndexPath))
-    assert(sortIndexFile.exists())
-    val sortIndexFiles = carbonTablePath.getSortIndexFiles(sortIndexFile.getParentFile, columnIdentifier.getColumnId)
-    assert(sortIndexFiles.length >= 1)
-    deleteFiles(files)
-  }
-
-  def deleteFiles(files: ListBuffer[String]) {
-    for (i <- 0 until files.length) {
-      val file = FileFactory.getCarbonFile(files(i), FileFactory.getFileType(files(i)))
-      file.delete()
-    }
-  }
-
-  override def afterAll {
-    sql("drop table if exists employee")
-    CarbonProperties.getInstance.addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME,
-        Integer.toString(CarbonCommonConstants.DEFAULT_MAX_QUERY_EXECUTION_TIME))
-  }
-
-  class DictGenerator(loadModel: CarbonLoadModel) extends Callable[String] {
-   override def call:String = {
-     var result = "Pass"
-      try {
-        GlobalDictionaryUtil
-          .generateGlobalDictionary(sqlContext,
-            loadModel,
-            sampleRelation.tableMeta.storePath)
-      } catch {
-        case ex: Exception => 
-          result = ex.getMessage
-          ex.printStackTrace()
-      }
-      result
-    }
-  }
-}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
deleted file mode 100644
index 8864d3e..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.util
-
-import java.io.File
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-
-/**
-  * Test Case for org.apache.carbondata.spark.util.GlobalDictionaryUtil
-  */
-class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
-
-  var sampleRelation: CarbonRelation = _
-  var dimSampleRelation: CarbonRelation = _
-  var complexRelation: CarbonRelation = _
-  var incrementalLoadTableRelation: CarbonRelation = _
-  var filePath: String = _
-  var dimFilePath: String = _
-  var complexfilePath: String = _
-  var complexfilePath1: String = _
-  var complexfilePath2: String = _
-
-  def buildCarbonLoadModel(relation: CarbonRelation,
-    filePath: String,
-    header: String): CarbonLoadModel = {
-    val carbonLoadModel = new CarbonLoadModel
-    carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
-    carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
-    // carbonLoadModel.setSchema(relation.tableMeta.schema)
-    val table = relation.tableMeta.carbonTable
-    val carbonSchema = new CarbonDataLoadSchema(table)
-    carbonLoadModel.setDatabaseName(table.getDatabaseName)
-    carbonLoadModel.setTableName(table.getFactTableName)
-    carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
-    carbonLoadModel.setFactFilePath(filePath)
-    carbonLoadModel.setCsvHeader(header)
-    carbonLoadModel.setCsvDelimiter(",")
-    carbonLoadModel.setComplexDelimiterLevel1("\\$")
-    carbonLoadModel.setComplexDelimiterLevel2("\\:")
-    carbonLoadModel.setStorePath(relation.tableMeta.storePath)
-    carbonLoadModel.setQuoteChar("\"")
-    carbonLoadModel.setSerializationNullFormat(
-      TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
-    carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
-      CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-      CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
-    carbonLoadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(
-      CarbonCommonConstants.CARBON_DATE_FORMAT,
-      CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))  
-    carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
-    carbonLoadModel.setMaxColumns("2000")
-    carbonLoadModel
-  }
-
-  override def beforeAll {
-    buildTestData
-    // second time comment this line
-    buildTable
-    buildRelation
-  }
-
-  def buildTestData() = {
-    filePath = s"${resourcesPath}/sample.csv"
-    dimFilePath = s"dimTableSample:${resourcesPath}/dimTableSample.csv"
-    complexfilePath1 = s"${resourcesPath}/complexdata1.csv"
-    complexfilePath2 = s"${resourcesPath}/complexdata2.csv"
-    complexfilePath = s"${resourcesPath}/complexdata.csv"
-  }
-
-  def buildTable() = {
-    try {
-      sql(
-        "CREATE TABLE IF NOT EXISTS sample (id STRING, name STRING, city STRING, " +
-          "age INT) STORED BY 'org.apache.carbondata.format'"
-      )
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-    try {
-      sql(
-        "CREATE TABLE IF NOT EXISTS dimSample (id STRING, name STRING, city STRING, " +
-          "age INT) STORED BY 'org.apache.carbondata.format'" +
-        "TBLPROPERTIES('DICTIONARY_EXCLUDE'='id,name')"
-      )
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-    try {
-      sql(
-        "create table complextypes (deviceInformationId INT, channelsId string, " +
-          "ROMSize string, purchasedate string, mobile struct<imei: string, imsi: string>, MAC " +
-          "array<string>, locationinfo array<struct<ActiveAreaId: INT, ActiveCountry: string, " +
-          "ActiveProvince: string, Activecity: string, ActiveDistrict: string, ActiveStreet: " +
-          "string>>, proddate struct<productionDate: string,activeDeactivedate: array<string>>, " +
-          "gamePointId INT,contractNumber INT) STORED BY 'org.apache.carbondata.format'" +
-          "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ROMSize')"
-
-      )
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-
-    try {
-      sql(
-        "create table incrementalLoadTable (deviceInformationId INT, channelsId " +
-          "string, ROMSize string, purchasedate string, mobile struct<imei: string, imsi: string>, " +
-          "MAC array<string>, locationinfo array<struct<ActiveAreaId: INT, ActiveCountry: " +
-          "string, ActiveProvince: string, Activecity: string, ActiveDistrict: string, ActiveStreet: " +
-          "string>>, proddate struct<productionDate: string,activeDeactivedate: array<string>>, " +
-          "gamePointId INT,contractNumber INT) STORED BY 'org.apache.carbondata.format'"+
-          "TBLPROPERTIES('DICTIONARY_INCLUDE'='deviceInformationId')"
-      )
-    } catch {
-      case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
-    }
-  }
-
-  def buildRelation() = {
-    val catalog = CarbonEnv.get.carbonMetastore
-    sampleRelation = catalog.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
-      "sample")(sqlContext)
-      .asInstanceOf[CarbonRelation]
-    dimSampleRelation = catalog
-      .lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME), "dimSample")(sqlContext)
-      .asInstanceOf[CarbonRelation]
-    complexRelation = catalog
-      .lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME), "complextypes")(sqlContext)
-      .asInstanceOf[CarbonRelation]
-    incrementalLoadTableRelation = catalog
-      .lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME), "incrementalLoadTable")(sqlContext)
-      .asInstanceOf[CarbonRelation]
-  }
-
-  test("[issue-80]Global Dictionary Generation") {
-
-    val carbonLoadModel = buildCarbonLoadModel(sampleRelation, filePath, null)
-    GlobalDictionaryUtil
-      .generateGlobalDictionary(sqlContext, carbonLoadModel,
-        sampleRelation.tableMeta.storePath
-      )
-
-    // test for dimension table
-    // TODO - Need to fill and send the dimension table data as per new DimensionRelation in
-    // CarbonDataLoadModel
-    // carbonLoadModel = buildCarbonLoadModel(dimSampleRelation, filePath, dimFilePath, null)
-    // GlobalDictionaryUtil.generateGlobalDictionary(CarbonHiveContext, carbonLoadModel,
-    // dimSampleRelation.tableMeta.dataPath, false)
-  }
-
-  test("[Issue-190]load csv file without header And support complex type") {
-    val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo," +
-      "proddate,gamePointId,contractNumber"
-    val carbonLoadModel = buildCarbonLoadModel(complexRelation, complexfilePath, header)
-    GlobalDictionaryUtil
-      .generateGlobalDictionary(sqlContext, carbonLoadModel,
-        complexRelation.tableMeta.storePath
-      )
-  }
-
-  test("[Issue-232]Issue in incremental data load for dictionary generation") {
-    val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo," +
-      "proddate,gamePointId,contractNumber"
-    // load 1
-    var carbonLoadModel = buildCarbonLoadModel(incrementalLoadTableRelation,
-      complexfilePath1,
-      header
-    )
-    GlobalDictionaryUtil
-      .generateGlobalDictionary(sqlContext, carbonLoadModel,
-        sampleRelation.tableMeta.storePath
-      )
-    DictionaryTestCaseUtil.
-      checkDictionary(incrementalLoadTableRelation, "deviceInformationId", "100010")
-
-    // load 2
-    carbonLoadModel = buildCarbonLoadModel(incrementalLoadTableRelation,
-      complexfilePath2,
-      header
-    )
-    GlobalDictionaryUtil
-      .generateGlobalDictionary(sqlContext, carbonLoadModel,
-        sampleRelation.tableMeta.storePath
-      )
-    DictionaryTestCaseUtil.
-      checkDictionary(incrementalLoadTableRelation, "deviceInformationId", "100077")
-  }
-
-}
diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
deleted file mode 100644
index 13d4167..0000000
--- a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql
-
-import scala.collection.mutable.Map
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.execution.command.Field
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-
-/**
-  * Stub class for calling the CarbonSqlParser
-  */
-private class TestCarbonSqlParserStub extends CarbonSqlParser {
-
-  def extractDimAndMsrFieldsTest(fields: Seq[Field],
-      tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
-    extractDimAndMsrFields(fields, tableProperties)
-  }
-
-
-}
-
-/**
-  * Test class to test Carbon Sql Parser
-  */
-class TestCarbonSqlParser extends QueryTest {
-
-  /**
-    * load all test fields
-    * @return
-    */
-  def loadAllFields: Seq[Field] = {
-    var fields: Seq[Field] = Seq[Field]()
-
-    var col1 = Field("col1", Option("String"), Option("col1"), None, null, Some("columnar"))
-    var col2 = Field("col2", Option("String"), Option("col2"), None, null, Some("columnar"))
-    var col3 = Field("col3", Option("String"), Option("col3"), None, null, Some("columnar"))
-    var col4 = Field("col4", Option("int"), Option("col4"), None, null, Some("columnar"))
-    var col5 = Field("col5", Option("String"), Option("col5"), None, null, Some("columnar"))
-    var col6 = Field("col6", Option("String"), Option("col6"), None, null, Some("columnar"))
-    var col7 = Field("col7", Option("String"), Option("col7"), None, null, Some("columnar"))
-    var col8 = Field("col8", Option("String"), Option("col8"), None, null, Some("columnar"))
-
-    fields :+= col1
-    fields :+= col2
-    fields :+= col3
-    fields :+= col4
-    fields :+= col5
-    fields :+= col6
-    fields :+= col7
-    fields :+= col8
-    fields
-  }
-
-  // Testing the extracting of Dims and no Dictionary
-  test("Test-extractDimColsAndNoDictionaryFields") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
-    val fields: Seq[Field] = loadAllFields
-
-    val stub = new TestCarbonSqlParserStub()
-    val (dimCols, _, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
-    // testing col
-
-    //All dimension fields should be available in dimensions list
-    assert(dimCols.size == 8)
-    assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
-    assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-    assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-    assert(dimCols.lift(3).get.column.equalsIgnoreCase("col4"))
-
-    //No dictionary column names will be available in noDictionary list
-    assert(noDictionary.size == 7)
-    assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
-    assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
-    assert(noDictionary.lift(2).get.equalsIgnoreCase("col3"))
-    assert(noDictionary.lift(3).get.equalsIgnoreCase("col5"))
-    assert(noDictionary.lift(4).get.equalsIgnoreCase("col6"))
-    assert(noDictionary.lift(5).get.equalsIgnoreCase("col7"))
-    assert(noDictionary.lift(6).get.equalsIgnoreCase("col8"))
-
-  }
-
-  test("Test-DimAndMsrColsWithNoDictionaryFields1") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1")
-    val fields: Seq[Field] = loadAllFields
-    val stub = new TestCarbonSqlParserStub()
-    val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
-    //below fields should be available in dimensions list
-    assert(dimCols.size == 7)
-    assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
-    assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-    assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
-    //below column names will be available in noDictionary list
-    assert(noDictionary.size == 7)
-    assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
-    assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
-    assert(noDictionary.lift(2).get.equalsIgnoreCase("col3"))
-    assert(noDictionary.lift(3).get.equalsIgnoreCase("col5"))
-    assert(noDictionary.lift(4).get.equalsIgnoreCase("col6"))
-    assert(noDictionary.lift(5).get.equalsIgnoreCase("col7"))
-    assert(noDictionary.lift(6).get.equalsIgnoreCase("col8"))
-
-    //check msr
-    assert(msrCols.size == 1)
-    assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
-  }
-
-  test("Test-DimAndMsrColsWithNoDictionaryFields2") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
-    val fields: Seq[Field] = loadAllFields
-    val stub = new TestCarbonSqlParserStub()
-    val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
-    //below dimension fields should be available in dimensions list
-    assert(dimCols.size == 7)
-    assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
-    assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-    assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
-    //below column names will be available in noDictionary list
-    assert(noDictionary.size == 6)
-    assert(noDictionary.lift(0).get.equalsIgnoreCase("col2"))
-    assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
-    assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
-    assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
-    assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
-    assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
-    //check msr
-    assert(msrCols.size == 1)
-    assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
-  }
-
-  test("Test-DimAndMsrColsWithNoDictionaryFields3") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
-    val fields: Seq[Field] = loadAllFields
-    val stub = new TestCarbonSqlParserStub()
-    val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields,
-      tableProperties)
-
-    //below dimension fields should be available in dimensions list
-    assert(dimCols.size == 8)
-    assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
-    assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-    assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-    assert(dimCols.lift(3).get.column.equalsIgnoreCase("col4"))
-
-    //below column names will be available in noDictionary list
-    assert(noDictionary.size == 7)
-    assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
-    assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
-    assert(noDictionary.lift(2).get.equalsIgnoreCase("col3"))
-    assert(noDictionary.lift(3).get.equalsIgnoreCase("col5"))
-    assert(noDictionary.lift(4).get.equalsIgnoreCase("col6"))
-    assert(noDictionary.lift(5).get.equalsIgnoreCase("col7"))
-    assert(noDictionary.lift(6).get.equalsIgnoreCase("col8"))
-
-    //check msr
-    assert(msrCols.size == 0)
-  }
-
-  test("Test-DimAndMsrColsWithNoDictionaryFields4") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col3", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
-    val fields: Seq[Field] = loadAllFields
-    val stub = new TestCarbonSqlParserStub()
-    val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
-    //below dimension fields should be available in dimensions list
-    assert(dimCols.size == 7)
-    assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
-    assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-    assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
-    //below column names will be available in noDictionary list
-    assert(noDictionary.size == 6)
-    assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
-    assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
-    assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
-    assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
-    assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
-    assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
-    //check msr
-    assert(msrCols.size == 1)
-    assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
-  }
-
-  test("Test-DimAndMsrColsWithNoDictionaryFields5") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
-    val fields: Seq[Field] = loadAllFields
-    val stub = new TestCarbonSqlParserStub()
-    val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
-    //below dimension fields should be available in dimensions list
-    assert(dimCols.size == 7)
-    assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
-    assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-    assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
-    //below column names will be available in noDictionary list
-    assert(noDictionary.size == 6)
-    assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
-    assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
-    assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
-    assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
-    assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
-    assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
-    //check msr
-    assert(msrCols.size == 1)
-    assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
-  }
-
-  test("Test-DimAndMsrColsWithNoDictionaryFields6") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
-    val fields: Seq[Field] = loadAllFields
-    val stub = new TestCarbonSqlParserStub()
-    val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
-    //below dimension fields should be available in dimensions list
-    assert(dimCols.size == 7)
-    assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
-    assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-    assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-
-    //below column names will be available in noDictionary list
-    assert(noDictionary.size == 6)
-    assert(noDictionary.lift(0).get.equalsIgnoreCase("col2"))
-    assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
-    assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
-    assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
-    assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
-    assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
-    //check msr
-    assert(msrCols.size == 1)
-    assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
-  }
-
-  test("Test-DimAndMsrColsWithNoDictionaryFields7") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2 ,col1  ",
-      CarbonCommonConstants.DICTIONARY_INCLUDE -> "col3 ,col4 "
-    )
-    val fields: Seq[Field] = loadAllFields
-    val stub = new TestCarbonSqlParserStub()
-    val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
-    //below dimension fields should be available in dimensions list
-    assert(dimCols.size == 8)
-    assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
-    assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-    assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
-    assert(dimCols.lift(3).get.column.equalsIgnoreCase("col4"))
-
-    //below column names will be available in noDictionary list
-    assert(noDictionary.size == 6)
-    assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
-    assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
-    assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
-    assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
-    assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
-    assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
-    //check msr
-    assert(msrCols.size == 0)
-  }
-
-  test("Test-DimAndMsrColsWithNoDictionaryFields8") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE-> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col3")
-    val fields: Seq[Field] = loadAllFields
-    val stub = new TestCarbonSqlParserStub()
-    val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
-    //below dimension fields should be available in dimensions list
-    assert(dimCols.size == 7)
-    assert(dimCols.lift(0).get.column.equalsIgnoreCase("col1"))
-    assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
-
-    //below column names will be available in noDictionary list
-    assert(noDictionary.size == 6)
-    assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
-    assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
-    assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
-    assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
-    assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
-    assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
-
-    //check msr
-    assert(msrCols.size == 1)
-    assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
-  }
-
-  // Testing the extracting of measures
-  test("Test-extractMsrColsFromFields") {
-    val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2",
-      CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
-    val fields: Seq[Field] = loadAllFields
-    val stub = new TestCarbonSqlParserStub()
-    val (_, msrCols, _, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
-
-    // testing col
-    assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
-
-  }
-
-}
-
-
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index a09bd62..ed01728 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -38,16 +38,6 @@
       <groupId>org.apache.carbondata</groupId>
       <artifactId>carbondata-spark-common</artifactId>
       <version>${project.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-hive-thriftserver_2.10</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-sql_2.10</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
diff --git a/pom.xml b/pom.xml
index e7dd0e5..39dc565 100644
--- a/pom.xml
+++ b/pom.xml
@@ -416,7 +416,7 @@
 
   <profiles>
     <profile>
-      <!--This profile does not build spark module, so user should explicitly give spark profile also like spark-1.6 -->
+      <!--This profile does not build spark module, so user should explicitly give spark profile also like spark-2.1 -->
       <id>build-with-format</id>
       <modules>
         <module>format</module>
@@ -426,15 +426,13 @@
       <!-- This profile only should be used for release prepare to cover all the modules -->
       <id>build-all</id>
       <properties>
-        <spark.version>1.6.2</spark.version>
-        <scala.binary.version>2.10</scala.binary.version>
-        <scala.version>2.10.4</scala.version>
+        <spark.version>2.1.0</spark.version>
+        <scala.binary.version>2.11</scala.binary.version>
+        <scala.version>2.11.8</scala.version>
         <maven.test.skip>true</maven.test.skip>
       </properties>
       <modules>
         <module>format</module>
-        <module>integration/spark</module>
-        <module>examples/spark</module>
         <module>integration/spark2</module>
         <module>examples/spark2</module>
         <module>integration/hive</module>
@@ -467,65 +465,6 @@
       </properties>
     </profile>
     <profile>
-      <id>spark-1.5</id>
-      <properties>
-        <spark.version>1.5.2</spark.version>
-        <scala.binary.version>2.10</scala.binary.version>
-        <scala.version>2.10.4</scala.version>
-      </properties>
-      <modules>
-        <module>integration/spark</module>
-        <module>examples/spark</module>
-        <module>examples/flink</module>
-      </modules>
-    </profile>
-    <profile>
-      <id>spark-1.6</id>
-      <properties>
-        <spark.version>1.6.2</spark.version>
-        <scala.binary.version>2.10</scala.binary.version>
-        <scala.version>2.10.4</scala.version>
-      </properties>
-      <modules>
-        <module>integration/spark</module>
-        <module>examples/spark</module>
-        <module>examples/flink</module>
-      </modules>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.eluder.coveralls</groupId>
-            <artifactId>coveralls-maven-plugin</artifactId>
-            <version>4.3.0</version>
-            <configuration>
-              <repoToken>Q8mcBW8dw34wjNiPAMx0SFQidspIwHw8g</repoToken>
-              <sourceEncoding>UTF-8</sourceEncoding>
-              <jacocoReports>
-                <jacocoReport>${basedir}/target/carbondata-coverage-report/carbondata-coverage-report.xml
-              </jacocoReport>
-              </jacocoReports>
-              <sourceDirectories>
-                <sourceDirectory>${basedir}/common/src/main/java</sourceDirectory>
-                <sourceDirectory>${basedir}/core/src/main/java</sourceDirectory>
-                <sourceDirectory>${basedir}/processing/src/main/java</sourceDirectory>
-                <sourceDirectory>${basedir}/hadoop/src/main/java</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/spark-common/src/main/scala</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/spark-common/src/main/java</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/spark-common-test/src/main/scala</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/spark-common-test/src/main/java</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/hive/src/main/scala</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/hive/src/main/java</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/presto/src/main/scala</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/presto/src/main/java</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/spark/src/main/scala</sourceDirectory>
-                <sourceDirectory>${basedir}/integration/spark/src/main/java</sourceDirectory>
-              </sourceDirectories>
-            </configuration>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
-    <profile>
       <id>spark-2.1</id>
       <activation>
         <activeByDefault>true</activeByDefault>
diff --git a/processing/pom.xml b/processing/pom.xml
index 03c8b57..7b53803 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -38,12 +38,6 @@
       <groupId>org.apache.carbondata</groupId>
       <artifactId>carbondata-core</artifactId>
       <version>${project.version}</version>
-      <exclusions>
-        <exclusion>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-sql_2.10</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>