carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [02/20] carbondata git commit: [CARBONDATA-1346] SDV cluster tests
Date Tue, 01 Aug 2017 06:00:09 GMT
http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/Vector2TestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/Vector2TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/Vector2TestCase.scala
new file mode 100644
index 0000000..21423f1
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/Vector2TestCase.scala
@@ -0,0 +1,625 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * Test Class for Vector2TestCase to verify all scenerios
+ */
+
+class Vector2TestCase extends QueryTest with BeforeAndAfterAll {
+         
+
+  //To check select all records with  vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_071", Include) {
+     sql(s"""CREATE TABLE uniqdatavector2 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect
+   sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' into table uniqdatavector2 OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+    sql(s"""select * from uniqdatavector2 """).collect
+
+
+  }
+
+
+  //To check  random measure select query with  vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_072", Include) {
+
+    sql(s"""select cust_name,DOB,DOJ from uniqdatavector2 where cust_id=10999""").collect
+
+     sql(s"""drop table uniqdatavector2""").collect
+  }
+
+
+  //To check select random columns  and order with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_073", Include) {
+     sql(s"""create table double1(id double, name string) STORED BY 'org.apache.carbondata.format' """).collect
+   sql(s"""load data  inpath '$resourcesPath/Data/InsertData/maxrange_double.csv' into table double1""").collect
+    sql(s"""select id from double1 order by id""").collect
+
+
+  }
+
+
+  //To check the logs of executor with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_074", Include) {
+
+    sql(s"""select id from double1 order by id""").collect
+
+
+  }
+
+
+  //To check  for select random measures with group by and having clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_075", Include) {
+
+    sql(s"""select id,count(*) from double1 group by id having count(*)=1""").collect
+
+
+  }
+
+
+  //To check for select count query with group by and having clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_076", Include) {
+
+    sql(s"""select id,count(id) from double1 group by id having count(*)=1""").collect
+
+     sql(s"""drop table double1""").collect
+  }
+
+
+  //To applied cast method  with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_077", Include) {
+     sql(s"""CREATE TABLE uniqdatavector22 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect
+   sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' into table uniqdatavector22 OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+    sql(s"""select cast(Double_COLUMN1 as int) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply sum method on a column with select query with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_078", Include) {
+
+    sql(s"""select sum(CUST_ID) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply the average method on a column with select query with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_079", Include) {
+
+    sql(s"""select avg(CUST_ID) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply the percentile_approx method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_080", Include) {
+
+    sql(s"""select percentile_approx(1, 0.5 ,500)  from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply the var_samp method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_081", Include) {
+
+    sql(s"""select var_samp(cust_id) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply the stddev_pop method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_082", Include) {
+
+    sql(s"""select stddev_pop(cust_id) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply the stddev_samp method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_083", Include) {
+
+    sql(s"""select stddev_samp(cust_id) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply percentile method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_084", Include) {
+
+    sql(s"""select percentile(0,1) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply min method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_085", Include) {
+
+    sql(s"""select min(CUST_ID) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To applied max method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_086", Include) {
+
+    sql(s"""select max(CUST_ID) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply sum method with plus operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_087", Include) {
+
+    sql(s"""select sum(CUST_ID+1) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply sum method with minus operator with vectorized carbon reader enabled
+
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_088", Include) {
+
+    sql(s"""select sum(CUST_ID-1) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply count method  with distinct operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_089", Include) {
+
+    sql(s"""select count(DISTINCT CUST_ID) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To check random measure select query with  AND operator and vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_090", Include) {
+
+    sql(s"""select cust_name,DOB,DOJ from uniqdatavector22 where cust_id=10999 and INTEGER_COLUMN1=2000 """).collect
+
+
+  }
+
+
+  //To check random measure select query with  OR operator and vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_091", Include) {
+
+    sql(s"""select cust_name,DOB,DOJ from uniqdatavector22 where cust_id=10999 or INTEGER_COLUMN1=2000 """).collect
+
+
+  }
+
+
+  //To apply count method with if operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_092", Include) {
+
+    sql(s"""select count(if(CUST_ID<1999,NULL,CUST_NAME)) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply in operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_093", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID IN(1,22)""").collect
+
+
+  }
+
+
+  //To apply not in operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_094", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID NOT IN(1,22)""").collect
+
+
+  }
+
+
+  //To apply between operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_095", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID BETWEEN 1 AND 11000""").collect
+
+
+  }
+
+
+  //To apply not between operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_096", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID NOT BETWEEN 1 AND 11000""").collect
+
+
+  }
+
+
+  //To apply between in operator with order by clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_097", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID in (1,10999)order by 'CUST_ID'""").collect
+
+
+  }
+
+
+  //To apply between in operator with group by clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_098", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID in (1,10999) group by CUST_NAME""").collect
+
+
+  }
+
+
+  //To apply  null clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_099", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID is null""").collect
+
+
+  }
+
+
+  //To applied not null clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_100", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID is not null""").collect
+
+
+  }
+
+
+  //To apply > operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_101", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID>1""").collect
+
+
+  }
+
+
+  //To apply < operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_102", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID<1""").collect
+
+
+  }
+
+
+  //To apply != operator with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_103", Include) {
+
+    sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID!=1""").collect
+
+
+  }
+
+
+  //To apply like clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_104", Include) {
+
+    sql(s"""select CUST_ID from uniqdatavector22 where CUST_ID like 10999""").collect
+
+
+  }
+
+
+  //To apply like% clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_105", Include) {
+
+    sql(s"""select CUST_ID from uniqdatavector22 where CUST_ID like '%10999%'""").collect
+
+
+  }
+
+
+  //To apply rlike clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_106", Include) {
+
+    sql(s"""select CUST_ID from uniqdatavector22 where CUST_ID rlike 10999""").collect
+
+
+  }
+
+
+  //To apply rlike% clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_107", Include) {
+
+    sql(s"""select CUST_ID from uniqdatavector22 where CUST_ID rlike '%10999'""").collect
+
+
+  }
+
+
+  //To apply alias clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_108", Include) {
+
+    sql(s"""select count(cust_id)+10.364 as a from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply aliase clause with group by clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_109", Include) {
+
+    sql(s"""select count(cust_id)+10.364 as a from uniqdatavector22 group by CUST_ID""").collect
+
+
+  }
+
+
+  //To apply aliase clause with order by clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_110", Include) {
+
+    sql(s"""select cust_id,count(cust_name) a from uniqdatavector22 group by cust_id order by cust_id""").collect
+
+
+  }
+
+
+  //To apply regexp_replace clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_111", Include) {
+
+    sql(s"""select regexp_replace(cust_id, 'i', 'ment')  from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply date_add method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_118", Include) {
+
+    sql(s"""SELECT date_add(DOB,1) FROM uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply date_sub method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_119", Include) {
+
+    sql(s"""SELECT date_sub(DOB,1) FROM uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply current_date method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_120", Include) {
+
+    sql(s"""SELECT current_date() FROM uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply add_month method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_121", Include) {
+
+    sql(s"""SELECT add_months(dob,1) FROM uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply last_day method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_122", Include) {
+
+    sql(s"""SELECT last_day(dob) FROM uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply next_day method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_123", Include) {
+
+    sql(s"""SELECT next_day(dob,'monday') FROM uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply months_between method on carbon table
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_124", Include) {
+
+    sql(s"""select months_between('2016-12-28', '2017-01-30') from uniqdatavector22""").collect
+
+
+  }
+
+
+  //Toapply date_diff method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_125", Include) {
+
+    sql(s"""select datediff('2009-03-01', '2009-02-27') from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply concat method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_126", Include) {
+
+    sql(s"""SELECT concat('hi','hi') FROM uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply lower method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_127", Include) {
+
+    sql(s"""SELECT lower('H') FROM uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply substr method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_128", Include) {
+
+    sql(s"""select substr(cust_id,3) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply trim method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_129", Include) {
+
+    sql(s"""select trim(cust_id) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply split method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_130", Include) {
+
+    sql(s"""select split('knoldus','ol') from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply split method  limit clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_131", Include) {
+
+    sql(s"""select split('knoldus','ol') from uniqdatavector22 limit 1""").collect
+
+
+  }
+
+
+  //To apply reverse on carbon table with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_132", Include) {
+
+    sql(s"""select reverse('knoldus') from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply replace on carbon table with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_133", Include) {
+
+    sql(s"""select regexp_replace('Tester', 'T', 't') from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply replace with limit clause with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_134", Include) {
+
+    sql(s"""select regexp_replace('Tester', 'T', 't') from uniqdatavector22 limit 1""").collect
+
+
+  }
+
+
+  //To apply FORMAT_STRING on carbon table with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_135", Include) {
+
+    sql(s"""select format_string('data', cust_name) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply sentences method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_136", Include) {
+
+    sql(s"""select sentences(cust_name) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply space method on carbon table with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_137", Include) {
+
+    sql(s"""select space(10) from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply rtrim method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_138", Include) {
+
+    sql(s"""select rtrim("     testing           ") from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply ascii method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_139", Include) {
+
+    sql(s"""select ascii('A') from uniqdatavector22""").collect
+
+
+  }
+
+
+  //To apply utc_timestamp method with vectorized carbon reader enabled
+  test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_140", Include) {
+
+    sql(s"""select from_utc_timestamp('2016-12-12 08:00:00','PST') from uniqdatavector22""").collect
+
+     sql(s"""drop table uniqdatavector22""").collect
+  }
+
+  val prop = CarbonProperties.getInstance()
+  val p1 = prop.getProperty("carbon.enable.vector.reader", CarbonCommonConstants.ENABLE_VECTOR_READER_DEFAULT)
+
+  override protected def beforeAll() {
+    // Adding new properties
+    prop.addProperty("carbon.enable.vector.reader", "false")
+  }
+
+  override def afterAll: Unit = {
+    //Reverting to old
+    prop.addProperty("carbon.enable.vector.reader", p1)
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
new file mode 100644
index 0000000..de946a0
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.cluster.sdv.suite
+
+import org.apache.spark.sql.test.TestQueryExecutor
+import org.scalatest.{BeforeAndAfterAll, Suites}
+
+import org.apache.carbondata.cluster.sdv.generated._
+
+/**
+ * Suite class for all tests.
+ */
+class SDVSuites extends Suites with BeforeAndAfterAll {
+
+  val suites =                new AlterTableTestCase ::
+                              new BadRecordTestCase ::
+                              new BatchSortLoad1TestCase ::
+                              new BatchSortLoad2TestCase ::
+                              new BatchSortQueryTestCase ::
+                              new ColumndictTestCase ::
+                              new DataLoadingTestCase ::
+                              new DataLoadingV3TestCase ::
+                              new InvertedindexTestCase ::
+                              new OffheapQuery1TestCase ::
+                              new OffheapQuery2TestCase ::
+                              new OffheapSort1TestCase ::
+                              new OffheapSort2TestCase ::
+                              new PartitionTestCase ::
+                              new QueriesBasicTestCase ::
+                              new QueriesBVATestCase ::
+                              new QueriesCompactionTestCase ::
+                              new QueriesExcludeDictionaryTestCase ::
+                              new QueriesIncludeDictionaryTestCase ::
+                              new QueriesNormalTestCase ::
+                              new QueriesRangeFilterTestCase ::
+                              new QueriesSparkBlockDistTestCase ::
+                              new ShowLoadsTestCase ::
+                              new SinglepassTestCase ::
+                              new SortColumnTestCase ::
+                              new TimestamptypesTestCase ::
+                              new V3offheapvectorTestCase ::
+                              new Vector1TestCase ::
+                              new Vector2TestCase ::Nil
+
+  override val nestedSuites = suites.toIndexedSeq
+
+  override protected def afterAll() = {
+    println("---------------- Stopping spark -----------------")
+    TestQueryExecutor.INSTANCE.stop()
+    println("---------------- Stopped spark -----------------")
+  }
+}
+
+/**
+ * Suite class for all tests.
+ */
+class SDVSuites1 extends Suites with BeforeAndAfterAll {
+
+  val suites =                 new AlterTableTestCase ::
+                               new BadRecordTestCase ::
+                               new BatchSortLoad1TestCase ::
+                               new BatchSortLoad2TestCase ::
+                               new BatchSortQueryTestCase ::
+                               new ColumndictTestCase ::
+                               new DataLoadingTestCase ::
+                               new DataLoadingV3TestCase ::
+                               new InvertedindexTestCase ::
+                               new OffheapQuery1TestCase ::
+                               new OffheapQuery2TestCase ::
+                               new OffheapSort1TestCase ::
+                               new OffheapSort2TestCase ::
+                               new PartitionTestCase ::
+                               new QueriesBasicTestCase :: Nil
+
+  override val nestedSuites = suites.toIndexedSeq
+
+  override protected def afterAll() = {
+    println("---------------- Stopping spark -----------------")
+    TestQueryExecutor.INSTANCE.stop()
+    println("---------------- Stopped spark -----------------")
+  }
+}
+
+/**
+ * Suite class for all tests.
+ */
+class SDVSuites2 extends Suites with BeforeAndAfterAll {
+
+  val suites =    new QueriesBVATestCase ::
+                  new QueriesCompactionTestCase ::
+                  new QueriesExcludeDictionaryTestCase ::
+                  new QueriesIncludeDictionaryTestCase ::
+                  new QueriesNormalTestCase ::
+                  new QueriesRangeFilterTestCase ::
+                  new QueriesSparkBlockDistTestCase ::
+                  new ShowLoadsTestCase ::
+                  new SinglepassTestCase ::
+                  new SortColumnTestCase ::
+                  new TimestamptypesTestCase ::
+                  new V3offheapvectorTestCase ::
+                  new Vector1TestCase ::
+                  new Vector2TestCase :: Nil
+
+  override val nestedSuites = suites.toIndexedSeq
+
+  override protected def afterAll() = {
+    println("---------------- Stopping spark -----------------")
+    TestQueryExecutor.INSTANCE.stop()
+    println("---------------- Stopped spark -----------------")
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala
new file mode 100644
index 0000000..37803a8
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.common.util
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.scalatest.{FunSuite, Outcome}
+
+
+private[spark] abstract class CarbonFunSuite extends FunSuite {
+
+  private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+
+  /**
+   * Log the suite name and the test name before and after each test.
+   *
+   * Subclasses should never override this method. If they wish to run
+   * custom code before and after each test, they should should mix in
+   * the {{org.scalatest.BeforeAndAfter}} trait instead.
+   */
+  final protected override def withFixture(test: NoArgTest): Outcome = {
+    val testName = test.text
+    val suiteName = this.getClass.getName
+    val shortSuiteName = suiteName.replaceAll("org.apache.spark", "o.a.s")
+    try {
+      LOGGER.info(s"\n\n===== TEST OUTPUT FOR $shortSuiteName: '$testName' =====\n")
+      test()
+    } finally {
+      LOGGER.info(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n")
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala
new file mode 100644
index 0000000..cdd415f
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.common.util
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, OneRowRelation}
+import org.apache.spark.sql.catalyst.util._
+
+/**
+ * Provides helper methods for comparing plans.
+ */
+class PlanTest extends CarbonFunSuite {
+
+  /** Fails the test if the two expressions do not match */
+  protected def compareExpressions(e1: Expression, e2: Expression): Unit = {
+    comparePlans(Filter(e1, OneRowRelation), Filter(e2, OneRowRelation))
+  }
+
+  /** Fails the test if the two plans do not match */
+  protected def comparePlans(plan1: LogicalPlan, plan2: LogicalPlan) {
+    val normalized1 = normalizeExprIds(plan1)
+    val normalized2 = normalizeExprIds(plan2)
+    if (normalized1 != normalized2) {
+      fail(
+        s"""
+           |== FAIL: Plans do not match ===
+           |${sideBySide(normalized1.treeString, normalized2.treeString).mkString("\n")}
+         """.stripMargin)
+    }
+  }
+
+  /**
+   * Since attribute references are given globally unique ids during analysis,
+   * we must normalize them to check if two different queries are identical.
+   */
+  protected def normalizeExprIds(plan: LogicalPlan) = {
+    plan transformAllExpressions {
+      case a: AttributeReference =>
+        AttributeReference(a.name, a.dataType, a.nullable)(exprId = ExprId(0))
+      case a: Alias =>
+        Alias(a.child, a.name)(exprId = ExprId(0))
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
new file mode 100644
index 0000000..292c160
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.common.util
+
+import java.io.{FileInputStream, ObjectInputStream, ObjectOutputStream}
+import java.math
+import java.math.RoundingMode
+import java.util.{Locale, TimeZone}
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import scala.collection.JavaConversions._
+
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.execution.command.LoadDataCommand
+import org.apache.spark.sql.test.{ResourceRegisterAndCopier, TestQueryExecutor}
+import org.apache.spark.sql.{DataFrame, Row, SQLContext}
+import org.scalatest.Suite
+
+import org.apache.carbondata.core.datastore.impl.FileFactory
+
+class QueryTest extends PlanTest with Suite {
+
+  val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+
+  val DOLLAR = "$"
+
+  // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
+  TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
+  // Add Locale setting
+  Locale.setDefault(Locale.US)
+
+  /**
+   * Runs the plan and makes sure the answer contains all of the keywords, or the
+   * none of keywords are listed in the answer
+   * @param df the [[DataFrame]] to be executed
+   * @param exists true for make sure the keywords are listed in the output, otherwise
+   *               to make sure none of the keyword are not listed in the output
+   * @param keywords keyword in string array
+   */
+  def checkExistence(df: DataFrame, exists: Boolean, keywords: String*) {
+    val outputs = df.collect().map(_.mkString).mkString
+    for (key <- keywords) {
+      if (exists) {
+        assert(outputs.contains(key), s"Failed for $df ($key doesn't exist in result)")
+      } else {
+        assert(!outputs.contains(key), s"Failed for $df ($key existed in the result)")
+      }
+    }
+  }
+
+  /**
+   * Runs the plan and makes sure the answer matches the expected result.
+   * @param df the [[DataFrame]] to be executed
+   * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s.
+   */
+  protected def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Unit = {
+    QueryTest.checkAnswer(df, expectedAnswer) match {
+      case Some(errorMessage) => fail(errorMessage)
+      case None =>
+    }
+  }
+
+  protected def checkAnswer(df: DataFrame, expectedAnswer: Row): Unit = {
+    checkAnswer(df, Seq(expectedAnswer))
+  }
+
+  protected def checkAnswer(df: DataFrame, expectedAnswer: DataFrame): Unit = {
+    checkAnswer(df, expectedAnswer.collect())
+  }
+
+  protected def checkAnswer(carbon: String, hive: String, uniqueIdentifier:String): Unit = {
+    val path = TestQueryExecutor.hiveresultpath + "/"+uniqueIdentifier
+    if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
+      val objinp = new ObjectInputStream(FileFactory.getDataInputStream(path, FileFactory.getFileType(path)))
+      val rows = objinp.readObject().asInstanceOf[Array[Row]]
+      objinp.close()
+      checkAnswer(sql(carbon), rows)
+    } else {
+      val rows = sql(hive).collect()
+      val obj = new ObjectOutputStream(FileFactory.getDataOutputStream(path, FileFactory.getFileType(path)))
+      obj.writeObject(rows)
+      obj.close()
+      checkAnswer(sql(carbon), rows)
+    }
+  }
+
+  protected def checkAnswer(carbon: String, expectedAnswer: Seq[Row], uniqueIdentifier:String): Unit = {
+    checkAnswer(sql(carbon), expectedAnswer)
+  }
+
+  def sql(sqlText: String): DataFrame = {
+    val frame = TestQueryExecutor.INSTANCE.sql(sqlText)
+    val plan = frame.queryExecution.logical
+    if (TestQueryExecutor.hdfsUrl.startsWith("hdfs")) {
+      plan match {
+        case l: LoadDataCommand =>
+          val copyPath = TestQueryExecutor.warehouse + "/" + l.table.table.toLowerCase +
+                         l.path.substring(l.path.lastIndexOf("/"), l.path.length)
+          ResourceRegisterAndCopier.copyLocalFile(l.path, copyPath)
+        case _ =>
+      }
+    }
+    frame
+  }
+
+  protected def dropTable(tableName: String): Unit ={
+    sql(s"DROP TABLE IF EXISTS $tableName")
+  }
+
+  val sqlContext: SQLContext = TestQueryExecutor.INSTANCE.sqlContext
+
+  val storeLocation = TestQueryExecutor.storeLocation
+  val resourcesPath = TestQueryExecutor.resourcesPath
+  val integrationPath = TestQueryExecutor.integrationPath
+}
+
+object QueryTest {
+
+  def checkAnswer(df: DataFrame, expectedAnswer: java.util.List[Row]): String = {
+    checkAnswer(df, expectedAnswer.toSeq) match {
+      case Some(errorMessage) => errorMessage
+      case None => null
+    }
+  }
+
+  import java.text.DecimalFormat
+
+  /**
+   * Runs the plan and makes sure the answer matches the expected result.
+   * If there was exception during the execution or the contents of the DataFrame does not
+   * match the expected result, an error message will be returned. Otherwise, a [[None]] will
+   * be returned.
+   * @param df the [[DataFrame]] to be executed
+   * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s.
+   */
+  def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Option[String] = {
+    val isSorted = df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty
+    def prepareAnswer(answer: Seq[Row]): Seq[Row] = {
+      // Converts data to types that we can do equality comparison using Scala collections.
+      // For BigDecimal type, the Scala type has a better definition of equality test (similar to
+      // Java's java.math.BigDecimal.compareTo).
+      // For binary arrays, we convert it to Seq to avoid of calling java.util.Arrays.equals for
+      // equality test.
+      val converted: Seq[Row] = answer.map { s =>
+        Row.fromSeq(s.toSeq.map {
+          case d: java.math.BigDecimal => BigDecimal(d)
+          case b: Array[Byte] => b.toSeq
+          case o => o
+        })
+      }
+      if (!isSorted) converted.sortBy(_.toString()) else converted
+    }
+    val sparkAnswer = try df.collect().toSeq catch {
+      case e: Exception =>
+        val errorMessage =
+          s"""
+             |Exception thrown while executing query:
+             |== Exception ==
+             |$e
+             |${org.apache.spark.sql.catalyst.util.stackTraceToString(e)}
+          """.stripMargin
+        return Some(errorMessage)
+    }
+
+    if (prepareAnswer(expectedAnswer) != prepareAnswer(sparkAnswer)) {
+      val errorMessage =
+        s"""
+           |Results do not match for query:
+           |== Results ==
+           |${
+          sideBySide(
+            s"== Correct Answer - ${expectedAnswer.size} ==" +:
+              prepareAnswer(expectedAnswer).map(_.toString()),
+            s"== Spark Answer - ${sparkAnswer.size} ==" +:
+              prepareAnswer(sparkAnswer).map(_.toString())).mkString("\n")
+        }
+      """.stripMargin
+      return Some(errorMessage)
+    }
+
+    return None
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala
new file mode 100644
index 0000000..7d6770b
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under onassemblye
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.common.util
+
+import org.scalatest.Tag
+
+object Include extends Tag("Include")
+
+object Exclude extends Tag("Exclude")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index 613d59f..fbb4f8d 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -137,6 +137,7 @@
         <version>1.0</version>
         <!-- Note config is repeated in surefire config -->
         <configuration>
+          <skipTests>false</skipTests>
           <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
           <junitxml>.</junitxml>
           <filereports>CarbonTestSuite.txt</filereports>
@@ -227,5 +228,11 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>sdvtest</id>
+      <properties>
+        <maven.test.skip>true</maven.test.skip>
+      </properties>
+    </profile>
   </profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
index adbc1b1..c09d285 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
@@ -128,7 +128,9 @@ class TestDataLoadWithColumnsMoreThanSchema extends QueryTest with BeforeAndAfte
       sql(s"LOAD DATA INPATH '$resourcesPath/seq_20Records.csv' into table smart_500_DE options('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='SID,PROBEID,INTERFACEID,GROUPID,GGSN_ID,SGSN_ID,dummy,SESSION_INDICATOR,BEGIN_TIME,BEGIN_TIME_MSEL,END_TIME,END_TIME_MSEL,PROT_CATEGORY,PROT_TYPE,L7_CARRIER_PROT,SUB_PROT_TYPE,MSISDN,IMSI,IMEI,ENCRYPT_VERSION,ROAMING_TYPE,ROAM_DIRECTION,MS_IP,SERVER_IP,MS_PORT,APN,SGSN_SIG_IP,GGSN_USER_IP,SGSN_USER_IP,MCC,MNC,RAT,LAC,RAC,SAC,CI,SERVER_DECIMAL,BROWSER_TIMESTAMP,TCP_CONN_STATES,GGSN_SIG_IP_BigInt_NEGATIVE,TCP_STATES_BIGINTPOSITIVE,dummy,TCP_WIN_SIZE,dummy,TCP_MSS,dummy,TCP_CONN_TIMES,dummy,TCP_CONN_2_FAILED_TIMES,dummy,TCP_CONN_3_FAILED_TIMES,HOST,STREAMING_URL,dummy,GET_STREAMING_FAILED_CODE,dummy,GET_STREAMING_FLAG,dummy,GET_NUM,dummy,GET_SUCCEED_NUM,dummy,GET_RETRANS_NUM,dummy,GET_TIMEOUT_NUM,INTBUFFER_FST_FLAG,INTBUFFER_FULL_FLAG,STALL_NUM,dummy,VIDEO_FRAME_RATE,dummy,VIDEO_CODEC_ID,dummy,VIDEO_WIDTH,dummy,VIDEO_HEIGHT,dummy,AUD
 IO_CODEC_ID,dummy,MEDIA_FILE_TYPE,dummy,PLAY_STATE,dummy,PLAY_STATE,dummy,STREAMING_FLAG,dummy,TCP_STATUS_INDICATOR,dummy,DISCONNECTION_FLAG,dummy,FAILURE_CODE,FLAG,TAC,ECI,dummy,TCP_SYN_TIME_MSEL,dummy,TCP_FST_SYN_DIRECTION,RAN_NE_USER_IP,HOMEMCC,HOMEMNC,dummy,CHARGE_FLAG,dummy,PREPAID_FLAG,dummy,USER_AGENT,dummy,MS_WIN_STAT_TOTAL_NUM,dummy,MS_WIN_STAT_SMALL_NUM,dummy,MS_ACK_TO_1STGET_DELAY,dummy,SERVER_ACK_TO_1STDATA_DELAY,dummy,STREAMING_TYPE,dummy,SOURCE_VIDEO_QUALITY,TETHERING_FLAG,CARRIER_ID,LAYER1ID,LAYER2ID,dummy,LAYER3ID,dummy,LAYER4ID,dummy,LAYER5ID,dummy,LAYER6ID,CHARGING_RULE_BASE_NAME,SP,dummy,EXTENDED_URL,SV,FIRST_SAI_CGI_ECGI,dummy,EXTENDED_URL_OTHER,SIGNALING_USE_FLAG,dummy,DNS_RETRANS_NUM,dummy,DNS_FAIL_CODE,FIRST_RAT,FIRST_RAT,MS_INDICATOR,LAST_SAI_CGI_ECGI,LAST_RAT,dummy,FIRST_LONGITUDE,dummy,FIRST_LATITUDE,dummy,FIRST_ALTITUDE,dummy,FIRST_RASTERLONGITUDE,dummy,FIRST_RASTERLATITUDE,dummy,FIRST_RASTERALTITUDE,dummy,FIRST_FREQUENCYSPOT,dummy,FIRST_CLUTTER,dummy,FIRS
 T_USERBEHAVIOR,dummy,FIRST_SPEED,dummy,FIRST_CREDIBILITY,dummy,LAST_LONGITUDE,dummy,LAST_LATITUDE,dummy,LAST_ALTITUDE,dummy,LAST_RASTERLONGITUDE,dummy,LAST_RASTERLATITUDE,dummy,LAST_RASTERALTITUDE,dummy,LAST_FREQUENCYSPOT,dummy,LAST_CLUTTER,dummy,LAST_USERBEHAVIOR,dummy,LAST_SPEED,dummy,LAST_CREDIBILITY,dummy,IMEI_CIPHERTEXT,APP_ID,dummy,DOMAIN_NAME,dummy,STREAMING_CACHE_IP,dummy,STOP_LONGER_THAN_MIN_THRESHOLD,dummy,STOP_LONGER_THAN_MAX_THRESHOLD,dummy,PLAY_END_STAT,dummy,STOP_START_TIME1,dummy,STOP_END_TIME1,dummy,STOP_START_TIME2,dummy,STOP_END_TIME2,dummy,STOP_START_TIME3,dummy,STOP_END_TIME3,dummy,STOP_START_TIME4,dummy,STOP_END_TIME4,dummy,STOP_START_TIME5,dummy,STOP_END_TIME5,dummy,STOP_START_TIME6,dummy,STOP_END_TIME6,dummy,STOP_START_TIME7,dummy,STOP_END_TIME7,dummy,STOP_START_TIME8,dummy,STOP_END_TIME8,dummy,STOP_START_TIME9,dummy,STOP_END_TIME9,dummy,STOP_START_TIME10,dummy,STOP_END_TIME10,dummy,FAIL_CLASS,RECORD_TYPE,dummy,NODATA_COUNT,dummy,VIDEO_NODATA_DURATION,dummy,VI
 DEO_SMOOTH_DURATION,dummy,VIDEO_SD_DURATION,dummy,VIDEO_HD_DURATION,dummy,VIDEO_UHD_DURATION,dummy,VIDEO_FHD_DURATION,dummy,FLUCTUATION,dummy,START_DOWNLOAD_THROUGHPUT,dummy,L7_UL_GOODPUT_FULL_MSS,dummy,SESSIONKEY,dummy,FIRST_UCELLID,dummy,LAST_UCELLID,dummy,UCELLID1,dummy,LONGITUDE1,dummy,LATITUDE1,dummy,UCELLID2,dummy,LONGITUDE2,dummy,LATITUDE2,dummy,UCELLID3,dummy,LONGITUDE3,dummy,LATITUDE3,dummy,UCELLID4,dummy,LONGITUDE4,dummy,LATITUDE4,dummy,UCELLID5,dummy,LONGITUDE5,dummy,LATITUDE5,dummy,UCELLID6,dummy,LONGITUDE6,dummy,LATITUDE6,dummy,UCELLID7,dummy,LONGITUDE7,dummy,LATITUDE7,dummy,UCELLID8,dummy,LONGITUDE8,dummy,LATITUDE8,dummy,UCELLID9,dummy,LONGITUDE9,dummy,LATITUDE9,dummy,UCELLID10,dummy,LONGITUDE10,dummy,LATITUDE10,dummy,INTBUFFER_FULL_DELAY,dummy,STALL_DURATION,dummy,STREAMING_DW_PACKETS,dummy,STREAMING_DOWNLOAD_DELAY,dummy,PLAY_DURATION,dummy,STREAMING_QUALITY,dummy,VIDEO_DATA_RATE,dummy,AUDIO_DATA_RATE,dummy,STREAMING_FILESIZE,dummy,STREAMING_DURATIOIN,dummy,TCP_SYN_TI
 ME,dummy,TCP_RTT_STEP1,CHARGE_ID,dummy,UL_REVERSE_TO_DL_DELAY,dummy,DL_REVERSE_TO_UL_DELAY,dummy,DATATRANS_DW_GOODPUT,dummy,DATATRANS_DW_TOTAL_DURATION,dummy,SUM_FRAGMENT_INTERVAL,dummy,TCP_FIN_TIMES,dummy,TCP_RESET_TIMES,dummy,URL_CLASSIFICATION,dummy,STREAMING_LQ_DURATIOIN,dummy,MAX_DNS_DELAY,dummy,MAX_DNS2SYN,dummy,MAX_LATANCY_OF_LINK_SETUP,dummy,MAX_SYNACK2FIRSTACK,dummy,MAX_SYNACK2LASTACK,dummy,MAX_ACK2GET_DELAY,dummy,MAX_FRAG_INTERVAL_PREDELAY,dummy,SUM_FRAG_INTERVAL_PREDELAY,dummy,SERVICE_DELAY_MSEC,dummy,HOMEPROVINCE,dummy,HOMECITY,dummy,SERVICE_ID,dummy,CHARGING_CLASS,dummy,DATATRANS_UL_DURATION,dummy,ASSOCIATED_ID,dummy,PACKET_LOSS_NUM,dummy,JITTER,dummy,MS_DNS_DELAY_MSEL,dummy,GET_STREAMING_DELAY,dummy,TCP_UL_RETRANS_WITHOUTPL,dummy,TCP_DW_RETRANS_WITHOUTPL,dummy,GET_MAX_UL_SIZE,dummy,GET_MIN_UL_SIZE,dummy,GET_MAX_DL_SIZE,dummy,GET_MIN_DL_SIZE,dummy,FLOW_SAMPLE_RATIO,dummy,UL_RTT_LONG_NUM,dummy,DW_RTT_LONG_NUM,dummy,UL_RTT_STAT_NUM,dummy,DW_RTT_STAT_NUM,dummy,USER_PROBE_U
 L_LOST_PKT,dummy,SERVER_PROBE_UL_LOST_PKT,dummy,SERVER_PROBE_DW_LOST_PKT,dummy,USER_PROBE_DW_LOST_PKT,dummy,AVG_DW_RTT_MICRO_SEC,dummy,AVG_UL_RTT_MICRO_SEC,dummy,RAN_NE_ID,dummy,AVG_UL_RTT,dummy,AVG_DW_RTT,dummy,CHARGING_CHARACTERISTICS,dummy,DL_SERIOUS_OUT_OF_ORDER_NUM,dummy,DL_SLIGHT_OUT_OF_ORDER_NUM,dummy,DL_FLIGHT_TOTAL_SIZE,dummy,DL_FLIGHT_TOTAL_NUM,dummy,DL_MAX_FLIGHT_SIZE,dummy,VALID_TRANS_DURATION,dummy,AIR_PORT_DURATION,dummy,RADIO_CONN_TIMES,dummy,UL_SERIOUS_OUT_OF_ORDER_NUM,dummy,UL_SLIGHT_OUT_OF_ORDER_NUM,dummy,UL_FLIGHT_TOTAL_SIZE,dummy,UL_FLIGHT_TOTAL_NUM,dummy,UL_MAX_FLIGHT_SIZE,dummy,USER_DL_SLIGHT_OUT_OF_ORDER_PACKETS,dummy,SERVER_UL_SLIGHT_OUT_OF_ORDER_PACKETS,dummy,DL_CONTINUOUS_RETRANSMISSION_DELAY,dummy,USER_HUNGRY_DELAY,dummy,SERVER_HUNGRY_DELAY,dummy,UPPERLAYER_IP_UL_FRAGMENTS,dummy,UPPERLAYER_IP_DL_FRAGMENTS,dummy,DOWNLAYER_IP_UL_FRAGMENTS,dummy,DOWNLAYER_IP_DL_FRAGMENTS,dummy,UPPERLAYER_IP_UL_PACKETS,dummy,UPPERLAYER_IP_DL_PACKETS,dummy,DOWNLAYER_IP_UL_PACKE
 TS,dummy,DOWNLAYER_IP_DL_PACKETS,dummy,TCP_UL_PACKAGES_WITHPL,dummy,TCP_DW_PACKAGES_WITHPL,dummy,TCP_UL_PACKAGES_WITHOUTPL,dummy,TCP_DW_PACKAGES_WITHOUTPL,dummy,TCP_UL_RETRANS_WITHPL,dummy,TCP_DW_RETRANS_WITHPL,L4_UL_THROUGHPUT,L4_DW_THROUGHPUT,L4_UL_GOODPUT,L4_DW_GOODPUT,NETWORK_UL_TRAFFIC,NETWORK_DL_TRAFFIC,L4_UL_PACKETS,L4_DW_PACKETS,TCP_RTT,TCP_UL_OUTOFSEQU,TCP_DW_OUTOFSEQU,TCP_UL_RETRANS,TCP_DW_RETRANS')")
       assert(true)
     } catch {
-      case _: Throwable => assert(false)
+      case e: Throwable =>
+        e.printStackTrace()
+        assert(false)
     }
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala
index 59f7a98..9018ec0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala
@@ -53,7 +53,8 @@ class DateDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfterAll
       case x: Throwable =>
         x.printStackTrace()
         CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+        .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+          CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     }
   }
 
@@ -131,7 +132,8 @@ class DateDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfterAll
   override def afterAll {
     sql("drop table directDictionaryTable")
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala
index 463489b..195b107 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala
@@ -56,7 +56,8 @@ class DateDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with Befo
       case x: Throwable =>
         x.printStackTrace()
         CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+        .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+          CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     }
   }
 
@@ -94,7 +95,8 @@ class DateDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with Befo
   override def afterAll {
     sql("drop table directDictionaryTable")
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala
index d00a108..20e1d2d 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala
@@ -52,7 +52,8 @@ class DateDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll {
       case x: Throwable =>
         x.printStackTrace()
         CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+        .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+          CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     }
   }
 
@@ -73,7 +74,8 @@ class DateDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll {
   override def afterAll {
     sql("drop table timestampTyeNullData")
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index ca337cc..b8d057c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -64,7 +64,8 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
       sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive")
     } catch {
       case x: Throwable => CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+          CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     }
   }
 
@@ -144,7 +145,8 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
     sql("drop table directDictionaryTable")
     sql("drop table directDictionaryTable_hive")
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
index 26f9ff7..4878c86 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
@@ -56,7 +56,8 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with
         + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')");
     } catch {
       case x: Throwable => CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+          CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     }
   }
 
@@ -91,7 +92,8 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with
   override def afterAll {
     sql("drop table directDictionaryTable")
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
index 9b07b1e..bc3d6dd 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
@@ -58,7 +58,8 @@ class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll {
 
     } catch {
       case x: Throwable => CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+          CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     }
   }
 
@@ -79,7 +80,8 @@ class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll {
   override def afterAll {
     sql("drop table timestampTyeNullData")
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
     CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala
index 4aa6954..54586c2 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala
@@ -39,8 +39,8 @@ class TestAllDataTypeForPartitionTable extends QueryTest with BeforeAndAfterAll
 
   override def afterAll = {
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat)
-      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+        CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
 
     dropTable
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala
index a05d908..0217486 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala
@@ -71,8 +71,6 @@ class TestCompactionForPartitionTable extends QueryTest with BeforeAndAfterAll {
 
   override def afterAll = {
     dropTable
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat)
   }
 
   def dropTable = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
index 41289b1..4ae5ece 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
@@ -356,8 +356,6 @@ class TestDDLForPartitionTable  extends QueryTest with BeforeAndAfterAll {
 
   override def afterAll = {
     dropTable
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat)
   }
 
   def dropTable = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
index 4bfc4ef..f7ac87c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
@@ -371,8 +371,6 @@ class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll
 
   override def afterAll = {
     dropTable
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat)
   }
 
   def dropTable = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala
index e02c6bf..df0eea3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala
@@ -272,8 +272,6 @@ class TestQueryForPartitionTable  extends QueryTest with BeforeAndAfterAll {
 
   override def afterAll = {
     dropTable
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat)
   }
 
   def dropTable = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala b/integration/spark-common-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala
new file mode 100644
index 0000000..7d6770b
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under onassemblye
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.common.util
+
+import org.scalatest.Tag
+
+object Include extends Tag("Include")
+
+object Exclude extends Tag("Exclude")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index ac42bf3..4e75334 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -129,6 +129,41 @@
           <target>1.7</target>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <version>3.0.0</version>
+        <executions>
+          <execution>
+            <id>compile</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>copy</goal>
+            </goals>
+            <configuration>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>org.apache.carbondata</groupId>
+                  <artifactId>carbondata-format</artifactId>
+                  <version>${project.version}</version>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>com.google.code.gson</groupId>
+                  <artifactId>gson</artifactId>
+                  <version>2.3.1</version>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>org.xerial.snappy</groupId>
+                  <artifactId>snappy-java</artifactId>
+                  <version>${snappy.version}</version>
+                </artifactItem>
+              </artifactItems>
+              <outputDirectory>${project.build.directory}/jars</outputDirectory>
+              <!-- other configurations here -->
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
index c908c08..b137d6d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
@@ -19,13 +19,14 @@ package org.apache.carbondata.spark.rdd
 
 import java.io.{ByteArrayInputStream, DataInputStream}
 
+import scala.collection.JavaConverters._
 import scala.reflect.ClassTag
 
 import org.apache.spark.{Dependency, OneToOneDependency, Partition, SparkContext, TaskContext}
 import org.apache.spark.rdd.RDD
 
 import org.apache.carbondata.core.metadata.schema.table.TableInfo
-import org.apache.carbondata.core.util.{CarbonSessionInfo, CarbonTaskInfo, SessionParams, ThreadLocalSessionInfo, ThreadLocalTaskInfo}
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, CarbonTaskInfo, SessionParams, ThreadLocalSessionInfo, ThreadLocalTaskInfo}
 
 /**
  * This RDD maintains session level ThreadLocal
@@ -35,6 +36,8 @@ abstract class CarbonRDD[T: ClassTag](@transient sc: SparkContext,
 
   val carbonSessionInfo: CarbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
 
+//  val addedProperty = CarbonProperties.getInstance().getAddedProperty
+
   /** Construct an RDD with just a one-to-one dependency on one parent */
   def this(@transient oneParent: RDD[_]) =
     this (oneParent.context, List(new OneToOneDependency(oneParent)))
@@ -47,6 +50,7 @@ abstract class CarbonRDD[T: ClassTag](@transient sc: SparkContext,
     val carbonTaskInfo = new CarbonTaskInfo
     carbonTaskInfo.setTaskId(System.nanoTime)
     ThreadLocalTaskInfo.setCarbonTaskInfo(carbonTaskInfo)
+//    addedProperty.asScala.map(f => CarbonProperties.getInstance().addProperty(f._1, f._2))
     internalCompute(split, context)
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common/src/main/scala/org/apache/spark/sql/test/ResourceRegisterAndCopier.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/ResourceRegisterAndCopier.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/ResourceRegisterAndCopier.scala
new file mode 100644
index 0000000..6364630
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/ResourceRegisterAndCopier.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.test
+
+import java.io.{BufferedReader, File, FileReader}
+import java.net.URL
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.hadoop.io.IOUtils
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonUtil
+
+/**
+ * All the registered resources will be checked into hdfs and copies to it if not exists.
+ */
+object ResourceRegisterAndCopier {
+
+  private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+
+  val link = "https://raw.githubusercontent" +
+             ".com/ravipesala/incubator-carbondata/sdv-test_data/integration/spark-common-test" +
+             "/src/test/resources"
+
+  def copyResourcesifNotExists(hdfsPath: String,
+      resourcePath: String,
+      dataFilesPath: String): Unit = {
+    val fileType = FileFactory.getFileType(hdfsPath)
+    val file = FileFactory.getCarbonFile(hdfsPath, fileType)
+    if (!file.exists()) {
+      sys.error(s"""Provided path $hdfsPath does not exist""")
+    }
+    val resources = readDataFiles(dataFilesPath)
+    resources.foreach { file =>
+      val hdfsDataPath = hdfsPath + "/" + file
+      val rsFile = FileFactory.getCarbonFile(hdfsDataPath, fileType)
+      if (!rsFile.exists()) {
+        val target = resourcePath + "/" + file
+        new File(resourcePath + "/" + file.substring(0, file.lastIndexOf("/"))).mkdirs()
+        downloadFile(link, file, target)
+        // copy it
+        copyLocalFile(hdfsDataPath, target)
+        new File(target).delete()
+      }
+    }
+  }
+
+  def readDataFiles(dataFilesPath: String): Seq[String] = {
+    val buffer = new ArrayBuffer[String]()
+    val reader = new BufferedReader(new FileReader(dataFilesPath))
+    var line = reader.readLine()
+    while (line != null) {
+      buffer += line
+      line = reader.readLine()
+    }
+    reader.close()
+    buffer
+  }
+
+  def copyLocalFile(dst: String,
+      src: String): Unit = {
+    LOGGER.info(s"Copying file : $src to  $dst")
+    if (FileFactory.isFileExist(src, FileFactory.getFileType(src))) {
+      val dataOutputStream = FileFactory.getDataOutputStream(dst,
+        FileFactory.getFileType(dst))
+      val dataInputStream = FileFactory.getDataInputStream(src,
+        FileFactory.getFileType(src))
+      IOUtils.copyBytes(dataInputStream, dataOutputStream, 8 * 1024)
+      CarbonUtil.closeStream(dataInputStream)
+      CarbonUtil.closeStream(dataOutputStream)
+    }
+  }
+
+  def downloadFile(relativeLink: String, fileToDownLoad: String, targetFile: String): Unit = {
+    import java.io.FileOutputStream
+    val link = relativeLink + "/" + fileToDownLoad
+    LOGGER.info(s"Downloading file $link")
+    val url = new URL(link)
+    val c = url.openConnection
+    c.setRequestProperty("User-Agent",
+      "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322;" +
+      " .NET CLR 1.2.30703)")
+
+    var input = c.getInputStream
+    val buffer = new Array[Byte](4096)
+    var n = input.read(buffer)
+
+    val output = new FileOutputStream(new File(targetFile))
+    while (n != -1) {
+      output.write(buffer, 0, n)
+      n = input.read(buffer)
+    }
+    output.close()
+    input.close()
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala
index 149e3b1..cf90912 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala
@@ -17,14 +17,18 @@
 
 package org.apache.spark.sql.test
 
-import java.io.File
+import java.io.{File, FilenameFilter}
 import java.util.ServiceLoader
 
+import scala.collection.mutable.ArrayBuffer
+import scala.util.Random
+
 import org.apache.spark.sql.{DataFrame, SQLContext}
 import org.apache.spark.util.Utils
 
 import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
 import org.apache.carbondata.core.util.CarbonProperties
 
 /**
@@ -33,6 +37,8 @@ import org.apache.carbondata.core.util.CarbonProperties
 trait TestQueryExecutorRegister {
   def sql(sqlText: String): DataFrame
 
+  def stop()
+
   def sqlContext: SQLContext
 }
 
@@ -44,16 +50,99 @@ object TestQueryExecutor {
     .getCanonicalPath
   LOGGER.info(s"project path: $projectPath")
   val integrationPath = s"$projectPath/integration"
-  val resourcesPath = s"$integrationPath/spark-common-test/src/test/resources"
-  val storeLocation = s"$integrationPath/spark-common/target/store"
-  val warehouse = s"$integrationPath/spark-common/target/warehouse"
   val metastoredb = s"$integrationPath/spark-common/target"
-  val timestampFormat = "dd-MM-yyyy"
+  val masterUrl = {
+    val property = System.getProperty("spark.master.url")
+    if (property == null) {
+      "local[2]"
+    } else {
+      property
+    }
+  }
+
+  val hdfsUrl = {
+    val property = System.getProperty("hdfs.url")
+    if (property == null) {
+      "local"
+    } else {
+      LOGGER.info("HDFS PATH given : " + property)
+      property
+    }
+  }
+
+  val resourcesPath = if (hdfsUrl.startsWith("hdfs://")) {
+    ResourceRegisterAndCopier.
+      copyResourcesifNotExists(hdfsUrl, s"$integrationPath/spark-common-test/src/test/resources",
+        s"$integrationPath//spark-common-cluster-test/src/test/resources/testdatafileslist.txt")
+    hdfsUrl
+  } else {
+    s"$integrationPath/spark-common-test/src/test/resources"
+  }
+
+  val storeLocation = if (hdfsUrl.startsWith("hdfs://")) {
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOCK_TYPE,
+      CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
+    val carbonFile = FileFactory.
+      getCarbonFile(s"$hdfsUrl/store", FileFactory.getFileType(s"$hdfsUrl/store"))
+    FileFactory.deleteAllCarbonFilesOfDir(carbonFile)
+    s"$hdfsUrl/store_" + System.nanoTime()
+  } else {
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOCK_TYPE,
+      CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL)
+    s"$integrationPath/spark-common/target/store"
+  }
+  val warehouse = if (hdfsUrl.startsWith("hdfs://")) {
+    val carbonFile = FileFactory.
+      getCarbonFile(s"$hdfsUrl/warehouse", FileFactory.getFileType(s"$hdfsUrl/warehouse"))
+    FileFactory.deleteAllCarbonFilesOfDir(carbonFile)
+    s"$hdfsUrl/warehouse_" + System.nanoTime()
+  } else {
+    s"$integrationPath/spark-common/target/warehouse"
+  }
+
+  val hiveresultpath = if (hdfsUrl.startsWith("hdfs://")) {
+    val p = s"$hdfsUrl/hiveresultpath"
+    FileFactory.mkdirs(p, FileFactory.getFileType(p))
+    p
+  } else {
+    val p = s"$integrationPath/spark-common/target/hiveresultpath"
+    new File(p).mkdirs()
+    p
+  }
+
+  LOGGER.info(s"""Store path taken $storeLocation""")
+  LOGGER.info(s"""Warehouse path taken $warehouse""")
+  LOGGER.info(s"""Resource path taken $resourcesPath""")
+
+  lazy val modules = Seq(TestQueryExecutor.projectPath + "/common/target",
+    TestQueryExecutor.projectPath + "/core/target",
+    TestQueryExecutor.projectPath + "/hadoop/target",
+    TestQueryExecutor.projectPath + "/processing/target",
+    TestQueryExecutor.projectPath + "/integration/spark-common/target",
+    TestQueryExecutor.projectPath + "/integration/spark2/target",
+    TestQueryExecutor.projectPath + "/integration/spark-common/target/jars")
+  lazy val jars = {
+    val jarsLocal = new ArrayBuffer[String]()
+    modules.foreach { path =>
+      val files = new File(path).listFiles(new FilenameFilter {
+        override def accept(dir: File, name: String) = {
+          name.endsWith(".jar")
+        }
+      })
+      files.foreach(jarsLocal += _.getAbsolutePath)
+    }
+    jarsLocal
+  }
 
   val INSTANCE = lookupQueryExecutor.newInstance().asInstanceOf[TestQueryExecutorRegister]
   CarbonProperties.getInstance()
     .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FORCE")
     .addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords")
+    .addProperty(CarbonCommonConstants.DICTIONARY_SERVER_PORT,
+      (CarbonCommonConstants.DICTIONARY_SERVER_PORT_DEFAULT.toInt + Random.nextInt(100)) + "")
+    .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "1024")
+      .addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, "1024")
+
   private def lookupQueryExecutor: Class[_] = {
     ServiceLoader.load(classOf[TestQueryExecutorRegister], Utils.getContextOrSparkClassLoader)
       .iterator().next().getClass

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml
index 04b5aa7..fc13c7a 100644
--- a/integration/spark/pom.xml
+++ b/integration/spark/pom.xml
@@ -188,5 +188,13 @@
       </plugin>
     </plugins>
   </build>
+  <profiles>
+    <profile>
+      <id>sdvtest</id>
+      <properties>
+        <maven.test.skip>true</maven.test.skip>
+      </properties>
+    </profile>
+  </profiles>
 
 </project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala b/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
index 27df623..bb00126 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala
@@ -32,13 +32,14 @@ class SparkTestQueryExecutor extends TestQueryExecutorRegister {
   override def sql(sqlText: String): DataFrame = SparkTestQueryExecutor.cc.sql(sqlText)
 
   override def sqlContext: SQLContext = SparkTestQueryExecutor.cc
+
+  override def stop(): Unit = SparkTestQueryExecutor.cc.sparkContext.stop()
 }
 
 object SparkTestQueryExecutor {
   private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
   LOGGER.info("use TestQueryExecutorImplV1")
   CarbonProperties.getInstance()
-    .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat)
     .addProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
       System.getProperty("java.io.tmpdir"))
     .addProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL)

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index 8279c5d..2f34c34 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -143,6 +143,7 @@
         <version>1.0</version>
         <!-- Note config is repeated in surefire config -->
         <configuration>
+          <skipTests>false</skipTests>
           <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
           <junitxml>.</junitxml>
           <filereports>CarbonTestSuite.txt</filereports>
@@ -166,5 +167,13 @@
       </plugin>
     </plugins>
   </build>
+  <profiles>
+    <profile>
+      <id>sdvtest</id>
+      <properties>
+        <maven.test.skip>true</maven.test.skip>
+      </properties>
+    </profile>
+  </profiles>
 
 </project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index c9eaf6d..a32146a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.cache.dictionary.ManageDictionaryAndBTree
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.datastore.impl.FileFactory
 import org.apache.carbondata.core.fileoperations.FileWriteOperation
+import org.apache.carbondata.core.indexstore.DataMapStoreManager
 import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
 import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
 import org.apache.carbondata.core.metadata.schema
@@ -425,6 +426,7 @@ class CarbonFileMetastore(conf: RuntimeConfig) extends CarbonMetaStore {
       CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sparkSession)
       // discard cached table info in cachedDataSourceTables
       sparkSession.sessionState.catalog.refreshTable(tableIdentifier)
+      DataMapStoreManager.getInstance().clearDataMap(identifier, "blocklet")
     }
   }
 


Mime
View raw message