carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [07/38] incubator-carbondata git commit: reuse test case for integration module
Date Sat, 07 Jan 2017 16:36:41 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
deleted file mode 100644
index ad695f4..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ /dev/null
@@ -1,695 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.dataload
-
-import java.io.File
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.scalatest.BeforeAndAfterAll
-
-/**
-  * Test Class for data loading with hive syntax and old syntax
-  *
-  */
-class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll {
-    sql("drop table if exists escapechar1")
-    sql("drop table if exists escapechar2")
-    sql("drop table if exists escapechar3")
-    sql("drop table if exists specialcharacter1")
-    sql("drop table if exists specialcharacter2")
-    sql("drop table if exists collessthanschema")
-    sql("drop table if exists decimalarray")
-    sql("drop table if exists decimalstruct")
-    sql("drop table if exists carbontable")
-    sql("drop table if exists hivetable")
-    sql("drop table if exists testtable")
-    sql("drop table if exists testhivetable")
-    sql("drop table if exists testtable1")
-    sql("drop table if exists testhivetable1")
-    sql("drop table if exists complexcarbontable")
-    sql("drop table if exists complex_t3")
-    sql("drop table if exists complex_hive_t3")
-    sql("drop table if exists header_test")
-    sql("drop table if exists duplicateColTest")
-    sql("drop table if exists mixed_header_test")
-    sql("drop table if exists primitivecarbontable")
-    sql("drop table if exists UPPERCASEcube")
-    sql("drop table if exists lowercaseCUBE")
-    sql("drop table if exists carbontable1")
-    sql("drop table if exists hivetable1")
-    sql("drop table if exists comment_test")
-    sql("drop table if exists smallinttable")
-    sql("drop table if exists smallinthivetable")
-    sql(
-      "CREATE table carbontable (empno int, empname String, designation String, doj String, " +
-        "workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, " +
-        "projectcode int, projectjoindate String, projectenddate String, attendance int," +
-        "utilization int,salary int) STORED BY 'org.apache.carbondata.format'"
-    )
-    sql(
-      "create table hivetable(empno int, empname String, designation string, doj String, " +
-        "workgroupcategory int, workgroupcategoryname String,deptno int, deptname String, " +
-        "projectcode int, projectjoindate String,projectenddate String, attendance String," +
-        "utilization String,salary String)row format delimited fields terminated by ','"
-    )
-
-  }
-
-  test("create table with smallint type and query smallint table") {
-    sql(
-      "create table smallinttable(empno smallint, empname String, designation string, " +
-        "doj String, workgroupcategory int, workgroupcategoryname String,deptno int, " +
-        "deptname String, projectcode int, projectjoindate String,projectenddate String, " +
-        "attendance String, utilization String,salary String)" +
-        "STORED BY 'org.apache.carbondata.format'"
-    )
-
-    sql(
-      "create table smallinthivetable(empno smallint, empname String, designation string, " +
-        "doj String, workgroupcategory int, workgroupcategoryname String,deptno int, " +
-        "deptname String, projectcode int, projectjoindate String,projectenddate String, " +
-        "attendance String, utilization String,salary String)" +
-        "row format delimited fields terminated by ','"
-    )
-
-    sql("LOAD DATA local inpath './src/test/resources/data.csv' INTO table smallinttable " +
-      "OPTIONS('USE_KETTLE'='false')")
-    sql("LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' overwrite " +
-      "INTO table smallinthivetable")
-
-    checkAnswer(
-      sql("select empno from smallinttable"),
-      sql("select empno from smallinthivetable")
-    )
-
-    sql("drop table if exists smallinttable")
-    sql("drop table if exists smallinthivetable")
-  }
-
-  test("test data loading and validate query output") {
-    //Create test cube and hive table
-    sql(
-      "CREATE table testtable (empno string, empname String, designation String, doj String, " +
-        "workgroupcategory string, workgroupcategoryname String, deptno string, deptname String, " +
-        "projectcode string, projectjoindate String, projectenddate String,attendance double," +
-        "utilization double,salary double) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES" +
-        "('DICTIONARY_EXCLUDE'='empno,empname,designation,doj,workgroupcategory," +
-        "workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate')"
-    )
-    sql(
-      "create table testhivetable(empno string, empname String, designation string, doj String, " +
-        "workgroupcategory string, workgroupcategoryname String,deptno string, deptname String, " +
-        "projectcode string, projectjoindate String,projectenddate String, attendance double," +
-        "utilization double,salary double)row format delimited fields terminated by ','"
-    )
-    //load data into test cube and hive table and validate query result
-    sql("LOAD DATA local inpath './src/test/resources/data.csv' INTO table testtable")
-    sql(
-      "LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' overwrite INTO table " +
-        "testhivetable"
-    )
-    checkAnswer(sql("select * from testtable"), sql("select * from testhivetable"))
-    //load data incrementally and validate query result
-    sql(
-      "LOAD DATA local inpath './src/test/resources/data.csv' INTO TABLE testtable OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-    sql(
-      "LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' INTO table testhivetable"
-    )
-    checkAnswer(sql("select * from testtable"), sql("select * from testhivetable"))
-    //drop test cube and table
-    sql("drop table if exists testtable")
-    sql("drop table if exists testhivetable")
-  }
-
-  /**
-    * TODO: temporarily changing cube names to different names,
-    * however deletion and creation of cube with same name
-    */
-  test("test data loading with different case file header and validate query output") {
-    //Create test cube and hive table
-    sql(
-      "CREATE table testtable1 (empno string, empname String, designation String, doj String, " +
-        "workgroupcategory string, workgroupcategoryname String, deptno string, deptname String, " +
-        "projectcode string, projectjoindate String, projectenddate String,attendance double," +
-        "utilization double,salary double) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES" +
-        "('DICTIONARY_EXCLUDE'='empno,empname,designation,doj,workgroupcategory," +
-        "workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate')"
-    )
-    sql(
-      "create table testhivetable1(empno string, empname String, designation string, doj String, " +
-        "workgroupcategory string, workgroupcategoryname String,deptno string, deptname String, " +
-        "projectcode string, projectjoindate String,projectenddate String, attendance double," +
-        "utilization double,salary double)row format delimited fields terminated by ','"
-    )
-    //load data into test cube and hive table and validate query result
-    sql(
-      "LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' INTO table testtable1 " +
-        "options('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='EMPno, empname,designation,doj," +
-        "workgroupcategory,workgroupcategoryname,   deptno,deptname,projectcode,projectjoindate," +
-        "projectenddate,  attendance,   utilization,SALARY')"
-    )
-    sql(
-      "LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' overwrite INTO table " +
-        "testhivetable1"
-    )
-    checkAnswer(sql("select * from testtable1"), sql("select * from testhivetable1"))
-    //drop test cube and table
-    sql("drop table if exists testtable1")
-    sql("drop table if exists testhivetable1")
-  }
-
-  test("test hive table data loading") {
-    sql(
-      "LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' overwrite INTO table " +
-        "hivetable"
-    )
-    sql("LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' INTO table hivetable")
-  }
-
-  test("test carbon table data loading using old syntax") {
-    sql(
-      "LOAD DATA local inpath './src/test/resources/data.csv' INTO TABLE carbontable OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
-    )
-  }
-
-  test("test carbon table data loading using new syntax compatible with hive") {
-    sql("LOAD DATA local inpath './src/test/resources/data.csv' INTO table carbontable")
-    sql(
-      "LOAD DATA local inpath './src/test/resources/data.csv' INTO table carbontable options" +
-        "('DELIMITER'=',', 'QUOTECHAR'='\"')"
-    )
-  }
-
-  test("test carbon table data loading using new syntax with overwrite option compatible with hive")
-  {
-    try {
-      sql("LOAD DATA local inpath './src/test/resources/data.csv' overwrite INTO table carbontable")
-    } catch {
-      case e: Throwable => {
-        assert(e.getMessage
-          .equals("Overwrite is not supported for carbon table with default.carbontable")
-        )
-      }
-    }
-  }
-
-  test("complex types data loading") {
-    sql("create table complexcarbontable(deviceInformationId int, channelsId string," +
-      "ROMSize string, purchasedate string, mobile struct<imei:string, imsi:string>," +
-      "MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
-      "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>," +
-      "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
-      "double,contractNumber double) " +
-      "STORED BY 'org.apache.carbondata.format' " +
-      "TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId')"
-    )
-    sql(
-      "LOAD DATA local inpath './src/test/resources/complexdata.csv' INTO table " +
-        "complexcarbontable " +
-        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId," +
-        "ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber'," +
-        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')"
-    )
-    sql("drop table if exists complexcarbontable")
-  }
-
-  test(
-    "complex types data loading with more unused columns and different order of complex columns " +
-      "in csv and create table"
-  ) {
-    sql("create table complexcarbontable(deviceInformationId int, channelsId string," +
-      "mobile struct<imei:string, imsi:string>, ROMSize string, purchasedate string," +
-      "MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
-      "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>," +
-      "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
-      "double,contractNumber double) " +
-      "STORED BY 'org.apache.carbondata.format' " +
-      "TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId','DICTIONARY_EXCLUDE'='channelsId')"
-    )
-    sql(
-      "LOAD DATA local inpath './src/test/resources/complextypediffentcolheaderorder.csv' INTO " +
-        "table complexcarbontable " +
-        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId," +
-        "ROMSize,purchasedate,MAC,abc,mobile,locationinfo,proddate,gamePointId,contractNumber'," +
-        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')"
-    )
-    sql("select count(*) from complexcarbontable")
-    sql("drop table if exists complexcarbontable")
-  }
-
-  test("test carbon table data loading with csv file Header in caps") {
-    sql("drop table if exists header_test")
-    sql(
-      "create table header_test(empno int, empname String, designation string, doj String, " +
-        "workgroupcategory int, workgroupcategoryname String,deptno int, deptname String, " +
-        "projectcode int, projectjoindate String,projectenddate String, attendance String," +
-        "utilization String,salary String) STORED BY 'org.apache.carbondata.format'"
-    )
-    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-    val csvFilePath = currentDirectory + "/src/test/resources/data_withCAPSHeader.csv"
-    sql("LOAD DATA local inpath '" + csvFilePath + "' INTO table header_test OPTIONS " +
-      "('DELIMITER'=',', 'QUOTECHAR'='\"')");
-    checkAnswer(sql("select empno from header_test"),
-      Seq(Row(11), Row(12))
-    )
-  }
-
-  test("test duplicate column validation") {
-    try {
-      sql("create table duplicateColTest(col1 string, Col1 string)")
-    }
-    catch {
-      case e: Exception => {
-        assert(e.getMessage.contains("Duplicate column name"))
-      }
-    }
-  }
-
-  test(
-    "test carbon table data loading with csv file Header in Mixed Case and create table columns " +
-      "in mixed case"
-  ) {
-    sql("drop table if exists mixed_header_test")
-    sql(
-      "create table mixed_header_test(empno int, empname String, Designation string, doj String, " +
-        "Workgroupcategory int, workgroupcategoryname String,deptno int, deptname String, " +
-        "projectcode int, projectjoindate String,projectenddate String, attendance String," +
-        "utilization String,salary String) STORED BY 'org.apache.carbondata.format'"
-    )
-    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-    val csvFilePath = currentDirectory + "/src/test/resources/data_withMixedHeader.csv"
-    sql("LOAD DATA local inpath '" + csvFilePath + "' INTO table mixed_header_test OPTIONS " +
-      "('DELIMITER'=',', 'QUOTECHAR'='\"')");
-    checkAnswer(sql("select empno from mixed_header_test"),
-      Seq(Row(11), Row(12))
-    )
-  }
-
-
-  test("complex types data loading with hive column having more than required column values") {
-    sql("create table complexcarbontable(deviceInformationId int, channelsId string," +
-      "ROMSize string, purchasedate string, mobile struct<imei:string, imsi:string>," +
-      "MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
-      "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>," +
-      "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
-      "double,contractNumber double) " +
-      "STORED BY 'org.apache.carbondata.format' " +
-      "TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId')"
-    )
-    sql(
-      "LOAD DATA local inpath './src/test/resources/complexdatastructextra.csv' INTO table " +
-        "complexcarbontable " +
-        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId," +
-        "ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber'," +
-        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')"
-    )
-    sql("drop table if exists complexcarbontable")
-  }
-
-  test("complex types & no dictionary columns data loading") {
-    sql("create table complexcarbontable(deviceInformationId int, channelsId string," +
-      "ROMSize string, purchasedate string, mobile struct<imei:string, imsi:string>," +
-      "MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
-      "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>," +
-      "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
-      "double,contractNumber double) " +
-      "STORED BY 'org.apache.carbondata.format' " +
-      "TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId', 'DICTIONARY_EXCLUDE'='ROMSize," +
-      "purchasedate')"
-    )
-    sql(
-      "LOAD DATA local inpath './src/test/resources/complexdata.csv' INTO table " +
-        "complexcarbontable " +
-        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId," +
-        "ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber'," +
-        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')"
-    );
-    sql("drop table if exists complexcarbontable")
-  }
-
-  test("array<string> and string datatype for same column is not working properly") {
-    sql("create table complexcarbontable(deviceInformationId int, MAC array<string>, channelsId string, "+
-        "ROMSize string, purchasedate string, gamePointId double,contractNumber double) STORED BY 'org.apache.carbondata.format' "+
-        "TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId')")
-    sql("LOAD DATA local inpath './src/test/resources/complexdatareordered.csv' INTO table complexcarbontable "+
-        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,MAC,channelsId,ROMSize,purchasedate,gamePointId,contractNumber',"+
-        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
-    sql("drop table if exists complexcarbontable")
-    sql("create table primitivecarbontable(deviceInformationId int, MAC string, channelsId string, "+
-        "ROMSize string, purchasedate string, gamePointId double,contractNumber double) STORED BY 'org.apache.carbondata.format' "+
-        "TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId')")
-    sql("LOAD DATA local inpath './src/test/resources/complexdatareordered.csv' INTO table primitivecarbontable "+
-        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,MAC,channelsId,ROMSize,purchasedate,gamePointId,contractNumber',"+
-        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
-    sql("drop table if exists primitivecarbontable")
-  }
-
-  test(
-    "test carbon table data loading when table name is in different case with create table, for " +
-      "UpperCase"
-  ) {
-    sql("create table UPPERCASEcube(empno Int, empname String, designation String, " +
-      "doj String, workgroupcategory Int, workgroupcategoryname String, deptno Int, " +
-      "deptname String, projectcode Int, projectjoindate String, projectenddate String, " +
-      "attendance Int,utilization Double,salary Double) STORED BY 'org.apache.carbondata.format'"
-    )
-    sql(
-      "LOAD DATA local inpath './src/test/resources/data.csv' INTO table uppercasecube OPTIONS" +
-        "('DELIMITER'=',', 'QUOTECHAR'='\"')"
-    )
-    sql("drop table if exists UpperCaseCube")
-  }
-
-  test(
-    "test carbon table data loading when table name is in different case with create table ,for " +
-      "LowerCase"
-  ) {
-    sql("create table lowercaseCUBE(empno Int, empname String, designation String, " +
-      "doj String, workgroupcategory Int, workgroupcategoryname String, deptno Int, " +
-      "deptname String, projectcode Int, projectjoindate String, projectenddate String, " +
-      "attendance Int,utilization Double,salary Double) STORED BY 'org.apache.carbondata.format'"
-    )
-    sql(
-      "LOAD DATA local inpath './src/test/resources/data.csv' INTO table LOWERCASECUBE OPTIONS" +
-        "('DELIMITER'=',', 'QUOTECHAR'='\"')"
-    )
-    sql("drop table if exists LowErcasEcube")
-  }
-
-  test("test carbon table data loading using escape char 1") {
-    sql("DROP TABLE IF EXISTS escapechar1")
-
-    sql(
-      """
-           CREATE TABLE IF NOT EXISTS escapechar1
-           (ID Int, date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int)
-           STORED BY 'org.apache.carbondata.format'
-      """
-    )
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
-    sql(
-      s"""
-           LOAD DATA LOCAL INPATH './src/test/resources/datawithbackslash.csv' into table escapechar1
-           OPTIONS('ESCAPECHAR'='@')
-        """
-    )
-    checkAnswer(sql("select count(*) from escapechar1"), Seq(Row(10)))
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
-    sql("DROP TABLE IF EXISTS escapechar1")
-  }
-
-  test("test carbon table data loading using escape char 2") {
-    sql("DROP TABLE IF EXISTS escapechar2")
-
-    sql(
-      """
-         CREATE TABLE escapechar2(imei string,specialchar string)
-         STORED BY 'org.apache.carbondata.format'
-      """
-    )
-
-    sql(
-      """
-       LOAD DATA LOCAL INPATH './src/test/resources/datawithescapecharacter.csv' into table escapechar2
-          options ('DELIMITER'=',', 'QUOTECHAR'='"','ESCAPECHAR'='\')
-      """
-    )
-    checkAnswer(sql("select count(*) from escapechar2"), Seq(Row(21)))
-    checkAnswer(sql("select specialchar from escapechar2 where imei = '1AA44'"), Seq(Row("escapeesc")))
-    sql("DROP TABLE IF EXISTS escapechar2")
-  }
-
-  test("test carbon table data loading using escape char 3") {
-    sql("DROP TABLE IF EXISTS escapechar3")
-
-    sql(
-      """
-         CREATE TABLE escapechar3(imei string,specialchar string)
-         STORED BY 'org.apache.carbondata.format'
-      """
-    )
-
-    sql(
-      """
-       LOAD DATA LOCAL INPATH './src/test/resources/datawithescapecharacter.csv' into table escapechar3
-          options ('DELIMITER'=',', 'QUOTECHAR'='"','ESCAPECHAR'='@')
-      """
-    )
-    checkAnswer(sql("select count(*) from escapechar3"), Seq(Row(21)))
-    checkAnswer(sql("select specialchar from escapechar3 where imei in ('1232','12323')"), Seq(Row
-    ("ayush@b.com"), Row("ayushb.com")
-    )
-    )
-    sql("DROP TABLE IF EXISTS escapechar3")
-  }
-
-  test("test carbon table data loading with special character 1") {
-    sql("DROP TABLE IF EXISTS specialcharacter1")
-
-    sql(
-      """
-         CREATE TABLE specialcharacter1(imei string,specialchar string)
-         STORED BY 'org.apache.carbondata.format'
-      """
-    )
-
-    sql(
-      """
-       LOAD DATA LOCAL INPATH './src/test/resources/datawithspecialcharacter.csv' into table specialcharacter1
-          options ('DELIMITER'=',', 'QUOTECHAR'='"')
-      """
-    )
-    checkAnswer(sql("select count(*) from specialcharacter1"), Seq(Row(37)))
-    checkAnswer(sql("select specialchar from specialcharacter1 where imei='1AA36'"), Seq(Row("\"i\"")))
-    sql("DROP TABLE IF EXISTS specialcharacter1")
-  }
-
-  test("test carbon table data loading with special character 2") {
-    sql("DROP TABLE IF EXISTS specialcharacter2")
-
-    sql(
-      """
-        CREATE table specialcharacter2(customer_id int, 124_string_level_province String, date_level String,
-        Time_level String, lname String, fname String, mi String, address1 String, address2
-        String, address3 String, address4 String, city String, country String, phone1 String,
-        phone2 String, marital_status String, yearly_income String, gender String, education
-        String, member_card String, occupation String, houseowner String, fullname String,
-        numeric_level double, account_num double, customer_region_id int, total_children int,
-        num_children_at_home int, num_cars_owned int)
-        STORED BY 'org.apache.carbondata.format'
-      """
-    )
-
-    sql(
-      """
-       LOAD DATA LOCAL INPATH './src/test/resources/datawithcomplexspecialchar.csv' into
-       table specialcharacter2 options ('DELIMITER'=',', 'QUOTECHAR'='"','ESCAPECHAR'='"')
-      """
-    )
-    checkAnswer(sql("select count(*) from specialcharacter2"), Seq(Row(150)))
-    checkAnswer(sql("select 124_string_level_province from specialcharacter2 where customer_id=103"),
-      Seq(Row("\"state province # 124\""))
-    )
-    sql("DROP TABLE IF EXISTS specialcharacter2")
-  }
-
-  test("test data which contain column less than schema"){
-    sql("DROP TABLE IF EXISTS collessthanschema")
-
-    sql(
-      """
-           CREATE TABLE IF NOT EXISTS collessthanschema
-           (ID Int, date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int)
-           STORED BY 'org.apache.carbondata.format'
-      """)
-
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
-    sql(s"""
-         LOAD DATA LOCAL INPATH './src/test/resources/lessthandatacolumndata.csv' into table collessthanschema
-        """)
-    checkAnswer(sql("select count(*) from collessthanschema"),Seq(Row(10)))
-    sql("DROP TABLE IF EXISTS collessthanschema")
-  }
-
-  test("test data which contain column with decimal data type in array."){
-    sql("DROP TABLE IF EXISTS decimalarray")
-
-    sql(
-      """
-           CREATE TABLE IF NOT EXISTS decimalarray
-           (ID decimal(5,5), date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int, complex
-           array<decimal(4,2)>)
-           STORED BY 'org.apache.carbondata.format'
-      """
-    )
-
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
-    sql(s"""
-         LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimal.csv' into table decimalarray
-        """)
-    checkAnswer(sql("select count(*) from decimalarray"),Seq(Row(8)))
-    sql("DROP TABLE IF EXISTS decimalarray")
-  }
-
-  test("test data which contain column with decimal data type in struct."){
-    sql("DROP TABLE IF EXISTS decimalstruct")
-
-    sql(
-      """
-           CREATE TABLE IF NOT EXISTS decimalstruct
-           (ID decimal(5,5), date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int, complex
-           struct<a:decimal(4,2)>)
-           STORED BY 'org.apache.carbondata.format'
-      """
-    )
-
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
-    sql(s"""
-         LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimal.csv' into table decimalstruct
-        """)
-    checkAnswer(sql("select count(*) from decimalstruct"),Seq(Row(8)))
-    sql("DROP TABLE IF EXISTS decimalstruct")
-  }
-
-  test("test data which contain column with decimal data type in array of struct."){
-    sql("DROP TABLE IF EXISTS complex_t3")
-    sql("DROP TABLE IF EXISTS complex_hive_t3")
-
-    sql(
-      """
-           CREATE TABLE complex_t3
-           (ID decimal, date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int, complex
-           array<struct<a:decimal(4,2),str:string>>)
-           STORED BY 'org.apache.carbondata.format'
-      """
-    )
-    sql(
-      """
-           CREATE TABLE complex_hive_t3
-           (ID decimal, date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int, complex
-           array<struct<a:decimal(4,2),str:string>>)
-           row format delimited fields terminated by ','
-      """
-    )
-
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
-    sql(s"""
-         LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNested.csv' into table complex_t3
-        """)
-    sql(s"""
-         LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNestedHive.csv' into table complex_hive_t3
-        """)
-    checkAnswer(sql("select count(*) from complex_t3"),sql("select count(*) from complex_hive_t3"))
-    checkAnswer(sql("select id from complex_t3 where salary = 15000"),sql("select id from complex_hive_t3 where salary = 15000"))
-  }
-
-  test("test data loading when delimiter is '|' and data with header") {
-    sql(
-      "CREATE table carbontable1 (empno string, empname String, designation String, doj String, " +
-        "workgroupcategory string, workgroupcategoryname String, deptno string, deptname String, " +
-        "projectcode string, projectjoindate String, projectenddate String,attendance double," +
-        "utilization double,salary double) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES" +
-        "('DICTIONARY_EXCLUDE'='empno,empname,designation,doj,workgroupcategory," +
-        "workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate')"
-    )
-    sql(
-      "create table hivetable1 (empno string, empname String, designation string, doj String, " +
-        "workgroupcategory string, workgroupcategoryname String,deptno string, deptname String, " +
-        "projectcode string, projectjoindate String,projectenddate String, attendance double," +
-        "utilization double,salary double)row format delimited fields terminated by ','"
-    )
-
-    sql(
-      "LOAD DATA local inpath './src/test/resources/datadelimiter.csv' INTO TABLE carbontable1 OPTIONS" +
-        "('DELIMITER'= '|', 'QUOTECHAR'= '\"')"
-    )
-
-    sql("LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' INTO table hivetable1")
-
-    checkAnswer(sql("select * from carbontable1"), sql("select * from hivetable1"))
-  }
-
-  test("test data loading with comment option") {
-    sql("drop table if exists comment_test")
-    sql(
-      "create table comment_test(imei string, age int, task bigint, num double, level decimal(10," +
-        "3), productdate timestamp, mark int, name string) STORED BY 'org.apache.carbondata.format'"
-    )
-    sql(
-      "LOAD DATA local inpath './src/test/resources/comment.csv' INTO TABLE comment_test " +
-        "options('DELIMITER' = ',', 'QUOTECHAR' = '.', 'COMMENTCHAR' = '?','FILEHEADER'='imei,age,task,num,level,productdate,mark,name')"
-    )
-    checkAnswer(sql("select imei from comment_test"),Seq(Row("\".carbon"),Row("#?carbon"), Row(""),
-      Row("~carbon,")))
-  }
-
-
-  override def afterAll {
-    sql("drop table if exists escapechar1")
-    sql("drop table if exists escapechar2")
-    sql("drop table if exists escapechar3")
-    sql("drop table if exists specialcharacter1")
-    sql("drop table if exists specialcharacter2")
-    sql("drop table if exists collessthanschema")
-    sql("drop table if exists decimalarray")
-    sql("drop table if exists decimalstruct")
-    sql("drop table if exists carbontable")
-    sql("drop table if exists hivetable")
-    sql("drop table if exists testtable")
-    sql("drop table if exists testhivetable")
-    sql("drop table if exists testtable1")
-    sql("drop table if exists testhivetable1")
-    sql("drop table if exists complexcarbontable")
-    sql("drop table if exists complex_t3")
-    sql("drop table if exists complex_hive_t3")
-    sql("drop table if exists header_test")
-    sql("drop table if exists duplicateColTest")
-    sql("drop table if exists mixed_header_test")
-    sql("drop table if exists primitivecarbontable")
-    sql("drop table if exists UPPERCASEcube")
-    sql("drop table if exists lowercaseCUBE")
-    sql("drop table if exists carbontable1")
-    sql("drop table if exists hivetable1")
-    sql("drop table if exists comment_test")
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
deleted file mode 100644
index 74e98f0..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.dataload
-
-import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.scalatest.BeforeAndAfterAll
-
-
-class TestLoadDataWithMalformedCarbonCommandException extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll {
-
-    sql("CREATE table TestLoadTableOptions (ID int, date String, country String, name String," +
-        "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format'")
-
-  }
-
-  override def afterAll {
-    sql("drop table TestLoadTableOptions")
-  }
-
-  def buildTableWithNoExistDictExclude() = {
-      sql(
-        """
-           CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int)
-           STORED BY 'org.apache.carbondata.format'
-           TBLPROPERTIES('DICTIONARY_EXCLUDE'='country,phonetype,CCC')
-        """)
-  }
-
-  def buildTableWithNoExistDictInclude() = {
-      sql(
-        """
-           CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int)
-           STORED BY 'org.apache.carbondata.format'
-           TBLPROPERTIES('DICTIONARY_INCLUDE'='AAA,country')
-        """)
-  }
-
-  def buildTableWithSameDictExcludeAndInclude() = {
-      sql(
-        """
-           CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int)
-           STORED BY 'org.apache.carbondata.format'
-           TBLPROPERTIES('DICTIONARY_INCLUDE'='country','DICTIONARY_EXCLUDE'='country')
-        """)
-  }
-
-  def buildTableWithSameDictExcludeAndIncludeWithSpaces() = {
-    sql(
-      """
-           CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Timestamp, country String,
-           name String, phonetype String, serialname String, salary Int)
-           STORED BY 'org.apache.carbondata.format'
-           TBLPROPERTIES('DICTIONARY_INCLUDE'='country','DICTIONARY_EXCLUDE'='country ')
-      """)
-  }
-
-  test("test load data with dictionary exclude columns which no exist in table.") {
-    try {
-      buildTableWithNoExistDictExclude()
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(e.getMessage.equals("DICTIONARY_EXCLUDE column: ccc does not exist in table. " +
-          "Please check create table statement."))
-      case _: Throwable => assert(false)
-    }
-  }
-
-  test("test load data with dictionary include columns which no exist in table.") {
-    try {
-      buildTableWithNoExistDictInclude()
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(e.getMessage.equals("DICTIONARY_INCLUDE column: aaa does not exist in table. " +
-          "Please check create table statement."))
-      case _: Throwable => assert(false)
-    }
-  }
-
-  test("test load data with dictionary include is same with dictionary exclude") {
-    try {
-      buildTableWithSameDictExcludeAndInclude()
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(e.getMessage.equals("DICTIONARY_EXCLUDE can not contain the same column: country " +
-          "with DICTIONARY_INCLUDE. Please check create table statement."))
-      case _: Throwable => assert(false)
-    }
-  }
-
-  test("test load data with invalid option") {
-    try {
-      sql("LOAD DATA LOCAL INPATH './src/test/resources/dataretention1.csv' INTO TABLE " +
-        "TestLoadTableOptions OPTIONS('QUOTECHAR'='\"', 'DELIMITERRR' =  ',')")
-      assert(false)
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(e.getMessage.equals("Error: Invalid option(s): delimiterrr"))
-      case _: Throwable => assert(false)
-    }
-  }
-
-  test("test load data with duplicate options") {
-    try {
-      sql("LOAD DATA LOCAL INPATH './src/test/resources/dataretention1.csv' INTO TABLE " +
-        "TestLoadTableOptions OPTIONS('DELIMITER' =  ',', 'quotechar'='\"', 'DELIMITER' =  '$')")
-      assert(false)
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(e.getMessage.equals("Error: Duplicate option(s): delimiter"))
-      case _: Throwable => assert(false)
-    }
-  }
-
-  test("test load data with case sensitive options") {
-    try {
-      sql(
-        "LOAD DATA local inpath './src/test/resources/dataretention1.csv' INTO table " +
-          "TestLoadTableOptions options('DeLIMITEr'=',', 'qUOtECHAR'='\"')"
-      )
-    } catch {
-      case _: Throwable => assert(false)
-    }
-  }
-
-  test("test load data with dictionary include is same with dictionary exclude with spaces") {
-    try {
-      buildTableWithSameDictExcludeAndIncludeWithSpaces()
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(e.getMessage.equals("DICTIONARY_EXCLUDE can not contain the same column: country " +
-          "with DICTIONARY_INCLUDE. Please check create table statement."))
-      case _: Throwable => assert(false)
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
deleted file mode 100644
index 6d5b34e..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.dataload
-
-import java.io.File
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-/**
-  * Test Class for data loading with hive syntax and old syntax
-  *
-  */
-class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll {
-    sql("DROP TABLE IF EXISTS nomeasureTest_sd")
-    sql(
-      "CREATE TABLE nomeasureTest (empno String, doj String) STORED BY 'org.apache.carbondata" +
-        ".format'"
-    )
-    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-    val testData = currentDirectory + "/src/test/resources/datasample.csv"
-    sql("LOAD DATA LOCAL INPATH '" + testData + "' into table nomeasureTest")
-  }
-
-  test("test data loading and validate query output") {
-
-    checkAnswer(
-      sql("select empno from nomeasureTest"),
-      Seq(Row("11"), Row("12"), Row("13"))
-    )
-  }
-
-  test("test data loading with single dictionary column") {
-    sql("DROP TABLE IF EXISTS nomeasureTest_sd")
-    sql("CREATE TABLE nomeasureTest_sd (city String) STORED BY 'org.apache.carbondata.format'")
-    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-    val testData = currentDirectory + "/src/test/resources/datasingleCol.csv"
-    sql("LOAD DATA LOCAL INPATH '" + testData + "' into table nomeasureTest_sd options " +
-      "('FILEHEADER'='city')"
-    )
-
-    checkAnswer(
-      sql("select city from nomeasureTest_sd"),
-      Seq(Row("CA"), Row("LA"), Row("AD"))
-    )
-  }
-
-  test("test data loading with single no dictionary column") {
-    sql("DROP TABLE IF EXISTS nomeasureTest_sd")
-    sql(
-      "CREATE TABLE nomeasureTest_sd (city String) STORED BY 'org.apache.carbondata.format' " +
-        "TBLPROPERTIES ('DICTIONARY_EXCLUDE'='city')"
-    )
-    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-    val testData = currentDirectory + "/src/test/resources/datasingleCol.csv"
-    sql("LOAD DATA LOCAL INPATH '" + testData + "' into table nomeasureTest_sd options " +
-      "('FILEHEADER'='city')"
-    )
-
-    checkAnswer(
-      sql("select city from nomeasureTest_sd"),
-      Seq(Row("CA"), Row("LA"), Row("AD"))
-    )
-  }
-
-  test("test data loading with single complex struct type column") {
-    //only data load check
-    sql("DROP TABLE IF EXISTS nomeasureTest_scd")
-    sql(
-      "CREATE TABLE nomeasureTest_scd (cityDetail struct<cityName:string,cityCode:string>) STORED" +
-        " " +
-        "BY 'org.apache.carbondata.format'"
-    )
-    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-    val testData = currentDirectory + "/src/test/resources/datasingleComplexCol.csv"
-    sql("LOAD DATA LOCAL INPATH '" + testData + "' into table nomeasureTest_scd options " +
-      "('DELIMITER'=',','QUOTECHAR'='\"','FILEHEADER'='cityDetail','COMPLEX_DELIMITER_LEVEL_1'=':')"
-    )
-  }
-
-  test("test data loading with single complex array type column") {
-    //only data load check
-    sql("DROP TABLE IF EXISTS nomeasureTest_scd")
-    sql(
-      "CREATE TABLE nomeasureTest_scd (cityDetail array<string>) STORED" +
-        " " +
-        "BY 'org.apache.carbondata.format'"
-    )
-    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-    val testData = currentDirectory + "/src/test/resources/datasingleComplexCol.csv"
-    sql("LOAD DATA LOCAL INPATH '" + testData + "' into table nomeasureTest_scd options " +
-      "('DELIMITER'=',','QUOTECHAR'='\"','FILEHEADER'='cityDetail'," +
-      "'COMPLEX_DELIMITER_LEVEL_1'=':')"
-    )
-  }
-
-  override def afterAll {
-    sql("drop table nomeasureTest")
-    sql("drop table nomeasureTest_sd")
-    sql("drop table nomeasureTest_scd")
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
deleted file mode 100644
index c5bc058..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.dataload
-
-import java.io.File
-
-import org.apache.spark.sql.common.util.{CarbonHiveContext, QueryTest}
-import org.apache.spark.util.FileUtils
-
-import org.apache.carbondata.processing.model.CarbonLoadModel
-import org.apache.carbondata.spark.util.GlobalDictionaryUtil
-
-/**
- * Test class of loading data for carbon table with not proper input file
- *
- */
-class TestLoadDataWithNotProperInputFile extends QueryTest {
-
-  test("test loading data with input path exists but has nothing") {
-    try {
-      val carbonLoadModel: CarbonLoadModel = new CarbonLoadModel
-      val dataPath = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath + "/src/test/resources/nullSample.csv"
-      carbonLoadModel.setFactFilePath(FileUtils.getPaths(dataPath))
-      GlobalDictionaryUtil.loadDataFrame(CarbonHiveContext, carbonLoadModel)
-    } catch {
-      case e: Throwable =>
-        assert(e.getMessage.contains("Please check your input path and make sure " +
-          "that files end with '.csv' and content is not empty"))
-    }
-  }
-
-  test("test loading data with input file not ends with '.csv'") {
-    try {
-      val carbonLoadModel: CarbonLoadModel = new CarbonLoadModel
-      val dataPath = new File(this.getClass.getResource("/").getPath + "/../../")
-        .getCanonicalPath + "/src/test/resources/noneCsvFormat.cs"
-      carbonLoadModel.setFactFilePath(FileUtils.getPaths(dataPath))
-      GlobalDictionaryUtil.loadDataFrame(CarbonHiveContext, carbonLoadModel)
-    } catch {
-      case e: Throwable =>
-        e.printStackTrace()
-        assert(e.getMessage.contains("Please check your input path and make sure " +
-          "that files end with '.csv' and content is not empty"))
-    }
-  }
-
-  test("test loading data with input file does not exist") {
-    try {
-      val carbonLoadModel: CarbonLoadModel = new CarbonLoadModel
-      val dataPath = new File(this.getClass.getResource("/").getPath + "/../../")
-        .getCanonicalPath + "/src/test/resources/input_file_does_not_exist.csv"
-      carbonLoadModel.setFactFilePath(FileUtils.getPaths(dataPath))
-      GlobalDictionaryUtil.loadDataFrame(CarbonHiveContext, carbonLoadModel)
-    } catch {
-      case e: Throwable =>
-        assert(e.getMessage.contains("The input file does not exist"))
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithSinglePass.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithSinglePass.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithSinglePass.scala
index 3e0b4c8..09a2765 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithSinglePass.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithSinglePass.scala
@@ -19,7 +19,6 @@
 
 package org.apache.carbondata.spark.testsuite.dataload
 
-import org.apache.spark.sql.common.util.CarbonHiveContext._
 import org.apache.spark.sql.common.util.QueryTest
 import org.scalatest.BeforeAndAfterAll
 
@@ -42,8 +41,8 @@ class TestLoadDataWithSinglePass extends QueryTest with BeforeAndAfterAll {
       """.stripMargin)
 
     sql(
-      """
-        |LOAD DATA local inpath './src/test/resources/dataDiff.csv' INTO TABLE table_two_pass
+      s"""
+        |LOAD DATA local inpath '$resourcesPath/dataDiff.csv' INTO TABLE table_two_pass
         |OPTIONS('DELIMITER'= ',', 'USE_KETTLE'='false', 'SINGLE_PASS'='false')
       """.stripMargin)
 
@@ -55,8 +54,8 @@ class TestLoadDataWithSinglePass extends QueryTest with BeforeAndAfterAll {
       """.stripMargin)
 
     sql(
-      """
-        |LOAD DATA local inpath './src/test/resources/dataDiff.csv' INTO TABLE table_one_pass
+      s"""
+        |LOAD DATA local inpath '$resourcesPath/dataDiff.csv' INTO TABLE table_one_pass
         |OPTIONS('DELIMITER'= ',', 'USE_KETTLE'='false', 'SINGLE_PASS'='true')
       """.stripMargin)
   }
@@ -76,10 +75,10 @@ class TestLoadDataWithSinglePass extends QueryTest with BeforeAndAfterAll {
         |STORED BY 'org.apache.carbondata.format'
       """.stripMargin)
     sql(
-      """
-        |LOAD DATA local inpath './src/test/resources/dataDiff.csv' INTO TABLE table_one_pass_2
+      s"""
+        |LOAD DATA local inpath '$resourcesPath/dataDiff.csv' INTO TABLE table_one_pass_2
         |OPTIONS('DELIMITER'= ',', 'USE_KETTLE'='false', 'SINGLE_PASS'='true', 'COLUMNDICT'=
-        |'country:./src/test/resources/columndictionary/country.csv, name:./src/test/resources/columndictionary/name.csv')
+        |'country:$resourcesPath/columndictionary/country.csv, name:$resourcesPath/columndictionary/name.csv')
       """.stripMargin)
 
     checkAnswer(
@@ -90,13 +89,13 @@ class TestLoadDataWithSinglePass extends QueryTest with BeforeAndAfterAll {
 
   test("test data loading use one pass when do incremental load") {
     sql(
-      """
-        |LOAD DATA local inpath './src/test/resources/dataIncrement.csv' INTO TABLE table_two_pass
+      s"""
+        |LOAD DATA local inpath '$resourcesPath/dataIncrement.csv' INTO TABLE table_two_pass
         |OPTIONS('DELIMITER'= ',', 'USE_KETTLE'='false', 'SINGLE_PASS'='false')
       """.stripMargin)
     sql(
-      """
-        |LOAD DATA local inpath './src/test/resources/dataIncrement.csv' INTO TABLE table_one_pass
+      s"""
+        |LOAD DATA local inpath '$resourcesPath/dataIncrement.csv' INTO TABLE table_one_pass
         |OPTIONS('DELIMITER'= ',', 'USE_KETTLE'='false', 'SINGLE_PASS'='true')
       """.stripMargin)
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
deleted file mode 100644
index 9f63368..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.dataload
-
-import java.io.File
-
-import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-import org.apache.spark.sql.Row
-
-/**
-  * Test Class for data loading into table whose name is key word
-  *
-  */
-class TestLoadTblNameIsKeyword extends QueryTest with BeforeAndAfterAll {
-  def currentPath: String = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-  val testData = new File(currentPath + "/src/test/resources/dimSample.csv")
-      .getCanonicalPath
-  override def beforeAll {
-    sql("drop table if exists STRING")
-    sql("drop table if exists DoUbLe")
-    sql("drop table if exists timestamp")
-    sql("""
-          CREATE TABLE IF NOT EXISTS STRING
-          (id Int, name String, city String)
-          STORED BY 'org.apache.carbondata.format'
-        """)
-    sql("""
-          CREATE TABLE IF NOT EXISTS DoUbLe
-          (id Int, name String, city String)
-          STORED BY 'org.apache.carbondata.format'
-        """)
-    sql("""
-          CREATE TABLE IF NOT EXISTS timestamp
-          (id Int, name String, city String)
-          STORED BY 'org.apache.carbondata.format'
-        """)
-  }
-
-  test("test load data whose name is a keyword of data type") {
-    sql(s"""
-          LOAD DATA LOCAL INPATH '$testData' into table STRING
-        """)
-    checkAnswer(
-      sql("""
-            SELECT count(*) from STRING
-          """),
-      Seq(Row(20)))
-  }
-
-  test("test case in-sensitiveness") {
-    sql(s"""
-          LOAD DATA LOCAL INPATH '$testData' into table DoUbLe
-        """)
-    checkAnswer(
-      sql("""
-            SELECT count(*) from DoUbLe
-          """),
-      Seq(Row(20)))
-  }
-
-  test("test other ddl whose table name a keyword of data type") {
-    sql("describe timestamp")
-    sql(s"""
-          LOAD DATA LOCAL INPATH '$testData' into table timestamp
-        """)
-    sql("show segments for table timestamp")
-    sql("delete segments from table timestamp where starttime before '2099-10-01 18:00:00'")
-    sql("clean files for table timestamp")
-  }
-
-  override def afterAll {
-    sql("drop table STRING")
-    sql("drop table DoUbLe")
-    sql("drop table timestamp")
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
index 17fe053..f4831e8 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
@@ -19,15 +19,12 @@
 
 package org.apache.carbondata.spark.testsuite.dataload
 
-import java.io.File
-
-import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
 import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
 
-import org.apache.carbondata.core.util.CarbonProperties
 import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.core.util.CarbonProperties
 
 /**
   * Test Class for no inverted index load and query
@@ -36,12 +33,8 @@ import org.scalatest.BeforeAndAfterAll
 
 class TestNoInvertedIndexLoadAndQuery extends QueryTest with BeforeAndAfterAll{
 
-  def currentPath: String = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-  val testData1 = new File(currentPath + "/src/test/resources/dimSample.csv")
-      .getCanonicalPath
-  val testData2 = new File(currentPath + "/src/test/resources/example-data.csv")
-      .getCanonicalPath
+  val testData1 = s"$resourcesPath/dimSample.csv"
+  val testData2 = s"$resourcesPath/example-data.csv"
 
   override def beforeAll {
     sql("DROP TABLE IF EXISTS index1")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala
deleted file mode 100644
index db6af37..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.dataload
-
-import java.io.File
-
-import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
-import org.apache.spark.sql.Row
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.scalatest.BeforeAndAfterAll
-
-/**
-  * Test Class for table block size
-  *
-  */
-
-class TestTableLevelBlockSize extends QueryTest with BeforeAndAfterAll{
-
-  def currentPath: String = new File(this.getClass.getResource("/").getPath + "/../../")
-      .getCanonicalPath
-  val testData1 = new File(currentPath + "/src/test/resources/dimSample.csv")
-      .getCanonicalPath
-  val testData2 = new File(currentPath + "/src/test/resources/example-data.csv")
-      .getCanonicalPath
-
-  override def beforeAll {
-    sql("DROP TABLE IF EXISTS table_blocksize1")
-    sql("DROP TABLE IF EXISTS table_blocksize2")
-    sql("DROP TABLE IF EXISTS table_blocksize3")
-  }
-
-  test("Value test: set table level blocksize value beyong [1,2048]") {
-    try {
-      sql(
-        """
-          CREATE TABLE IF NOT EXISTS table_blocksize1
-          (ID Int, date Timestamp, country String,
-          name String, phonetype String, serialname String, salary Int)
-          STORED BY 'org.apache.carbondata.format'
-          TBLPROPERTIES('table_blocksize'='4096 MB')
-        """)
-      assert(false)
-    } catch {
-      case e : MalformedCarbonCommandException => {
-        assert(e.getMessage.equals("Invalid table_blocksize value found: 4096, " +
-            "only int value from 1 MB to 2048 MB is supported."))
-      }
-    }
-  }
-
-  test("Value test: set table level blocksize in not int value") {
-    try {
-      sql(
-        """
-          CREATE TABLE IF NOT EXISTS table_blocksize2
-          (ID Int, date Timestamp, country String,
-          name String, phonetype String, serialname String, salary Int)
-          STORED BY 'org.apache.carbondata.format'
-          TBLPROPERTIES('table_blocksize'='10Y4 MB')
-        """)
-      assert(false)
-    } catch {
-      case e : MalformedCarbonCommandException => {
-        assert(e.getMessage.equals("Invalid table_blocksize value found: 10y4, " +
-            "only int value from 1 MB to 2048 MB is supported."))
-      }
-    }
-  }
-
-  test("Function test: set table level blocksize load and agg query") {
-
-    sql(
-      """
-        CREATE TABLE IF NOT EXISTS table_blocksize3
-        (ID Int, date Timestamp, country String,
-        name String, phonetype String, serialname String, salary Int)
-        STORED BY 'org.apache.carbondata.format'
-        TBLPROPERTIES('table_blocksize'='512 MB')
-      """)
-
-    CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
-
-    sql(s"""
-           LOAD DATA LOCAL INPATH '$testData2' into table table_blocksize3
-           """)
-
-    checkAnswer(
-      sql("""
-           SELECT country, count(salary) AS amount
-           FROM table_blocksize3
-           WHERE country IN ('china','france')
-           GROUP BY country
-          """),
-      Seq(Row("china", 849), Row("france", 101))
-    )
-
-  }
-
-  override def afterAll {
-    sql("DROP TABLE IF EXISTS table_blocksize1")
-    sql("DROP TABLE IF EXISTS table_blocksize2")
-    sql("DROP TABLE IF EXISTS table_blocksize3")
-    CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
deleted file mode 100644
index ebe4b01..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.dataretention
-
-import java.io.File
-import java.util
-import java.util.concurrent.{Callable, Executors}
-
-import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-/**
- * This class contains DataRetention concurrency test cases
- */
-class DataRetentionConcurrencyTestCase extends QueryTest with BeforeAndAfterAll {
-
-  val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-    .getCanonicalPath
-  val resource = currentDirectory + "/src/test/resources/"
-  val executorService = Executors.newFixedThreadPool(10)
-
-  override def beforeAll {
-
-    sql("drop table if exists concurrent")
-    sql(
-      "create table concurrent (ID int, date String, country String, name " +
-      "String," +
-      "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format'"
-
-    )
-    sql(
-      "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE concurrent " +
-      "OPTIONS('DELIMITER' =  ',')")
-  }
-
-  override def afterAll {
-    executorService.shutdownNow()
-    sql("drop table if exists concurrent")
-  }
-
-  test("DataRetention_Concurrency_load_id") {
-
-    val tasks = new util.ArrayList[Callable[String]]()
-    tasks
-      .add(new QueryTask("LOAD DATA LOCAL INPATH '" + resource +
-                         "dataretention1.csv' INTO TABLE concurrent OPTIONS('DELIMITER' =  ',')"))
-    tasks.add(new QueryTask("Delete segment 0 from table concurrent"))
-    tasks.add(new QueryTask("clean files for table concurrent"))
-    val results = executorService.invokeAll(tasks)
-    for (i <- 0 until tasks.size()) {
-      val res = results.get(i).get
-      assert("PASS".equals(res))
-    }
-    sql("show segments for table concurrent").show()
-
-  }
-
-  test("DataRetention_Concurrency_load_date") {
-
-    sql(
-      "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE concurrent " +
-      "OPTIONS('DELIMITER' =  ',')")
-
-    val tasks = new util.ArrayList[Callable[String]]()
-    tasks
-      .add(new QueryTask("LOAD DATA LOCAL INPATH '" + resource +
-                         "dataretention1.csv' INTO TABLE concurrent OPTIONS('DELIMITER' =  ',')"))
-    tasks
-      .add(new QueryTask(
-        "DELETE SEGMENTS FROM TABLE concurrent where STARTTIME before '2099-01-01 00:00:00'"))
-    tasks.add(new QueryTask("clean files for table concurrent"))
-    val results = executorService.invokeAll(tasks)
-    for (i <- 0 until tasks.size()) {
-      val res = results.get(i).get
-      assert("PASS".equals(res))
-    }
-    sql("show segments for table concurrent").show()
-
-  }
-
-  class QueryTask(query: String) extends Callable[String] {
-    override def call(): String = {
-      var result = "PASS"
-      try {
-        LOGGER.info("Executing :" + Thread.currentThread().getName)
-        sql(query)
-      } catch {
-        case ex: Exception =>
-          result = "FAIL"
-      }
-      result
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
deleted file mode 100644
index 26699f2..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.dataretention
-
-import java.io.File
-import java.text.SimpleDateFormat
-
-import org.apache.carbondata.core.updatestatus.SegmentStatusManager
-import org.apache.carbondata.locks.{LockUsage, CarbonLockFactory, ICarbonLock}
-import org.apache.commons.lang3.time.DateUtils
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.carbon.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.load.LoadMetadataDetails
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
-/**
- * This class contains data retention test cases
- * Created by Manohar on 5/9/2016.
- */
-class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
-
-  val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
-    .getCanonicalPath
-  val resource = currentDirectory + "/src/test/resources/"
-
-  val storeLocation = new File(this.getClass.getResource("/").getPath + "/../test").getCanonicalPath
-  val absoluteTableIdentifierForLock: AbsoluteTableIdentifier = new
-      AbsoluteTableIdentifier(storeLocation,
-        new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "retentionlock", "200"))
-  val absoluteTableIdentifierForRetention: AbsoluteTableIdentifier = new
-      AbsoluteTableIdentifier(storeLocation,
-        new CarbonTableIdentifier(
-          CarbonCommonConstants.DATABASE_DEFAULT_NAME, "DataRetentionTable".toLowerCase(), "300"))
-  val carbonTablePath = CarbonStorePath
-    .getCarbonTablePath(absoluteTableIdentifierForRetention.getStorePath,
-      absoluteTableIdentifierForRetention.getCarbonTableIdentifier).getMetadataDirectoryPath
-
-  var carbonDateFormat = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP)
-  var defaultDateFormat = new SimpleDateFormat(CarbonCommonConstants
-    .CARBON_TIMESTAMP_DEFAULT_FORMAT)
-  val carbonTableStatusLock: ICarbonLock = CarbonLockFactory
-    .getCarbonLockObj(absoluteTableIdentifierForLock.getCarbonTableIdentifier, LockUsage.TABLE_STATUS_LOCK)
-  val carbonDeleteSegmentLock: ICarbonLock = CarbonLockFactory
-    .getCarbonLockObj(absoluteTableIdentifierForLock.getCarbonTableIdentifier, LockUsage.DELETE_SEGMENT_LOCK)
-  val carbonCleanFilesLock: ICarbonLock = CarbonLockFactory
-    .getCarbonLockObj(absoluteTableIdentifierForLock.getCarbonTableIdentifier, LockUsage.CLEAN_FILES_LOCK)
-  val carbonMetadataLock: ICarbonLock = CarbonLockFactory
-    .getCarbonLockObj(absoluteTableIdentifierForLock.getCarbonTableIdentifier, LockUsage.METADATA_LOCK)
-
-
-  override def beforeAll {
-    CarbonProperties.getInstance.addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "1")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
-    sql(
-      "CREATE table DataRetentionTable (ID int, date String, country String, name " +
-      "String," +
-      "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format'"
-
-    )
-    sql(
-      "CREATE table retentionlock (ID int, date String, country String, name " +
-      "String," +
-      "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format'"
-
-    )
-
-    sql(
-      "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE retentionlock " +
-      "OPTIONS('DELIMITER' =  ',')")
-
-    sql(
-      "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE DataRetentionTable " +
-      "OPTIONS('DELIMITER' =  ',')")
-    sql(
-      "LOAD DATA LOCAL INPATH '" + resource + "dataretention2.csv' INTO TABLE DataRetentionTable " +
-      "OPTIONS('DELIMITER' = ',')")
-
-  }
-
-  override def afterAll {
-    sql("drop table DataRetentionTable")
-    sql("drop table carbon_TABLE_1")
-    sql("drop table retentionlock")
-    CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
-  }
-
-
-  private def getSegmentStartTime(segments: Array[LoadMetadataDetails],
-      segmentId: Integer): String = {
-    val segmentLoadTimeString = segments(segmentId).getLoadStartTime()
-    var loadTime = carbonDateFormat.parse(carbonDateFormat.format(segmentLoadTimeString))
-    // add one min to execute delete before load start time command
-    loadTime = DateUtils.addMinutes(loadTime, 1)
-    defaultDateFormat.format(loadTime)
-
-  }
-
-
-  test("RetentionTest_withoutDelete") {
-    checkAnswer(
-      sql("SELECT country, count(salary) AS amount FROM DataRetentionTable WHERE country" +
-          " IN ('china','ind','aus','eng') GROUP BY country"
-      ),
-      Seq(Row("aus", 9), Row("ind", 9))
-    )
-  }
-
-  test("RetentionTest_DeleteSegmentsByLoadTime") {
-    val segments: Array[LoadMetadataDetails] =
-      SegmentStatusManager.readLoadMetadata(carbonTablePath)
-    // check segment length, it should be 3 (loads)
-    if (segments.length != 2) {
-      assert(false)
-    }
-
-    val actualValue: String = getSegmentStartTime(segments, 1)
-    // delete segments (0,1) which contains ind, aus
-    sql(
-      "DELETE SEGMENTS FROM TABLE DataRetentionTable where STARTTIME before '" + actualValue + "'")
-
-    // load segment 2 which contains eng
-    sql(
-      "LOAD DATA LOCAL INPATH '" + resource + "dataretention3.csv' INTO TABLE DataRetentionTable " +
-      "OPTIONS('DELIMITER' = ',')")
-    checkAnswer(
-      sql("SELECT country, count(salary) AS amount FROM DataRetentionTable WHERE country" +
-          " IN ('china','ind','aus','eng') GROUP BY country"
-      ),
-      Seq(Row("eng", 9))
-    )
-  }
-
-  test("RetentionTest3_DeleteByLoadId") {
-    // delete segment 2 and load ind segment
-    sql("DELETE SEGMENT 2 FROM TABLE DataRetentionTable")
-    sql(
-      "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE DataRetentionTable " +
-      "OPTIONS('DELIMITER' = ',')")
-    checkAnswer(
-      sql("SELECT country, count(salary) AS amount FROM DataRetentionTable WHERE country" +
-          " IN ('china','ind','aus','eng') GROUP BY country"
-      ),
-      Seq(Row("ind", 9))
-    )
-
-    // these queries should execute without any error.
-    sql("show segments for table DataRetentionTable")
-    sql("clean files for table DataRetentionTable")
-  }
-
-  test("RetentionTest4_DeleteByInvalidLoadId") {
-    try {
-      // delete segment with no id
-      sql("DELETE SEGMENT FROM TABLE DataRetentionTable")
-      assert(false)
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(e.getMessage.contains("should not be empty"))
-      case _: Throwable => assert(false)
-    }
-  }
-
-  test("test delete segments by load date with case-insensitive table name") {
-    sql(
-      """
-      CREATE TABLE IF NOT EXISTS carbon_TABLE_1
-      (ID Int, date Timestamp, country String,
-      name String, phonetype String, serialname String, salary Int)
-      STORED BY 'org.apache.carbondata.format'
-      TBLPROPERTIES('DICTIONARY_EXCLUDE'='country,phonetype,serialname',
-      'DICTIONARY_INCLUDE'='ID')
-      """)
-
-    sql("LOAD DATA LOCAL INPATH '" + resource +
-      "emptyDimensionData.csv' into table CarBon_tAbLE_1")
-
-    checkAnswer(
-      sql("select count(*) from cArbon_TaBlE_1"), Seq(Row(20)))
-
-    sql("delete segments from table carbon_TABLE_1 " +
-      "where starttime before '2099-07-28 11:00:00'")
-
-    checkAnswer(
-      sql("select count(*) from caRbon_TabLe_1"), Seq(Row(0)))
-
-  }
-  test("RetentionTest_DeleteSegmentsByLoadTimeValiadtion") {
-
-    try {
-      sql(
-        "DELETE SEGMENTS FROM TABLE DataRetentionTable where STARTTIME before" +
-        " 'abcd-01-01 00:00:00'")
-      assert(false)
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(e.getMessage.contains("Invalid load start time format"))
-      case _: Throwable => assert(false)
-    }
-
-    try {
-      sql(
-        "DELETE SEGMENTS FROM TABLE DataRetentionTable where STARTTIME before" +
-        " '2099:01:01 00:00:00'")
-      assert(false)
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(e.getMessage.contains("Invalid load start time format"))
-      case _: Throwable => assert(false)
-    }
-
-    checkAnswer(
-      sql("SELECT country, count(salary) AS amount FROM DataRetentionTable WHERE country" +
-          " IN ('china','ind','aus','eng') GROUP BY country"
-      ),
-      Seq(Row("ind", 9))
-    )
-    sql("DELETE SEGMENTS FROM TABLE DataRetentionTable where STARTTIME before '2099-01-01'")
-    checkAnswer(
-      sql("SELECT country, count(salary) AS amount FROM DataRetentionTable WHERE country" +
-          " IN ('china','ind','aus','eng') GROUP BY country"), Seq())
-
-
-  }
-
-  test("RetentionTest_InvalidDeleteCommands") {
-
-    // All these queries should fail.
-    try {
-      sql("DELETE LOADS FROM TABLE DataRetentionTable where STARTTIME before '2099-01-01'")
-      throw new MalformedCarbonCommandException("Invalid query")
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(!e.getMessage.equalsIgnoreCase("Invalid query"))
-      case _: Throwable => assert(true)
-    }
-
-    try {
-      sql("DELETE LOAD 2 FROM TABLE DataRetentionTable")
-      throw new MalformedCarbonCommandException("Invalid query")
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(!e.getMessage.equalsIgnoreCase("Invalid query"))
-      case _: Throwable => assert(true)
-    }
-
-    try {
-      sql("show loads for table DataRetentionTable")
-      throw new MalformedCarbonCommandException("Invalid query")
-    } catch {
-      case e: MalformedCarbonCommandException =>
-        assert(!e.getMessage.equalsIgnoreCase("Invalid query"))
-      case _: Throwable => assert(true)
-    }
-
-  }
-
-  test("RetentionTest_Locks") {
-
-    sql(
-      "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE retentionlock " +
-      "OPTIONS('DELIMITER' = ',')")
-    carbonDeleteSegmentLock.lockWithRetries()
-    carbonTableStatusLock.lockWithRetries()
-    carbonCleanFilesLock.lockWithRetries()
-    // delete segment 0 it should fail
-    try {
-      sql("DELETE SEGMENT 0 FROM TABLE retentionlock")
-      throw new MalformedCarbonCommandException("Invalid")
-    } catch {
-      case me: MalformedCarbonCommandException =>
-        assert(false)
-      case ex: Exception =>
-        assert(true)
-    }
-
-    // it should fail
-    try {
-      sql("DELETE SEGMENTS FROM TABLE retentionlock where STARTTIME before " +
-          "'2099-01-01 00:00:00.0'")
-      throw new MalformedCarbonCommandException("Invalid")
-    } catch {
-      case me: MalformedCarbonCommandException =>
-        assert(false)
-      case ex: Exception =>
-        assert(true)
-    }
-
-    // it should fail
-    try {
-      sql("clean files for table retentionlock")
-      throw new MalformedCarbonCommandException("Invalid")
-    } catch {
-      case me: MalformedCarbonCommandException =>
-        assert(false)
-      case ex: Exception =>
-        assert(true)
-    }
-    carbonTableStatusLock.unlock()
-    carbonCleanFilesLock.unlock()
-    carbonDeleteSegmentLock.unlock()
-
-    sql("DELETE SEGMENT 0 FROM TABLE retentionlock")
-    //load and delete should execute parallely
-    carbonMetadataLock.lockWithRetries()
-    sql("DELETE SEGMENT 1 FROM TABLE retentionlock")
-    carbonMetadataLock.unlock()
-  }
-}


Mime
View raw message