carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: Lionx0929
Date Sat, 29 Oct 2016 05:12:07 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master d840562a6 -> 267af1f55


Lionx0929

mend


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/72d076ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/72d076ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/72d076ac

Branch: refs/heads/master
Commit: 72d076acbfad0b6beb07471f3d5148d2f63e2191
Parents: d840562
Author: X-Lion <xlion.pku@gmail.com>
Authored: Thu Sep 29 22:33:18 2016 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Sat Oct 29 13:01:43 2016 +0800

----------------------------------------------------------------------
 .../DirectDictionaryGenerator.java              |   2 +
 .../DirectDictionaryKeyGeneratorFactory.java    |  14 +-
 .../TimeStampDirectDictionaryGenerator.java     |  39 ++++--
 .../hadoop/test/util/StoreCreator.java          |   2 +
 .../carbondata/spark/load/CarbonLoadModel.java  |   7 +
 .../carbondata/spark/load/CarbonLoaderUtil.java |   2 +
 .../org/apache/spark/sql/CarbonSqlParser.scala  |   2 +-
 .../execution/command/carbonTableSchema.scala   |  29 +++-
 .../src/test/resources/timeStampFormatData1.csv |  12 ++
 .../src/test/resources/timeStampFormatData2.csv |  11 ++
 .../TestLoadDataWithDiffTimestampFormat.scala   | 138 +++++++++++++++++++
 .../api/dataloader/DataLoadModel.java           |   6 +
 .../dataprocessor/DataProcessTaskStatus.java    |   6 +
 .../dataprocessor/IDataProcessStatus.java       |   2 +
 .../graphgenerator/GraphGenerator.java          |   3 +
 .../csvbased/CarbonCSVBasedSeqGenMeta.java      |  14 +-
 .../csvbased/CarbonCSVBasedSeqGenStep.java      |  43 +++++-
 .../TimeStampDirectDictionaryGeneratorTest.java |  10 +-
 .../carbondata/test/util/StoreCreator.java      |   2 +
 19 files changed, 311 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
index 595760a..2eab97d 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
@@ -53,4 +53,6 @@ public interface DirectDictionaryGenerator {
    */
   int generateDirectSurrogateKey(String memberStr, String format);
 
+  void initialize();
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
index b038789..f7148b3 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
@@ -19,7 +19,9 @@
 package org.apache.carbondata.core.keygenerator.directdictionary;
 
 import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator;
+import org.apache.carbondata.core.util.CarbonProperties;
 
 /**
  * Factory for DirectDictionary Key generator
@@ -39,15 +41,23 @@ public final class DirectDictionaryKeyGeneratorFactory {
    * @param dataType DataType
    * @return the generator instance
    */
-  public static DirectDictionaryGenerator getDirectDictionaryGenerator(DataType dataType)
{
+  public static DirectDictionaryGenerator getDirectDictionaryGenerator(DataType dataType,
+      String dateFormat) {
     DirectDictionaryGenerator directDictionaryGenerator = null;
     switch (dataType) {
       case TIMESTAMP:
-        directDictionaryGenerator = TimeStampDirectDictionaryGenerator.instance;
+        directDictionaryGenerator = new TimeStampDirectDictionaryGenerator(dateFormat);
         break;
       default:
 
     }
     return directDictionaryGenerator;
   }
+
+  public static DirectDictionaryGenerator getDirectDictionaryGenerator(DataType dataType)
{
+    return getDirectDictionaryGenerator(dataType,
+        CarbonProperties.getInstance()
+            .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+                CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 7b8e49f..a9ceab4 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -39,12 +39,9 @@ import static org.apache.carbondata.core.keygenerator.directdictionary.timestamp
  */
 public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGenerator {
 
-  private TimeStampDirectDictionaryGenerator() {
+  private ThreadLocal<SimpleDateFormat> simpleDateFormatLocal = new ThreadLocal<>();
 
-  }
-
-  public static TimeStampDirectDictionaryGenerator instance =
-      new TimeStampDirectDictionaryGenerator();
+  private String dateFormat;
 
   /**
    * The value of 1 unit of the SECOND, MINUTE, HOUR, or DAY in millis.
@@ -109,6 +106,17 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
     cutOffTimeStamp = cutOffTimeStampLocal;
   }
 
+  public TimeStampDirectDictionaryGenerator(String dateFormat) {
+    this.dateFormat = dateFormat;
+    initialize();
+  }
+
+  public TimeStampDirectDictionaryGenerator( ) {
+    this(CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+            CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+  }
+
   /**
    * The method take member String as input and converts
    * and returns the dictionary key
@@ -117,15 +125,11 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
    * @return dictionary value
    */
   @Override public int generateDirectSurrogateKey(String memberStr) {
-    SimpleDateFormat timeParser = new SimpleDateFormat(CarbonProperties.getInstance()
-        .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-            CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
-    timeParser.setLenient(false);
     if (null == memberStr || memberStr.trim().isEmpty() || memberStr
         .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
       return 1;
     }
-    return getDirectSurrogateForMember(memberStr, timeParser);
+    return getDirectSurrogateForMember(memberStr);
   }
 
   /**
@@ -139,20 +143,18 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
     if (null == format) {
       return generateDirectSurrogateKeyForNonTimestampType(memberStr);
     } else {
-      SimpleDateFormat timeParser = new SimpleDateFormat(format);
-      timeParser.setLenient(false);
       if (null == memberStr || memberStr.trim().isEmpty() || memberStr
           .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
         return 1;
       }
-      return getDirectSurrogateForMember(memberStr, timeParser);
+      return getDirectSurrogateForMember(memberStr);
     }
   }
 
-  private int getDirectSurrogateForMember(String memberStr, SimpleDateFormat timeParser)
{
+  private int getDirectSurrogateForMember(String memberStr) {
     Date dateToStr = null;
     try {
-      dateToStr = timeParser.parse(memberStr);
+      dateToStr = simpleDateFormatLocal.get().parse(memberStr);
     } catch (ParseException e) {
       LOGGER.debug(
           "Cannot convert " + memberStr + " to Time/Long type value. Value considered as
null." + e
@@ -212,4 +214,11 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
     }
   }
 
+  public void initialize(){
+    if (simpleDateFormatLocal.get() == null) {
+      simpleDateFormatLocal.set(new SimpleDateFormat(dateFormat));
+      simpleDateFormatLocal.get().setLenient(false);
+    }
+  }
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 27a1a3e..0280820 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -357,6 +357,7 @@ public class StoreCreator {
     dataProcessTaskStatus.setEscapeCharacter("\\");
     dataProcessTaskStatus.setQuoteCharacter("\"");
     dataProcessTaskStatus.setCommentCharacter("#");
+    dataProcessTaskStatus.setDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
     info.setDatabaseName(databaseName);
     info.setTableName(tableName);
     info.setSerializationNullFormat(
@@ -472,6 +473,7 @@ public class StoreCreator {
     model.setEscapeCharacter(dataProcessTaskStatus.getEscapeCharacter());
     model.setQuoteCharacter(dataProcessTaskStatus.getQuoteCharacter());
     model.setCommentCharacter(dataProcessTaskStatus.getCommentCharacter());
+    model.setDateFormat(dataProcessTaskStatus.getDateFormat());
     if (null != loadMetadataDetails && !loadMetadataDetails.isEmpty()) {
       model.setLoadNames(
           CarbonDataProcessorUtil.getLoadNameFromLoadMetaDataDetails(loadMetadataDetails));

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoadModel.java
b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoadModel.java
index 106ad71..b33601d 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoadModel.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoadModel.java
@@ -106,6 +106,8 @@ public class CarbonLoadModel implements Serializable {
    */
   private String commentChar;
 
+  private String dateFormat;
+
   /**
    * defines the string that should be treated as null while loadind data
    */
@@ -395,6 +397,7 @@ public class CarbonLoadModel implements Serializable {
     copyObj.escapeChar = escapeChar;
     copyObj.quoteChar = quoteChar;
     copyObj.commentChar = commentChar;
+    copyObj.dateFormat = dateFormat;
     copyObj.maxColumns = maxColumns;
     return copyObj;
   }
@@ -597,6 +600,10 @@ public class CarbonLoadModel implements Serializable {
     this.commentChar = commentChar;
   }
 
+  public String getDateFormat() { return dateFormat; }
+
+  public void setDateFormat(String dateFormat) { this.dateFormat = dateFormat; }
+
   /**
    * @return
    */

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 7c345e4..ce67e66 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -136,6 +136,7 @@ public final class CarbonLoaderUtil {
     model.setTaskNo(loadModel.getTaskNo());
     model.setFactTimeStamp(loadModel.getFactTimeStamp());
     model.setMaxColumns(loadModel.getMaxColumns());
+    model.setDateFormat(loadModel.getDateFormat());
     boolean hdfsReadMode =
         dataProcessTaskStatus.getCsvFilePath() != null
                 && dataProcessTaskStatus.getCsvFilePath().startsWith("hdfs:");
@@ -194,6 +195,7 @@ public final class CarbonLoaderUtil {
     dataProcessTaskStatus.setQuoteCharacter(loadModel.getQuoteChar());
     dataProcessTaskStatus.setCommentCharacter(loadModel.getCommentChar());
     dataProcessTaskStatus.setRddIteratorKey(loadModel.getRddIteratorKey());
+    dataProcessTaskStatus.setDateFormat(loadModel.getDateFormat());
     SchemaInfo info = new SchemaInfo();
 
     info.setDatabaseName(databaseName);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index da14b3e..ab7debf 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -945,7 +945,7 @@ class CarbonSqlParser()
     val supportedOptions = Seq("DELIMITER", "QUOTECHAR", "FILEHEADER", "ESCAPECHAR", "MULTILINE",
       "COMPLEX_DELIMITER_LEVEL_1", "COMPLEX_DELIMITER_LEVEL_2", "COLUMNDICT",
       "SERIALIZATION_NULL_FORMAT", "BAD_RECORDS_LOGGER_ENABLE", "BAD_RECORDS_ACTION",
-      "ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "USE_KETTLE"
+      "ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "USE_KETTLE", "DATEFORMAT"
     )
     var isSupported = true
     val invalidOptions = StringBuilder.newBuilder

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 5599ae0..05339cc 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -23,8 +23,7 @@ import java.util
 import java.util.UUID
 
 import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-import scala.collection.mutable.Map
+import scala.collection.mutable.{ArrayBuffer, Map}
 import scala.language.implicitConversions
 import scala.util.Random
 
@@ -1143,6 +1142,8 @@ case class LoadTableUsingKettle(
       val allDictionaryPath = options.getOrElse("all_dictionary_path", "")
       val complex_delimiter_level_1 = options.getOrElse("complex_delimiter_level_1", "\\$")
       val complex_delimiter_level_2 = options.getOrElse("complex_delimiter_level_2", "\\:")
+      val dateFormat = options.getOrElse("dateformat", null)
+      validateDateFormat(dateFormat, table)
       val multiLine = options.getOrElse("multiline", "false").trim.toLowerCase match {
         case "true" => true
         case "false" => false
@@ -1156,6 +1157,7 @@ case class LoadTableUsingKettle(
       carbonLoadModel.setEscapeChar(escapeChar)
       carbonLoadModel.setQuoteChar(quoteChar)
       carbonLoadModel.setCommentChar(commentchar)
+      carbonLoadModel.setDateFormat(dateFormat)
       carbonLoadModel
         .setSerializationNullFormat(
           TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + "," + serializationNullFormat)
@@ -1244,6 +1246,29 @@ case class LoadTableUsingKettle(
     Seq.empty
   }
 
+  private  def validateDateFormat(dateFormat: String, table: CarbonTable): Unit = {
+    val dimensions = table.getDimensionByTableName(tableName).asScala
+    if (dateFormat != null) {
+      if (dateFormat.trim == "") {
+        throw new MalformedCarbonCommandException("Error: Option DateFormat is set an empty
" +
+          "string.")
+      } else {
+        var dateFormats: Array[String] = dateFormat.split(CarbonCommonConstants.COMMA)
+        for (singleDateFormat <- dateFormats) {
+          val dateFormatSplits: Array[String] = singleDateFormat.split(":", 2)
+          val columnName = dateFormatSplits(0).trim.toLowerCase
+          if (!dimensions.exists(_.getColName.equals(columnName))) {
+            throw new MalformedCarbonCommandException("Error: Wrong Column Name " +
+              dateFormatSplits(0) + " is provided in Option DateFormat.")
+          }
+          if (dateFormatSplits.length < 2 || dateFormatSplits(1).trim.isEmpty) {
+            throw new MalformedCarbonCommandException("Error: Option DateFormat is not provided
" +
+              "for " + "Column " + dateFormatSplits(0) + ".")
+          }
+        }
+      }
+    }
+  }
 }
 
 private[sql] case class DropTableCommand(ifExistsSet: Boolean, databaseNameOp: Option[String],

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/integration/spark/src/test/resources/timeStampFormatData1.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/timeStampFormatData1.csv b/integration/spark/src/test/resources/timeStampFormatData1.csv
new file mode 100644
index 0000000..eed124d
--- /dev/null
+++ b/integration/spark/src/test/resources/timeStampFormatData1.csv
@@ -0,0 +1,12 @@
+ID,date,starttime,country,name,phonetype,serialname,salary
+1,2015/7/23,2016-7-23 01:01:30,china,aaa1,phone197,ASD69643,15000
+2,2015/7/24,2016-7-24 01:02:30,china,aaa2,phone756,ASD42892,15001
+3,2015/7/25,2016-7-25 01:03:30,china,aaa3,phone1904,ASD37014,15002
+4,2015/7/26,2016-7-26 01:04:30,china,aaa4,phone2435,ASD66902,15003
+5,2015/7/27,2016-7-27 01:05:30,china,aaa5,phone2441,ASD90633,15004
+6,2015/7/28,2016-7-28 01:06:30,china,aaa6,phone294,ASD59961,15005
+7,2015/7/29,2016-7-29 01:07:30,china,aaa7,phone610,ASD14875,15006
+8,2015/7/30,2016-7-30 01:08:30,china,aaa8,phone1848,ASD57308,15007
+9,2015/7/18,2016-7-18 01:09:30,china,aaa9,phone706,ASD86717,15008
+10,2015/7/19,2016-7-19 01:10:30,usa,aaa10,phone685,ASD30505,15009
+

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/integration/spark/src/test/resources/timeStampFormatData2.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/timeStampFormatData2.csv b/integration/spark/src/test/resources/timeStampFormatData2.csv
new file mode 100644
index 0000000..e8f255b
--- /dev/null
+++ b/integration/spark/src/test/resources/timeStampFormatData2.csv
@@ -0,0 +1,11 @@
+ID,date,starttime,country,name,phonetype,serialname,salary
+11,2015-7-18,2016/07/18 02:02:02,china,aaa11,phone1554,ASD26101,15010
+12,2015-7-19,2016/07/19 02:12:02,china,aaa12,phone1781,ASD85711,15011
+13,2015-7-20,2016/07/20 02:22:02,china,aaa13,phone943,ASD39200,15012
+14,2015-7-21,2016/07/21 02:32:02,china,aaa14,phone1954,ASD80468,15013
+15,2015-7-22,2016/07/22 02:42:02,china,aaa15,phone451,ASD1954,15014
+16,2015-7-23,2016/07/23 02:52:02,china,aaa16,phone390,ASD38513,15015
+17,2015-7-24,2016/07/24 02:42:02,china,aaa17,phone1929,ASD86213,15016
+18,2015-7-25,2016/07/25 02:32:02,usa,aaa18,phone910,ASD88812,15017
+19,2015-7-26,2016/07/26 02:22:02,china,aaa19,phone2151,ASD9316,15018
+20,2015-7-27,2016/07/27 02:12:02,china,aaa20,phone2625,ASD62597,15019

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
new file mode 100644
index 0000000..3fe3c21
--- /dev/null
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.dataload
+
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+import java.sql.Timestamp
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.spark.sql.Row
+
+class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterAll {
+  override def beforeAll {
+    sql("DROP TABLE IF EXISTS t3")
+    sql("""
+           CREATE TABLE IF NOT EXISTS t3
+           (ID Int, date Timestamp, starttime Timestamp, country String,
+           name String, phonetype String, serialname String, salary Int)
+           STORED BY 'carbondata'
+        """)
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+  }
+
+  test("test load data with different timestamp format") {
+      sql(s"""
+           LOAD DATA LOCAL INPATH './src/test/resources/timeStampFormatData1.csv' into table
t3
+           OPTIONS('dateformat' = 'starttime:yyyy-MM-dd HH:mm:ss')
+           """)
+      sql(s"""
+           LOAD DATA LOCAL INPATH './src/test/resources/timeStampFormatData2.csv' into table
t3
+           OPTIONS('dateformat' = ' date : yyyy-MM-dd , StartTime : yyyy/MM/dd HH:mm:ss')
+           """)
+      checkAnswer(
+        sql("SELECT date FROM t3 WHERE ID = 1"),
+        Seq(Row(Timestamp.valueOf("2015-07-23 00:00:00.0")))
+      )
+      checkAnswer(
+        sql("SELECT starttime FROM t3 WHERE ID = 1"),
+        Seq(Row(Timestamp.valueOf("2016-07-23 01:01:30.0")))
+      )
+      checkAnswer(
+        sql("SELECT date FROM t3 WHERE ID = 18"),
+        Seq(Row(Timestamp.valueOf("2015-07-25 00:00:00.0")))
+      )
+      checkAnswer(
+        sql("SELECT starttime FROM t3 WHERE ID = 18"),
+        Seq(Row(Timestamp.valueOf("2016-07-25 02:32:02.0")))
+      )
+  }
+
+  test("test load data with different timestamp format with wrong setting") {
+    try {
+      sql(s"""
+           LOAD DATA LOCAL INPATH './src/test/resources/timeStampFormatData1.csv' into table
t3
+           OPTIONS('dateformat' = '')
+           """)
+      assert(false)
+    } catch {
+      case ex: MalformedCarbonCommandException =>
+        assertResult(ex.getMessage)("Error: Option DateFormat is set an empty string.")
+      case _ => assert(false)
+    }
+
+    try {
+      sql(s"""
+           LOAD DATA LOCAL INPATH './src/test/resources/timeStampFormatData1.csv' into table
t3
+           OPTIONS('dateformat' = 'fasfdas:yyyy/MM/dd')
+           """)
+      assert(false)
+    } catch {
+      case ex: MalformedCarbonCommandException =>
+        assertResult(ex.getMessage)("Error: Wrong Column Name fasfdas is provided in Option
DateFormat.")
+      case _ => assert(false)
+    }
+
+    try {
+      sql(s"""
+           LOAD DATA LOCAL INPATH './src/test/resources/timeStampFormatData1.csv' into table
t3
+           OPTIONS('dateformat' = 'date:  ')
+           """)
+      assert(false)
+    } catch {
+      case ex: MalformedCarbonCommandException =>
+        assertResult(ex.getMessage)("Error: Option DateFormat is not provided for Column
date.")
+      case _ => assert(false)
+    }
+
+    try {
+      sql(s"""
+           LOAD DATA LOCAL INPATH './src/test/resources/timeStampFormatData1.csv' into table
t3
+           OPTIONS('dateformat' = 'date  ')
+           """)
+      assert(false)
+    } catch {
+      case ex: MalformedCarbonCommandException =>
+        assertResult(ex.getMessage)("Error: Option DateFormat is not provided for Column
date  .")
+      case _ => assert(false)
+    }
+
+    try {
+      sql(s"""
+           LOAD DATA LOCAL INPATH './src/test/resources/timeStampFormatData1.csv' into table
t3
+           OPTIONS('dateformat' = ':yyyy/MM/dd  ')
+           """)
+      assert(false)
+    } catch {
+      case ex: MalformedCarbonCommandException =>
+        assertResult(ex.getMessage)("Error: Wrong Column Name  is provided in Option DateFormat.")
+      case _ => assert(false)
+    }
+
+  }
+
+  override def afterAll {
+    sql("DROP TABLE IF EXISTS t3")
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java
b/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java
index 739035a..5a7f14d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java
@@ -67,6 +67,8 @@ public class DataLoadModel {
 
   private String rddIteratorKey;
 
+  private String dateFormat;
+
   private String maxColumns;
   /**
    * @return Returns the schemaInfo.
@@ -215,6 +217,10 @@ public class DataLoadModel {
   public void setCommentCharacter(String commentCharacter) {
     this.commentCharacter = commentCharacter;
   }
+
+  public String getDateFormat() { return dateFormat; }
+
+  public void setDateFormat(String dateFormat) { this.dateFormat = dateFormat; }
   /**
    * @return
    */

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
index e8581ad..ae0853c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
@@ -93,6 +93,8 @@ public class DataProcessTaskStatus implements IDataProcessStatus, Serializable
{
 
   private String rddIteratorKey;
 
+  private String dateFormat;
+
   public DataProcessTaskStatus(String databaseName, String tableName) {
     this.databaseName = databaseName;
     this.tableName = tableName;
@@ -307,4 +309,8 @@ public class DataProcessTaskStatus implements IDataProcessStatus, Serializable
{
   public void setRddIteratorKey(String rddIteratorKey) {
     this.rddIteratorKey = rddIteratorKey;
   }
+
+  public String getDateFormat() { return dateFormat; }
+
+  public void setDateFormat(String dateFormat) { this.dateFormat = dateFormat; }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
index efd4ba8..a21eb66 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
@@ -195,4 +195,6 @@ public interface IDataProcessStatus {
   String getCommentCharacter();
 
   String getRddIteratorKey();
+
+  String getDateFormat();
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
index bc02d1a..eb10549 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
@@ -187,6 +187,7 @@ public class GraphGenerator {
   private String escapeCharacter;
   private String quoteCharacter;
   private String commentCharacter;
+  private String dateFormat;
   /**
    * task id, each spark task has a unique id
    */
@@ -225,6 +226,7 @@ public class GraphGenerator {
     this.taskNo = dataLoadModel.getTaskNo();
     this.quoteCharacter = dataLoadModel.getQuoteCharacter();
     this.commentCharacter = dataLoadModel.getCommentCharacter();
+    this.dateFormat = dataLoadModel.getDateFormat();
     this.factTimeStamp = dataLoadModel.getFactTimeStamp();
     this.segmentId = segmentId;
     this.escapeCharacter = dataLoadModel.getEscapeCharacter();
@@ -570,6 +572,7 @@ public class GraphGenerator {
         graphConfiguration.getColumnAndTableNameColumnMapForAgg());
     seqMeta.setForgienKeyPrimayKeyString(graphConfiguration.getForgienKeyAndPrimaryKeyMapString());
     seqMeta.setTableName(graphConfiguration.getTableName());
+    seqMeta.setDateFormat(dateFormat);
     seqMeta.setModifiedDimension(modifiedDimension);
     seqMeta.setForeignKeyHierarchyString(graphConfiguration.getForeignKeyHierarchyString());
     seqMeta.setPrimaryKeysString(graphConfiguration.getPrimaryKeyString());

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
index c896acf..248a6e0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
@@ -19,7 +19,6 @@
 
 package org.apache.carbondata.processing.surrogatekeysgenerator.csvbased;
 
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -109,9 +108,9 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements
StepMetaIn
    */
   protected Map<String, String[]> hierColumnMap;
   /**
-   * timeFormat
+   * dateFormat
    */
-  protected SimpleDateFormat timeFormat;
+  protected String dateFormat;
   /**
    * timeDimeIndex
    */
@@ -605,6 +604,10 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements
StepMetaIn
     this.generateDimFiles = generateDimFiles;
   }
 
+  public String getDateFormat() { return dateFormat; }
+
+  public void setDateFormat(String dateFormat) { this.dateFormat = dateFormat; }
+
   /**
    * set sensible defaults for a new step
    *
@@ -651,6 +654,7 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements
StepMetaIn
     columnSchemaDetails = "";
     columnsDataTypeString="";
     tableOption = "";
+    dateFormat = "";
   }
 
   // helper method to allocate the arrays
@@ -684,6 +688,7 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements
StepMetaIn
         .append(XMLHandler.addTagValue("complexDelimiterLevel1", complexDelimiterLevel1));
     retval.append("    ")
         .append(XMLHandler.addTagValue("complexDelimiterLevel2", complexDelimiterLevel2));
+    retval.append("    ").append(XMLHandler.addTagValue("dateFormat", dateFormat));
     retval.append("    ").append(XMLHandler.addTagValue("primaryKeysString", primaryKeysString));
     retval.append("    ").append(XMLHandler.addTagValue("carbonMeasureNames", carbonMeasureNames));
     retval.append("    ").append(XMLHandler.addTagValue("actualDimNames", actualDimNames));
@@ -740,6 +745,7 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements
StepMetaIn
       columnPropertiesString = XMLHandler.getTagValue(stepnode, "columnPropertiesString");
       complexDelimiterLevel1 = XMLHandler.getTagValue(stepnode, "complexDelimiterLevel1");
       complexDelimiterLevel2 = XMLHandler.getTagValue(stepnode, "complexDelimiterLevel2");
+      dateFormat = XMLHandler.getTagValue(stepnode, "dateFormat");
       primaryKeysString = XMLHandler.getTagValue(stepnode, "primaryKeysString");
       carbonMeasureNames = XMLHandler.getTagValue(stepnode, "carbonMeasureNames");
       actualDimNames = XMLHandler.getTagValue(stepnode, "actualDimNames");
@@ -1333,6 +1339,7 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements
StepMetaIn
       taskNo = rep.getStepAttributeString(idStep, "taskNo");
       columnSchemaDetails = rep.getStepAttributeString(idStep, "columnSchemaDetails");
       tableOption = rep.getStepAttributeString(idStep, "tableOption");
+      dateFormat = rep.getStepAttributeString(idStep, "dateFormat");
       int nrKeys = rep.countNrStepAttributes(idStep, "lookup_keyfield");
       allocate(nrKeys);
     } catch (Exception e) {
@@ -1386,6 +1393,7 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements
StepMetaIn
       rep.saveStepAttribute(idTransformation, idStep, "taskNo", taskNo);
       rep.saveStepAttribute(idTransformation, idStep, "columnSchemaDetails", columnSchemaDetails);
       rep.saveStepAttribute(idTransformation, idStep, "tableOption", tableOption);
+      rep.saveStepAttribute(idTransformation, idStep, "dateFormat", dateFormat);
     } catch (Exception e) {
       throw new KettleException(
           BaseMessages.getString(pkg, "CarbonStep.Exception.UnableToSaveStepInfoToRepository")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index ca2c7ce..bfd2941 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -266,6 +266,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
 
   private GenericDataType[] complexTypes;
 
+  private DirectDictionaryGenerator[] directDictionaryGenerators;
+
   /**
    * Constructor
    *
@@ -449,7 +451,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
             .parseBoolean(meta.getTableOptionWrapper().get(BAD_RECORDS_LOGGER_ENABLE.getName()));
         String bad_records_action =
             meta.getTableOptionWrapper().get(BAD_RECORDS_ACTION.getName());
-        if(null != bad_records_action) {
+        if (null != bad_records_action) {
           LoggerAction loggerAction = null;
           try {
             loggerAction = LoggerAction.valueOf(bad_records_action.toUpperCase());
@@ -470,6 +472,34 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
               break;
           }
         }
+        HashMap<String, String> dateformatsHashMap = new HashMap<String, String>();
+        if (meta.dateFormat != null) {
+          String[] dateformats = meta.dateFormat.split(CarbonCommonConstants.COMMA);
+          for (String dateFormat:dateformats) {
+            String[] dateFormatSplits = dateFormat.split(":", 2);
+            dateformatsHashMap.put(dateFormatSplits[0].toLowerCase().trim(),
+                dateFormatSplits[1].trim());
+          }
+        }
+        String[] DimensionColumnIds = meta.getDimensionColumnIds();
+        directDictionaryGenerators =
+            new DirectDictionaryGenerator[DimensionColumnIds.length];
+        for (int i = 0; i < DimensionColumnIds.length; i++) {
+          ColumnSchemaDetails columnSchemaDetails = columnSchemaDetailsWrapper.get(
+              DimensionColumnIds[i]);
+          if (columnSchemaDetails.isDirectDictionary()) {
+            String columnName = columnSchemaDetails.getColumnName();
+            DataType columnType = columnSchemaDetails.getColumnType();
+            if (dateformatsHashMap.containsKey(columnName)) {
+              directDictionaryGenerators[i] =
+                  DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(
+                      columnType, dateformatsHashMap.get(columnName));
+            } else {
+              directDictionaryGenerators[i] =
+                  DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(columnType);
+            }
+          }
+        }
       }
       // no more input to be expected...
       if (r == null) {
@@ -836,6 +866,12 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
 
   private void doProcess() throws RuntimeException {
     try {
+      for (DirectDictionaryGenerator directDictionaryGenerator: directDictionaryGenerators)
{
+        if (directDictionaryGenerator != null) {
+          directDictionaryGenerator.initialize();
+        }
+      }
+
       while (true) {
         Object[] r = null;
         synchronized (getRowLock) {
@@ -1176,11 +1212,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
             }
             ColumnSchemaDetails details = columnSchemaDetailsWrapper.get(dimensionColumnIds[m]);
             if (details.isDirectDictionary()) {
-              DirectDictionaryGenerator directDictionaryGenerator1 =
-                  DirectDictionaryKeyGeneratorFactory
-                      .getDirectDictionaryGenerator(details.getColumnType());
               surrogateKeyForHrrchy[0] =
-                  directDictionaryGenerator1.generateDirectSurrogateKey(tuple);
+                  directDictionaryGenerators[m].generateDirectSurrogateKey(tuple);
               if (!isSerialized && surrogateKeyForHrrchy[0] == 1) {
                 addEntryToBadRecords(r, j, columnName,
                     details.getColumnType().name());

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
b/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
index 1a9fe37..70cdd6d 100644
--- a/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
+++ b/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
@@ -37,7 +37,7 @@ public class TimeStampDirectDictionaryGeneratorTest {
   private int surrogateKey = -1;
 
   @Before public void setUp() throws Exception {
-    TimeStampDirectDictionaryGenerator generator = TimeStampDirectDictionaryGenerator.instance;
+    TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator(
);
     surrogateKey = generator.generateDirectSurrogateKey("2015-10-20 12:30:01");
   }
 
@@ -47,7 +47,7 @@ public class TimeStampDirectDictionaryGeneratorTest {
    * @throws Exception
    */
   @Test public void generateDirectSurrogateKey() throws Exception {
-    TimeStampDirectDictionaryGenerator generator = TimeStampDirectDictionaryGenerator.instance;
+    TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator(
);
     // default timestamp format is "yyyy-MM-dd HH:mm:ss" and the data being passed
     // in "dd/MM/yyyy" so the parsing should fail and method should return 1.
     int key = generator.generateDirectSurrogateKey("20/12/2014");
@@ -63,7 +63,7 @@ public class TimeStampDirectDictionaryGeneratorTest {
    * @throws Exception
    */
   @Test public void getValueFromSurrogate() throws Exception {
-    TimeStampDirectDictionaryGenerator generator = TimeStampDirectDictionaryGenerator.instance;
+    TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator(
);
     long valueFromSurrogate = (long) generator.getValueFromSurrogate(surrogateKey);
     Date date = new Date(valueFromSurrogate / 1000);
     SimpleDateFormat timeParser = new SimpleDateFormat(CarbonProperties.getInstance()
@@ -80,7 +80,7 @@ public class TimeStampDirectDictionaryGeneratorTest {
    * @throws Exception
    */
   @Test public void lowerBoundaryValueTest() throws Exception {
-    TimeStampDirectDictionaryGenerator generator = TimeStampDirectDictionaryGenerator.instance;
+    TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator(
);
     long valueFromSurrogate = (long) generator.getValueFromSurrogate(2);
     Date date = new Date(valueFromSurrogate / 1000);
     SimpleDateFormat timeParser = new SimpleDateFormat(CarbonProperties.getInstance()
@@ -98,7 +98,7 @@ public class TimeStampDirectDictionaryGeneratorTest {
    * @throws Exception
    */
   @Test public void upperBoundaryValueTest() throws Exception {
-    TimeStampDirectDictionaryGenerator generator = TimeStampDirectDictionaryGenerator.instance;
+    TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator(
);
     int surrogateKey = generator.generateDirectSurrogateKey("2038-01-01 05:30:00");
     long valueFromSurrogate = (long) generator.getValueFromSurrogate(surrogateKey);
     Date date = new Date(valueFromSurrogate / 1000);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72d076ac/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
index 22c2917..7b15f38 100644
--- a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
@@ -367,6 +367,7 @@ public class StoreCreator {
     dataProcessTaskStatus.setEscapeCharacter("\\");
     dataProcessTaskStatus.setQuoteCharacter("\"");
     dataProcessTaskStatus.setCommentCharacter("#");
+    dataProcessTaskStatus.setDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
     info.setDatabaseName(databaseName);
     info.setTableName(tableName);
 
@@ -476,6 +477,7 @@ public class StoreCreator {
     model.setEscapeCharacter(dataProcessTaskStatus.getEscapeCharacter());
     model.setQuoteCharacter(dataProcessTaskStatus.getQuoteCharacter());
     model.setCommentCharacter(dataProcessTaskStatus.getCommentCharacter());
+    model.setDateFormat(dataProcessTaskStatus.getDateFormat());
     if (null != loadMetadataDetails && !loadMetadataDetails.isEmpty()) {
       model.setLoadNames(
           CarbonDataProcessorUtil.getLoadNameFromLoadMetaDataDetails(loadMetadataDetails));



Mime
View raw message