carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: Fix Date and Timestamp format issues
Date Thu, 09 Mar 2017 15:11:29 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 847ec3819 -> 5fb1ec088


Fix Date and Timestamp format issues

Fix Date and Timestamp format issues

Fix comments

fix scala style issues


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/374b00f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/374b00f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/374b00f8

Branch: refs/heads/master
Commit: 374b00f8fd96672a689149e98c069f00d1b25cb1
Parents: 847ec38
Author: chenliang613 <chenliang613@huawei.com>
Authored: Wed Mar 8 21:19:10 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Thu Mar 9 23:09:14 2017 +0800

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |  6 ++++--
 .../examples/AllDictionaryExample.scala         |  6 +++---
 .../carbondata/examples/AlluxioExample.scala    |  6 +++---
 .../carbondata/examples/CarbonExample.scala     |  6 +++---
 .../examples/DataUpdateDeleteExample.scala      |  6 +++---
 examples/spark2/src/main/resources/data.csv     | 20 ++++++++++----------
 .../examples/SparkSessionExample.scala          | 11 +++++++----
 7 files changed, 33 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/374b00f8/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 9e890ef..8d1f2d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -383,7 +383,8 @@ public final class CarbonCommonConstants {
    */
   public static final String CARBON_TIMESTAMP_DEFAULT_FORMAT = "yyyy-MM-dd HH:mm:ss";
   /**
-   * CARBON_TIMESTAMP_DEFAULT_FORMAT
+   * Property for specifying the format of TIMESTAMP data type column.
+   * e.g. yyyy/MM/dd HH:mm:ss, or using CARBON_TIMESTAMP_DEFAULT_FORMAT
    */
   public static final String CARBON_TIMESTAMP_FORMAT = "carbon.timestamp.format";
 
@@ -392,7 +393,8 @@ public final class CarbonCommonConstants {
    */
   public static final String CARBON_DATE_DEFAULT_FORMAT = "yyyy-MM-dd";
   /**
-   * CARBON_DATE_DEFAULT_FORMAT
+   * Property for specifying the format of DATE data type column.
+   * e.g. yyyy/MM/dd , or using CARBON_DATE_DEFAULT_FORMAT
    */
   public static final String CARBON_DATE_FORMAT = "carbon.date.format";
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/374b00f8/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
index 558139a..df5ee01 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
@@ -31,15 +31,15 @@ object AllDictionaryExample {
     // extract all dictionary files from source data
     AllDictionaryUtil.extractDictionary(cc.sparkContext,
       testData, allDictFile, csvHeader, dictCol)
-    // Specify timestamp format based on raw data
+    // Specify date format based on raw data
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
 
     cc.sql("DROP TABLE IF EXISTS t3")
 
     cc.sql("""
            CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Timestamp, country String,
+           (ID Int, date Date, country String,
            name String, phonetype String, serialname String, salary Int,floatField float)
            STORED BY 'carbondata'
            """)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/374b00f8/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
b/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
index 826819d..12901b5 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
@@ -36,15 +36,15 @@ object AlluxioExample {
     cc.sparkContext.hadoopConfiguration.set("fs.alluxio.impl", "alluxio.hadoop.FileSystem")
     FileFactory.getConfiguration.set("fs.alluxio.impl", "alluxio.hadoop.FileSystem")
 
-    // Specify timestamp format based on raw data
+    // Specify date format based on raw data
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
 
     cc.sql("DROP TABLE IF EXISTS t3")
 
     cc.sql("""
            CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Timestamp, country String,
+           (ID Int, date Date, country String,
            name String, phonetype String, serialname String, salary Int)
            STORED BY 'carbondata'
            """)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/374b00f8/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
b/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
index c6c103b..abf3aa9 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
@@ -27,16 +27,16 @@ object CarbonExample {
     val cc = ExampleUtils.createCarbonContext("CarbonExample")
     val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
 
-    // Specify timestamp format based on raw data
+    // Specify date format based on raw data
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
 
     cc.sql("DROP TABLE IF EXISTS t3")
 
     // Create table, 6 dimensions, 1 measure
     cc.sql("""
            CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Timestamp, country String,
+           (ID Int, date Date, country String,
            name String, phonetype String, serialname char(10), salary Int)
            STORED BY 'carbondata'
            """)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/374b00f8/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
index a6c35cc..7be392d 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataUpdateDeleteExample.scala
@@ -28,9 +28,9 @@ object DataUpdateDeleteExample {
     val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
     val testData1 = ExampleUtils.currentPath + "/src/main/resources/data_update.csv"
 
-    // Specify timestamp format based on raw data
+    // Specify date format based on raw data
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
 
     cc.sql("DROP TABLE IF EXISTS t3")
     cc.sql("DROP TABLE IF EXISTS update_table")
@@ -38,7 +38,7 @@ object DataUpdateDeleteExample {
     // Create table, 6 dimensions, 1 measure
     cc.sql("""
            CREATE TABLE IF NOT EXISTS t3
-           (ID Int, date Timestamp, country String,
+           (ID Int, date Date, country String,
            name String, phonetype String, serialname char(10), salary Int)
            STORED BY 'carbondata'
            """)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/374b00f8/examples/spark2/src/main/resources/data.csv
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/resources/data.csv b/examples/spark2/src/main/resources/data.csv
index da474a0..b7adf55 100644
--- a/examples/spark2/src/main/resources/data.csv
+++ b/examples/spark2/src/main/resources/data.csv
@@ -1,10 +1,10 @@
-1,10,100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23 11:01:01,aaa,2.5,'foo'#'bar'#'world'
-5,17,140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27 11:01:02,bbb,2.5,'foo'#'bar'#'world'
-1,11,100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23 11:01:03,ccc,2.5,'foo'#'bar'#'world'
-1,10,150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24 11:01:04,ddd,2.5,'foo'#'bar'#'world'
-1,10,100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23 11:01:05,eeee,3.5,'foo'#'bar'#'world'
-3,14,160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26 11:01:06,ff,2.5,'foo'#'bar'#'world'
-2,10,100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23 11:01:07,ggg,2.5,'foo'#'bar'#'world'
-1,10,100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23 11:01:08,hhh,2.5,'foo'#'bar'#'world'
-4,16,130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23 11:01:09,iii,2.5,'foo'#'bar'#'world'
-1,10,100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23 11:01:10,jjj,2.5,'foo'#'bar'#'world'
+1,10,100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23,aaa,2.5,'foo'#'bar'#'world'
+5,17,140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27,bbb,2.5,'foo'#'bar'#'world'
+1,11,100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23,ccc,2.5,'foo'#'bar'#'world'
+1,10,150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24,ddd,2.5,'foo'#'bar'#'world'
+1,10,100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23,eeee,3.5,'foo'#'bar'#'world'
+3,14,160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26,ff,2.5,'foo'#'bar'#'world'
+2,10,100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23,ggg,2.5,'foo'#'bar'#'world'
+1,10,100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23,hhh,2.5,'foo'#'bar'#'world'
+4,16,130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23,iii,2.5,'foo'#'bar'#'world'
+1,10,100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23,jjj,2.5,'foo'#'bar'#'world'

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/374b00f8/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
index 8e01fec..381b2e8 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
@@ -60,7 +60,8 @@ object SparkSessionExample {
     spark.sparkContext.setLogLevel("WARN")
 
     CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+        .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
 
     // Create table
     spark.sql(
@@ -116,8 +117,8 @@ object SparkSessionExample {
       s"""
          | INSERT INTO TABLE carbon_table
          | SELECT shortField, intField, bigintField, doubleField, stringField,
-         | from_unixtime(unix_timestamp(timestampField,'yyyy/M/dd')) timestampField, decimalField,
-         | cast(to_date(from_unixtime(unix_timestamp(dateField,'yyyy/M/dd'))) as date), charField
+         | from_unixtime(unix_timestamp(timestampField,'yyyy/MM/dd HH:mm:ss')) timestampField,
+         | decimalField,from_unixtime(unix_timestamp(dateField,'yyyy/MM/dd')), charField
          | FROM csv_table
        """.stripMargin)
 
@@ -127,8 +128,10 @@ object SparkSessionExample {
              where stringfield = 'spark' and decimalField > 40
               """).show
 
+    // Shows with raw data's timestamp format
     spark.sql("""
-             SELECT *
+             SELECT
+             stringField, date_format(timestampField, "yyyy/MM/dd HH:mm:ss") as timestampField
              FROM carbon_table where length(stringField) = 5
               """).show
 


Mime
View raw message