carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [01/47] incubator-carbondata git commit: [CARBONDATA-79] When timestamp is in the middle of other complex types, data load is failing (#841)
Date Mon, 01 Aug 2016 10:04:59 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 10ed89ab5 -> 80cfa39c6


[CARBONDATA-79] When timestamp is in the middle of other complex types, data load is failing
(#841)



Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a326bb8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a326bb8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a326bb8f

Branch: refs/heads/master
Commit: a326bb8f7e71a6900fd977e9f3b63998d0abca2d
Parents: ad48c4c
Author: nareshpr <prnaresh.naresh@gmail.com>
Authored: Tue Jul 19 16:57:39 2016 +0530
Committer: Venkata Ramana G <g.ramana.v1@gmail.com>
Committed: Tue Jul 19 16:57:39 2016 +0530

----------------------------------------------------------------------
 integration/spark/src/test/resources/timestampdata.csv  |  2 ++
 .../TestComplexPrimitiveTimestampDirectDictionary.scala | 12 ++++++++++++
 .../csvbased/CarbonCSVBasedDimSurrogateKeyGen.java      |  2 ++
 3 files changed, 16 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a326bb8f/integration/spark/src/test/resources/timestampdata.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/timestampdata.csv b/integration/spark/src/test/resources/timestampdata.csv
new file mode 100644
index 0000000..b0339bc
--- /dev/null
+++ b/integration/spark/src/test/resources/timestampdata.csv
@@ -0,0 +1,2 @@
+imei001,rat$aaa1,111$111,2015-01-01 13:00:00.000$2015-01-01 13:00:00.000,16$64,babu$001
+imei002,rat$aaa2,111$112,2015-01-01 13:00:00.000$2015-01-02 13:00:00.000,16$65,babu$002
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a326bb8f/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
index 43bf7ea..611a1ec 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
@@ -53,6 +53,18 @@ class TestComplexPrimitiveTimestampDirectDictionary extends QueryTest with
Befor
      sql("select * from complexhivetimestamptable"))
   }
   
+  test("timestamp complex type in the middle of complex types") {
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss.SSS")
+    sql("CREATE TABLE testtimestampcarbon(imei string,rat array<string>, sid array<int>,
end_time array<Timestamp>, probeid array<double>, contact struct<name:string,
id:string>)STORED BY 'org.apache.carbondata.format'")
+    sql("LOAD DATA local inpath './src/test/resources/timestampdata.csv' INTO TABLE testtimestampcarbon
options('DELIMITER'=',', 'QUOTECHAR'='\"','COMPLEX_DELIMITER_LEVEL_1'='$', 'FILEHEADER'='imei,rat,sid,end_time,probeid,contact')")
+    sql("CREATE TABLE testtimestamphive(imei string,rat array<string>, sid array<int>,
end_time array<Timestamp>, probeid array<double>, contact struct<name:string,
id:string>)row format delimited fields terminated by ',' collection items terminated by
'$'")
+    sql("LOAD DATA local inpath './src/test/resources/timestampdata.csv' INTO TABLE testtimestamphive")
+    checkAnswer(sql("select * from testtimestampcarbon"), sql("select * from testtimestamphive"))
+    sql("drop table if exists testtimestampcarbon")
+    sql("drop table if exists testtimestamphive")
+  }
+  
   override def afterAll {
 	  sql("drop table if exists complexcarbontimestamptable")
     sql("drop table if exists complexhivetimestamptable")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a326bb8f/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
index dac4b6c..f9c3b6f 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
@@ -357,6 +357,8 @@ public abstract class CarbonCSVBasedDimSurrogateKeyGen {
               columnsInfo.getColumnSchemaDetailsWrapper().get(eachPrimitive.getColumnId());
           if (columnSchemaDetails.isDirectDictionary()) {
             isDirectDictionary.add(true);
+          } else {
+            isDirectDictionary.add(false);
           }
         }
       } else {


Mime
View raw message