carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [1/2] incubator-carbondata git commit: fix dictionary with null value
Date Tue, 05 Jul 2016 22:17:17 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master dc12d1040 -> beb0c9fd8


fix dictionary with null value

fix testcase


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/525ceb8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/525ceb8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/525ceb8e

Branch: refs/heads/master
Commit: 525ceb8e7c96b82de490a4b0cc4eb1b4f8fbe90c
Parents: dc12d10
Author: eason-lyx <linyixin@huawei.com>
Authored: Tue Jul 5 14:40:12 2016 +0800
Committer: chenliang613 <chenliang613@apache.org>
Committed: Wed Jul 6 03:44:40 2016 +0530

----------------------------------------------------------------------
 .../spark/rdd/CarbonGlobalDictionaryRDD.scala   |  4 +++-
 .../test/resources/lessthandatacolumndata.csv   | 11 +++++++++++
 .../TestDataWithDicExcludeAndInclude.scala      |  8 +++++++-
 .../dataload/TestLoadDataWithHiveSyntax.scala   | 20 ++++++++++++++++++++
 4 files changed, 41 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/525ceb8e/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index a78daa8..d3f7913 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -67,7 +67,9 @@ case class PrimitiveParser(dimension: CarbonDimension,
 
   def parseString(input: String): Unit = {
     if (hasDictEncoding) {
-      set.add(input)
+      if (StringUtils.isNotEmpty(input)) {
+        set.add(input)
+      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/525ceb8e/integration/spark/src/test/resources/lessthandatacolumndata.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/lessthandatacolumndata.csv b/integration/spark/src/test/resources/lessthandatacolumndata.csv
new file mode 100644
index 0000000..98f0daf
--- /dev/null
+++ b/integration/spark/src/test/resources/lessthandatacolumndata.csv
@@ -0,0 +1,11 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,ASD69643,15000
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001
+3,2015/7/25,china
+4,2015/7/26,china,aaa4,phone2435,ASD66902,15003
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004
+6,2015/7/28,china,aaa6,phone294,ASD59961,15005
+7,2015/7/29,china,aaa7,phone610,
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/525ceb8e/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
index e5c25dc..b381079 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
@@ -39,6 +39,10 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with
BeforeA
     filePath = pwd + "/src/test/resources/emptyDimensionData.csv"
   }
 
+  def dropTable() = {
+    sql("DROP TABLE IF EXISTS t3")
+  }
+
   def buildTable() = {
     try {
       sql(
@@ -69,12 +73,14 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with
BeforeA
   }
 
   override def beforeAll {
+    dropTable
     buildTestData
     buildTable
     loadTable
   }
 
   test("test load data with dictionary exclude & include and with empty dimension") {
+    sql("select ID from t3").show()
     checkAnswer(
       sql("select ID from t3"), Seq(Row(1), Row(2), Row(3), Row(4), Row(5), Row(6), Row(7),
         Row(8), Row(9), Row(10), Row(11), Row(12), Row(13), Row(14), Row(15), Row(16), Row
@@ -83,7 +89,7 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with
BeforeA
   }
 
   override def afterAll {
-    sql("drop table t3")
+    dropTable
     CarbonProperties.getInstance()
       .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/525ceb8e/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
index 1a6259a..e32d9c6 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
@@ -293,6 +293,26 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll
{
     sql("DROP TABLE IF EXISTS t3")
   }
 
+  test("test data which contain column less than schema"){
+    sql("DROP TABLE IF EXISTS t3")
+
+    sql(
+      """
+           CREATE TABLE IF NOT EXISTS t3
+           (ID Int, date Timestamp, country String,
+           name String, phonetype String, serialname String, salary Int)
+           STORED BY 'org.apache.carbondata.format'
+
+      """)
+
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+    sql(s"""
+         LOAD DATA LOCAL INPATH './src/test/resources/lessthandatacolumndata.csv' into table
t3
+        """)
+    checkAnswer(sql("select count(*) from t3"),Seq(Row(10)))
+  }
+
   override def afterAll {
     sql("drop table carbontable")
     sql("drop table hivetable")


Mime
View raw message