carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [49/50] incubator-carbondata git commit: null value handled during compaction
Date Tue, 18 Apr 2017 12:04:25 GMT
null value handled during compaction


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8ed7931b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8ed7931b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8ed7931b

Branch: refs/heads/branch-1.1
Commit: 8ed7931bae6858a47c4989dde7223b20eecc7803
Parents: 4397d05
Author: rahulforallp <rahul.kumar@knoldus.in>
Authored: Tue Apr 18 12:26:51 2017 +0530
Committer: rahulforallp <rahul.kumar@knoldus.in>
Committed: Tue Apr 18 12:26:51 2017 +0530

----------------------------------------------------------------------
 .../rowreader/AddColumnTestCases.scala          | 22 ++++++++++++++++++++
 .../merger/CompactionResultSortProcessor.java   |  7 ++++---
 2 files changed, 26 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8ed7931b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
index e6fb265..06f480b 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
@@ -131,6 +131,28 @@ class AddColumnTestCases extends QueryTest with BeforeAndAfterAll {
     sql("DROP TABLE IF EXISTS carbon_table")
   }
 
+
+  test("test add column compaction") {
+    sql("DROP TABLE IF EXISTS carbon_table")
+    sql(
+      "CREATE TABLE carbon_table(intField int,stringField string,charField string,timestampField
" +
+      "timestamp)STORED BY 'carbondata' TBLPROPERTIES" +
+      "('DICTIONARY_EXCLUDE'='charField')")
+    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO TABLE carbon_table
" +
+        s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO TABLE carbon_table
" +
+        s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO TABLE carbon_table
" +
+        s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO TABLE carbon_table
" +
+        s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+    sql("Alter table carbon_table add columns(decimalField decimal(6,2))")
+
+    sql("Alter table carbon_table compact 'minor'")
+
+    sql("DROP TABLE IF EXISTS carbon_table")
+  }
+
   override def afterAll {
     sql("DROP TABLE IF EXISTS addcolumntest")
     sql("drop table if exists hivetable")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8ed7931b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index ebf3683..8c1f577 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -184,8 +184,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor
{
    *
    * @param resultIteratorList
    */
-  private void processResult(List<RawResultIterator> resultIteratorList)
-      throws Exception {
+  private void processResult(List<RawResultIterator> resultIteratorList) throws Exception
{
     for (RawResultIterator resultIterator : resultIteratorList) {
       while (resultIterator.hasNext()) {
         addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));
@@ -250,7 +249,9 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor
{
   private Object getConvertedMeasureValue(Object value, char aggType) {
     switch (aggType) {
       case CarbonCommonConstants.BIG_DECIMAL_MEASURE:
-        value = ((org.apache.spark.sql.types.Decimal) value).toJavaBigDecimal();
+        if (value != null) {
+          value = ((org.apache.spark.sql.types.Decimal) value).toJavaBigDecimal();
+        }
         return value;
       default:
         return value;


Mime
View raw message