carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kunalkap...@apache.org
Subject carbondata git commit: [CARBONDATA-2940]Fix BufferUnderFlowException for ComplexPushDown
Date Tue, 18 Sep 2018 12:00:39 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 02a89a9c6 -> 61fcdf286


[CARBONDATA-2940]Fix BufferUnderFlowException for ComplexPushDown

Problem:
BufferUnderFlow Exception was thrown, as buffer data was filled with wrong values.
Solution:
Check for if column is of Primitive Dimension column with ComplexParentOrdinal = -1 and not
Complex and increment Buffer index for NoDictionaryKeys.

This closes #2727


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/61fcdf28
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/61fcdf28
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/61fcdf28

Branch: refs/heads/master
Commit: 61fcdf286cabfce199b402b63ffc55bbf5cc64de
Parents: 02a89a9
Author: Indhumathi27 <indhumathim27@gmail.com>
Authored: Mon Sep 17 12:32:49 2018 +0530
Committer: kunal642 <kunalkapoor642@gmail.com>
Committed: Tue Sep 18 17:29:49 2018 +0530

----------------------------------------------------------------------
 .../core/datastore/blocklet/BlockletEncodedColumnPage.java        | 2 +-
 .../core/scan/collector/impl/DictionaryBasedResultCollector.java  | 3 +++
 .../spark/testsuite/complexType/TestComplexDataType.scala         | 3 +++
 3 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/61fcdf28/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java
index 135b1e2..da4e211 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java
@@ -213,7 +213,7 @@ public class BlockletEncodedColumnPage {
           new DecoderBasedFallbackEncoder(encodedColumnPage, pageIndex, localDictionaryGenerator)));
     } else {
       fallbackFutureQueue.add(fallbackExecutorService.submit(
-          new ActualDataBasedFallbackEncoder(encodedColumnPage, encodedColumnPageList.size())));
+          new ActualDataBasedFallbackEncoder(encodedColumnPage, pageIndex)));
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/61fcdf28/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index 1faf2fd..69b142f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -231,6 +231,9 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
         childColumnByteBuffer
             .put(queryDimensions[i].getDimension(), buffer);
         mergedComplexDimensionDataMap.put(complexParentOrdinal, childColumnByteBuffer);
+      } else if (!queryDimensions[i].getDimension().isComplex()) {
+        // If Dimension is not a Complex Column, then increment index for noDictionaryComplexColumn
+        noDictionaryComplexColumnIndex++;
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/61fcdf28/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
index 8527380..220451b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
@@ -1015,6 +1015,9 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
       "create table table1 (id int, name string, structField struct<intval:int, stringval:string>)
stored by 'carbondata'")
     sql("insert into table1 values(null,'aaa','23$bb')")
     checkAnswer(sql("select * from table1"),Seq(Row(null,"aaa", Row(23,"bb"))))
+    checkAnswer(sql("select id,name,structField.intval,structField.stringval from table1"),Seq(Row(null,"aaa",23,"bb")))
+    checkAnswer(sql("select id,name,structField.intval,structField.stringval,name from table1"),Seq(Row(null,"aaa",23,"bb","aaa")))
+    checkAnswer(sql("select id,name,structField.intval,name,structField.stringval from table1"),Seq(Row(null,"aaa",23,"aaa","bb")))
   }
 
 }


Mime
View raw message