carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kumarvisha...@apache.org
Subject carbondata git commit: [CARBONDATA-2031] Fix ArrayIndexOutOfBoundException when filter query is applied on column where all values are null and column is noinverted index column
Date Wed, 17 Jan 2018 13:41:44 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 4f9aeaea8 -> 383b2ed35


[CARBONDATA-2031] Fix ArrayIndexOutOfBoundException when filter query is applied on column
where all values are null and column is noinverted index column

Fix ArrayIndexOutOfBoundException when filter query is applied on column where all values
are null and column is noinverted index column.when all the values are null in no inverted
index column the number of exclude filter keys are null, hence just return the bitset if the
exclude filters to be applied are none.

This closes #1809


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/383b2ed3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/383b2ed3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/383b2ed3

Branch: refs/heads/master
Commit: 383b2ed357f93c0a18fcbfb85e36e7156a0ee610
Parents: 4f9aeae
Author: akashrn5 <akashnilugal@gmail.com>
Authored: Tue Jan 16 12:17:53 2018 +0530
Committer: kumarvishal <kumarvishal.1802@gmail.com>
Committed: Wed Jan 17 19:08:41 2018 +0530

----------------------------------------------------------------------
 .../scan/filter/executer/ExcludeFilterExecuterImpl.java     | 4 ++++
 .../dataload/TestNoInvertedIndexLoadAndQuery.scala          | 9 +++++++++
 2 files changed, 13 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/383b2ed3/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index f680579..465bee6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -378,6 +378,10 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
     BitSet bitSet = new BitSet(numerOfRows);
     bitSet.flip(0, numerOfRows);
     byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
+    // filterValues can be null when the dictionary chunk and surrogate size both are one
+    if (filterValues.length == 0) {
+      return bitSet;
+    }
     // binary search can only be applied if column is sorted
     if (isNaturalSorted) {
       int startIndex = 0;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/383b2ed3/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
index baf3997..e8affc4 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
@@ -49,6 +49,7 @@ class TestNoInvertedIndexLoadAndQuery extends QueryTest with BeforeAndAfterAll
{
     sql("DROP TABLE IF EXISTS index2")
     sql("DROP TABLE IF EXISTS hiveNoInvertedIndexTable")
     sql("DROP TABLE IF EXISTS carbonNoInvertedIndexTable")
+    sql("DROP TABLE IF EXISTS testNull")
   }
 
   test("no inverted index load and point query") {
@@ -281,10 +282,18 @@ class TestNoInvertedIndexLoadAndQuery extends QueryTest with BeforeAndAfterAll
{
       true,"NOINVERTEDINDEX")
   }
 
+  test("filter query on dictionary and no inverted index column where all values are null"){
+    sql("""create table testNull (c1 string,c2 int,c3 string,c5 string) STORED BY 'carbondata'
TBLPROPERTIES('DICTIONARY_INCLUDE'='C2','NO_INVERTED_INDEX'='C2')""")
+    sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table testNull OPTIONS('delimiter'=';','fileheader'='c1,c2,c3,c5')""")
+    sql("""select c2 from testNull where c2 is null""").show()
+    checkAnswer(sql("""select c2 from testNull where c2 is null"""), Seq(Row(null), Row(null),
Row(null), Row(null), Row(null), Row(null)))
+  }
+
   override def afterAll {
     sql("drop table if exists index1")
     sql("drop table if exists index2")
     sql("drop table if exists indexFormat")
+    sql("drop table if exists testNull")
     clean
   }
 


Mime
View raw message