carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-2120]Fixed is null filter issue
Date Fri, 02 Feb 2018 18:54:06 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 27ec6515a -> 6c097cbf3


[CARBONDATA-2120]Fixed is null filter issue

Problem: Is null filter is failing for numeric data type(No dictionary column).

Root cause: Min max calculation is wrong when no dictionary column is not the first column.

As it is not the first column null value can come in between and min max for null value is
getting updated only when first row is null

Solution: Update the min max in all the case when value is null or not null for all type

This closes #1912


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6c097cbf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6c097cbf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6c097cbf

Branch: refs/heads/master
Commit: 6c097cbf310e8d2199e57cde4fcc417122a8a1ca
Parents: 27ec651
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Fri Feb 2 17:53:57 2018 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Sat Feb 3 00:23:39 2018 +0530

----------------------------------------------------------------------
 .../page/statistics/LVStringStatsCollector.java | 28 ++++++++---------
 .../core/util/path/CarbonTablePath.java         |  2 +-
 .../src/test/resources/newsample.csv            |  7 +++++
 .../testsuite/filterexpr/TestIsNullFilter.scala | 32 ++++++++++++++++++++
 4 files changed, 53 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c097cbf/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
index 61acec9..23795c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
@@ -73,28 +73,26 @@ public class LVStringStatsCollector implements ColumnPageStatsCollector
{
   @Override
   public void update(byte[] value) {
     // input value is LV encoded
+    byte[] newValue = null;
     assert (value.length >= 2);
     if (value.length == 2) {
       assert (value[0] == 0 && value[1] == 0);
-      if (min == null && max == null) {
-        min = new byte[0];
-        max = new byte[0];
-      }
-      return;
+      newValue = new byte[0];
+    } else {
+      int length = (value[0] << 8) + (value[1] & 0xff);
+      assert (length > 0);
+      newValue = new byte[value.length - 2];
+      System.arraycopy(value, 2, newValue, 0, newValue.length);
     }
-    int length = (value[0] << 8) + (value[1] & 0xff);
-    assert (length > 0);
-    byte[] v = new byte[value.length - 2];
-    System.arraycopy(value, 2, v, 0, v.length);
     if (min == null && max == null) {
-      min = v;
-      max = v;
+      min = newValue;
+      max = newValue;
     } else {
-      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(min, v) > 0) {
-        min = v;
+      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(min, newValue) > 0) {
+        min = newValue;
       }
-      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(max, v) < 0) {
-        max = v;
+      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(max, newValue) < 0) {
+        max = newValue;
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c097cbf/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index d8c64c4..5a63d2f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -75,7 +75,7 @@ public class CarbonTablePath extends Path {
    * @param carbonFilePath
    */
   public static String getFolderContainingFile(String carbonFilePath) {
-    return carbonFilePath.substring(0, carbonFilePath.lastIndexOf(File.separator));
+    return carbonFilePath.substring(0, carbonFilePath.lastIndexOf('/'));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c097cbf/integration/spark-common-test/src/test/resources/newsample.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/newsample.csv b/integration/spark-common-test/src/test/resources/newsample.csv
new file mode 100644
index 0000000..38cd3dd
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/newsample.csv
@@ -0,0 +1,7 @@
+id,name,time
+1,'one',2014-01-01 08:00:00
+2,'two',2014-01-02 08:02:00
+3,'three',2014-01-03 08:03:00
+4,'four',2014-01-04 08:04:00
+null,null,null
+5,'five',2020-10-20 08:01:20
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c097cbf/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestIsNullFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestIsNullFilter.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestIsNullFilter.scala
new file mode 100644
index 0000000..9e85c9e
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestIsNullFilter.scala
@@ -0,0 +1,32 @@
+package org.apache.carbondata.spark.testsuite.filterexpr
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestIsNullFilter extends QueryTest with BeforeAndAfterAll {
+  override def beforeAll: Unit = {
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+    sql("drop table if exists main")
+    sql("create table main(id int, name string, time timestamp) STORED BY 'org.apache.carbondata.format'")
+    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/newsample.csv' into table main OPTIONS('bad_records_action'='force')")
+  }
+
+  test("select * from main where time is null") {
+    checkAnswer(
+      sql("select count(*) from main where time is null"),
+      Seq(Row(1)))
+  }
+
+  override def afterAll: Unit = {
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+    sql("drop table if exists main")
+  }
+
+}


Mime
View raw message