parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alexleven...@apache.org
Subject incubator-parquet-mr git commit: PARQUET-173: Fixes `StatisticsFilter` for `And` filter predicate
Date Tue, 03 Feb 2015 20:53:47 GMT
Repository: incubator-parquet-mr
Updated Branches:
  refs/heads/master 3df3372a1 -> 80417356f


PARQUET-173: Fixes `StatisticsFilter` for `And` filter predicate

<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/apache/incubator-parquet-mr/108)
<!-- Reviewable:end -->

Author: Cheng Lian <lian@databricks.com>

Closes #108 from liancheng/PARQUET-173 and squashes the following commits:

d188f0b [Cheng Lian] Fixes test case
be2c8a1 [Cheng Lian] Fixes `StatisticsFilter` for `And` filter predicate


Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/80417356
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/80417356
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/80417356

Branch: refs/heads/master
Commit: 80417356f04c5ee1cd6f636e9b043db3f2de24f2
Parents: 3df3372
Author: Cheng Lian <lian@databricks.com>
Authored: Tue Feb 3 12:53:37 2015 -0800
Committer: Alex Levenson <alexlevenson@twitter.com>
Committed: Tue Feb 3 12:53:37 2015 -0800

----------------------------------------------------------------------
 .../java/parquet/filter2/statisticslevel/StatisticsFilter.java | 6 +++++-
 .../parquet/filter2/statisticslevel/TestStatisticsFilter.java  | 4 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/80417356/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java
b/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java
index f7eea41..d6a484c 100644
--- a/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java
+++ b/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java
@@ -242,7 +242,11 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean>
{
 
   @Override
   public Boolean visit(And and) {
-    return and.getLeft().accept(this) && and.getRight().accept(this);
+    // seems unintuitive to put an || not an && here but we can
+    // drop a chunk of records if we know that either the left or
+    // the right predicate agrees that no matter what we don't
+    // need this chunk.
+    return and.getLeft().accept(this) || and.getRight().accept(this);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/80417356/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
b/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
index 03502d2..d558f9b 100644
--- a/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
+++ b/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
@@ -222,8 +222,8 @@ public class TestStatisticsFilter {
     FilterPredicate yes = eq(intColumn, 9);
     FilterPredicate no = eq(doubleColumn, 50D);
     assertTrue(canDrop(and(yes, yes), columnMetas));
-    assertFalse(canDrop(and(yes, no), columnMetas));
-    assertFalse(canDrop(and(no, yes), columnMetas));
+    assertTrue(canDrop(and(yes, no), columnMetas));
+    assertTrue(canDrop(and(no, yes), columnMetas));
     assertFalse(canDrop(and(no, no), columnMetas));
   }
 


Mime
View raw message