drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [3/5] drill git commit: DRILL-2353: Add interpreter based partition pruning.
Date Fri, 20 Mar 2015 05:15:20 GMT
DRILL-2353: Add interpreter based partition pruning.

Integrate Jacques's interpreter based partition pruning with Jason's interpreter refactoring
that removed interpreter module and added that functionality in the exec module.

Ensure boolean operators are correctly handled when traversing expression tree to find partition
filters.

Resolve merge conflicts after rebasing to master branch.

Additional fixes for handling OR conditions.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/48c9c01d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/48c9c01d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/48c9c01d

Branch: refs/heads/master
Commit: 48c9c01df011d63fa2c118624fe91c085bbe1883
Parents: 7af5f9a
Author: Aman Sinha <asinha@mapr.com>
Authored: Sun Feb 15 23:04:50 2015 -0800
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Thu Mar 19 22:14:57 2015 -0700

----------------------------------------------------------------------
 .../planner/sql/HivePartitionDescriptor.java    |   6 +
 .../org/apache/drill/exec/expr/TestPrune.java   |  38 +++
 .../fn/interpreter/InterpreterEvaluator.java    |  11 +
 .../planner/FileSystemPartitionDescriptor.java  |  22 +-
 .../drill/exec/planner/PartitionDescriptor.java |   7 +
 .../exec/planner/logical/DrillRuleSets.java     |   8 +-
 .../partition/FindPartitionConditions.java      | 302 +++++++++++++++++++
 .../logical/partition/PruneScanRule.java        | 296 ++++++++++++++++++
 .../org/apache/drill/exec/expr/TestPrune.java   |  47 +++
 .../exec/planner/logical/FilterSplitTest.java   | 170 +++++++++++
 10 files changed, 904 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionDescriptor.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionDescriptor.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionDescriptor.java
index e6ca21e..8307dff 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionDescriptor.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionDescriptor.java
@@ -53,4 +53,10 @@ public class HivePartitionDescriptor implements PartitionDescriptor {
   public int getMaxHierarchyLevel() {
     return MAX_NESTED_SUBDIRS;
   }
+
+  @Override
+  public Integer getIdIfValid(String name) {
+    return partitionMap.get(name);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/exec/interpreter/src/test/java/org/apache/drill/exec/expr/TestPrune.java
----------------------------------------------------------------------
diff --git a/exec/interpreter/src/test/java/org/apache/drill/exec/expr/TestPrune.java b/exec/interpreter/src/test/java/org/apache/drill/exec/expr/TestPrune.java
new file mode 100644
index 0000000..7e75165
--- /dev/null
+++ b/exec/interpreter/src/test/java/org/apache/drill/exec/expr/TestPrune.java
@@ -0,0 +1,38 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.expr;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.util.TestTools;
+import org.junit.Test;
+
+public class TestPrune extends BaseTestQuery {
+
+  String MULTILEVEL = TestTools.getWorkingPath() + "/../java-exec/src/test/resources/multilevel";
+
+  @Test
+  public void pruneCompound() throws Exception {
+    test(String.format("select * from dfs.`%s/csv` where x is null and dir1 in ('Q1', 'Q2')",
MULTILEVEL));
+  }
+
+  @Test
+  public void pruneSimple() throws Exception {
+    test(String.format("select * from dfs.`%s/csv` where dir1 in ('Q1', 'Q2')", MULTILEVEL));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java
index 4f8e126..35c35ec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java
@@ -19,6 +19,9 @@ package org.apache.drill.exec.expr.fn.interpreter;
 
 import com.google.common.base.Preconditions;
 import io.netty.buffer.DrillBuf;
+
+import java.lang.reflect.Field;
+
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.expression.BooleanOperator;
 import org.apache.drill.common.expression.CastExpression;
@@ -31,6 +34,10 @@ import org.apache.drill.common.expression.NullExpression;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.expression.TypedNullConstant;
 import org.apache.drill.common.expression.ValueExpressions;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
 import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.exec.expr.DrillFuncHolderExpr;
@@ -42,7 +49,11 @@ import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
 import org.apache.drill.exec.expr.fn.DrillSimpleFuncHolder;
 import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.DateHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
 import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.expr.holders.TimeHolder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
 import org.apache.drill.exec.expr.holders.ValueHolder;
 import org.apache.drill.exec.ops.QueryDateTimeInfo;
 import org.apache.drill.exec.ops.UdfUtilities;

http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
index 4c1f8e8..9ad14b1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
@@ -17,6 +17,11 @@
  */
 package org.apache.drill.exec.planner;
 
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+
 // partition descriptor for file system based tables
 public class FileSystemPartitionDescriptor implements PartitionDescriptor {
 
@@ -24,10 +29,14 @@ public class FileSystemPartitionDescriptor implements PartitionDescriptor
{
 
   private final String partitionLabel;
   private final int partitionLabelLength;
+  private final Map<String, Integer> partitions = Maps.newHashMap();
 
   public FileSystemPartitionDescriptor(String partitionLabel) {
     this.partitionLabel = partitionLabel;
     this.partitionLabelLength = partitionLabel.length();
+    for(int i =0; i < 10; i++){
+      partitions.put(partitionLabel + i, i);
+    }
   }
 
   @Override
@@ -38,11 +47,22 @@ public class FileSystemPartitionDescriptor implements PartitionDescriptor
{
 
   @Override
   public boolean isPartitionName(String name) {
-    return name.matches(partitionLabel +"[0-9]");
+    return partitions.containsKey(name);
+  }
+
+  @Override
+  public Integer getIdIfValid(String name) {
+    return partitions.get(name);
   }
 
   @Override
   public int getMaxHierarchyLevel() {
     return MAX_NESTED_SUBDIRS;
   }
+
+  public String getName(int index){
+    return partitionLabel + index;
+  }
+
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PartitionDescriptor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PartitionDescriptor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PartitionDescriptor.java
index 02a6a8f..35fdae9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PartitionDescriptor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PartitionDescriptor.java
@@ -31,6 +31,13 @@ public interface PartitionDescriptor {
   // Given a column name return boolean to indicate if its a partition column or not
   public boolean isPartitionName(String name);
 
+  /**
+   * Check to see if the name is a partition name.
+   * @param name The field name you want to compare to partition names.
+   * @return Return index if valid, otherwise return null;
+   */
+  public Integer getIdIfValid(String name);
+
   // Maximum level of partition nesting/ hierarchy supported
   public int getMaxHierarchyLevel();
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
index 496bc9a..b1a7189 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
@@ -24,6 +24,7 @@ import java.util.List;
 import net.hydromatic.optiq.tools.RuleSet;
 
 import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.planner.logical.partition.PruneScanRule;
 import org.apache.drill.exec.planner.physical.ConvertCountToDirectScan;
 import org.apache.drill.exec.planner.physical.FilterPrule;
 import org.apache.drill.exec.planner.physical.HashAggPrule;
@@ -103,8 +104,11 @@ public class DrillRuleSets {
 //      PushSortPastProjectRule.INSTANCE, //
 
       DrillPushProjIntoScan.INSTANCE,
-      DrillPushPartitionFilterIntoScan.FILTER_ON_PROJECT,
-      DrillPushPartitionFilterIntoScan.FILTER_ON_SCAN,
+
+//      DrillPushPartitionFilterIntoScan.FILTER_ON_PROJECT,
+//      DrillPushPartitionFilterIntoScan.FILTER_ON_SCAN,
+      PruneScanRule.getFilterOnProject(context),
+      PruneScanRule.getFilterOnScan(context),
 
       ////////////////////////////////
       DrillScanRule.INSTANCE,

http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/FindPartitionConditions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/FindPartitionConditions.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/FindPartitionConditions.java
new file mode 100644
index 0000000..3acf29d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/FindPartitionConditions.java
@@ -0,0 +1,302 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical.partition;
+
+import java.util.ArrayDeque;
+import java.util.BitSet;
+import java.util.Deque;
+import java.util.List;
+import java.util.Iterator;
+
+import org.eigenbase.rex.RexBuilder;
+import org.eigenbase.rex.RexCall;
+import org.eigenbase.rex.RexCorrelVariable;
+import org.eigenbase.rex.RexDynamicParam;
+import org.eigenbase.rex.RexFieldAccess;
+import org.eigenbase.rex.RexInputRef;
+import org.eigenbase.rex.RexLiteral;
+import org.eigenbase.rex.RexNode;
+import org.eigenbase.rex.RexOver;
+import org.eigenbase.rex.RexRangeRef;
+import org.eigenbase.rex.RexVisitorImpl;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.fun.SqlRowOperator;
+import org.eigenbase.sql.fun.SqlStdOperatorTable;
+import org.eigenbase.util.Stacks;
+import org.eigenbase.util.Util;
+
+import com.google.common.collect.Lists;
+
+
+public class FindPartitionConditions extends RexVisitorImpl<Void> {
+  /** Whether an expression is a directory filter, and if so, whether
+   * it can be pushed into the scan.
+   */
+  enum PushDirFilter {
+    NO_PUSH, PUSH
+  }
+
+  /**
+   * During top-down traversal of the expression tree, keep track of the
+   * boolean operators such that if a directory filter is found, it will
+   * be added as a child of the current boolean operator.
+   *
+   * NOTE: this auxiliary class is necessary because RexNodes are immutable.
+   * If they were mutable, we could have easily added/dropped inputs as we
+   * encountered directory filters.
+   */
+  public class BooleanOpState {
+    private SqlOperator booleanOp;
+    private List<RexNode> children = Lists.newArrayList();
+    public BooleanOpState(SqlOperator op) {
+      booleanOp = op;
+    }
+    public SqlOperator getOp() {
+      return booleanOp;
+    }
+    public void addChild(RexNode n) {
+      children.add(n);
+    }
+    public List<RexNode> getChildren() {
+      return children;
+    }
+    public void clear() {
+      children.clear();
+    }
+  }
+
+  private final BitSet dirs;
+
+  private final List<PushDirFilter> pushStatusStack =  Lists.newArrayList();
+  private final Deque<SqlOperator> parentCallTypeStack = new ArrayDeque<SqlOperator>();
+  private final Deque<BooleanOpState> opStack = new ArrayDeque<BooleanOpState>();
+
+  private RexBuilder builder = null;
+  private RexNode resultCondition = null;
+
+  public FindPartitionConditions(BitSet dirs) {
+    // go deep
+    super(true);
+    this.dirs = dirs;
+  }
+
+  public FindPartitionConditions(BitSet dirs, RexBuilder builder) {
+    // go deep
+    super(true);
+    this.dirs = dirs;
+    this.builder = builder;
+  }
+
+  public void analyze(RexNode exp) {
+    assert pushStatusStack.isEmpty();
+
+    exp.accept(this);
+
+    // Deal with top of stack
+    assert pushStatusStack.size() == 1;
+    assert parentCallTypeStack.isEmpty();
+    PushDirFilter rootPushDirFilter = pushStatusStack.get(0);
+    if (rootPushDirFilter == PushDirFilter.PUSH) {
+      // The entire subtree was directory filter, so add it to the result.
+      addResult(exp);
+    }
+    pushStatusStack.clear();
+  }
+
+  public RexNode getFinalCondition() {
+    return resultCondition;
+  }
+
+  private Void pushVariable() {
+    pushStatusStack.add(PushDirFilter.NO_PUSH);
+    return null;
+  }
+
+  private void addResult(RexNode exp) {
+    // when we find a directory filter, add it to the current boolean operator's
+    // children (if one exists)
+    if (!opStack.isEmpty()) {
+      BooleanOpState op = opStack.peek();
+      op.addChild(exp);
+    } else {
+      resultCondition = exp;
+    }
+  }
+
+  /**
+   * For an OR node that is marked as NO_PUSH, there could be 3 situations:
+   * 1. left child has a partition condition, right child does not.  In this case, we should
not push any child of this OR
+   * 2. left child does not have partition condition, right child has one.  Again, we should
not push any child of this OR
+   * 3. left and right child both have partition condition but both sides may have had other
non-partition conditions. In
+   *    this case, we can push the partition conditions by building a new OR combining both
children.
+   * In this method we clear the children of the OR for cases 1 and 2 and leave it alone
for case 3
+   */
+  private void clearOrChildrenIfSingle() {
+    if (!opStack.isEmpty()) {
+      BooleanOpState op = opStack.peek();
+      assert op.getOp().getKind() == SqlKind.OR;
+      if (op.getChildren().size() == 1) {
+        op.clear();
+      }
+    }
+  }
+
+  /**
+   * If the top of the parentCallTypeStack is an AND or OR, get the corresponding
+   * top item from the BooleanOpState stack and examine its children - these must
+   * be the directory filters we are interested in.  Create a new filter condition
+   * using the boolean operation and the children. Add this new filter as a child
+   * of the parent boolean operator - thus the filter condition gets built bottom-up.
+   */
+  private void popAndBuildFilter() {
+    SqlOperator op1 = null;
+    if (!parentCallTypeStack.isEmpty()) {
+      op1 = parentCallTypeStack.pop();
+    }
+    if (op1 != null
+        && (op1.getKind() == SqlKind.AND || op1.getKind() == SqlKind.OR)
+        && !opStack.isEmpty()) {
+      BooleanOpState op = opStack.pop();
+      int size = op.getChildren().size();
+      RexNode newFilter = null;
+      if (size > 1) {
+        newFilter = builder.makeCall(op.getOp(),  op.getChildren());
+      } else if (size == 1) {
+        newFilter = op.getChildren().get(0);
+      }
+      if (newFilter != null) {
+        // add this new filter to my parent boolean operator's children
+        if (!opStack.isEmpty()) {
+          op = opStack.peek();
+          op.addChild(newFilter);
+        } else {
+          resultCondition = newFilter;
+        }
+      }
+    }
+  }
+
+
+  public Void visitInputRef(RexInputRef inputRef) {
+    if(dirs.get(inputRef.getIndex())){
+      pushStatusStack.add(PushDirFilter.PUSH);
+    }else{
+      pushStatusStack.add(PushDirFilter.NO_PUSH);
+    }
+    return null;
+  }
+
+  public Void visitLiteral(RexLiteral literal) {
+    pushStatusStack.add(PushDirFilter.PUSH);
+    return null;
+  }
+
+  public Void visitOver(RexOver over) {
+    // assume NO_PUSH until proven otherwise
+    analyzeCall(over, PushDirFilter.NO_PUSH);
+    return null;
+  }
+
+  public Void visitCorrelVariable(RexCorrelVariable correlVariable) {
+    return pushVariable();
+  }
+
+  public Void visitCall(RexCall call) {
+    // assume PUSH until proven otherwise
+    analyzeCall(call, PushDirFilter.PUSH);
+    return null;
+  }
+
+  private void analyzeCall(RexCall call, PushDirFilter callPushDirFilter) {
+    parentCallTypeStack.push(call.getOperator());
+    if (call.getKind() == SqlKind.AND || call.getKind() == SqlKind.OR) {
+      opStack.push(new BooleanOpState(call.getOperator()));
+    }
+
+    // visit operands, pushing their states onto stack
+    super.visitCall(call);
+
+    // look for NO_PUSH operands
+    int operandCount = call.getOperands().size();
+    List<PushDirFilter> operandStack = Util.last(pushStatusStack, operandCount);
+    for (PushDirFilter operandPushDirFilter : operandStack) {
+      if (operandPushDirFilter == PushDirFilter.NO_PUSH) {
+        callPushDirFilter = PushDirFilter.NO_PUSH;
+      }
+    }
+
+    // Even if all operands are PUSH, the call itself may
+    // be non-deterministic.
+    if (!call.getOperator().isDeterministic()) {
+      callPushDirFilter = PushDirFilter.NO_PUSH;
+    } else if (call.getOperator().isDynamicFunction()) {
+      // For now, treat it same as non-deterministic.
+      callPushDirFilter = PushDirFilter.NO_PUSH;
+    }
+
+    // Row operator itself can't be reduced to a PUSH
+    if ((callPushDirFilter == PushDirFilter.PUSH)
+        && (call.getOperator() instanceof SqlRowOperator)) {
+      callPushDirFilter = PushDirFilter.NO_PUSH;
+    }
+
+
+    if (callPushDirFilter == PushDirFilter.NO_PUSH) {
+      if (call.getKind() == SqlKind.AND) {
+        // one or more children is not a push-able directory filter. If this is an AND, add
+        // all the ones that are push-able directory filters.
+        for (int iOperand = 0; iOperand < operandCount; ++iOperand) {
+          PushDirFilter pushDirFilter = operandStack.get(iOperand);
+          RexNode n = call.getOperands().get(iOperand);
+          if (pushDirFilter == PushDirFilter.PUSH && !(n.getKind() == SqlKind.AND
|| n.getKind() == SqlKind.OR)) {
+            addResult(n);
+          }
+        }
+      } else if (call.getKind() == SqlKind.OR) {
+        clearOrChildrenIfSingle();
+      }
+    }
+    else if (callPushDirFilter == PushDirFilter.PUSH && !(call.getKind() == SqlKind.AND
|| call.getKind() == SqlKind.OR)) {
+      addResult(call);
+    }
+
+    // pop operands off of the stack
+    operandStack.clear();
+
+    // pop this parent call operator off the stack and build the intermediate filters as
we go
+    popAndBuildFilter();
+
+    // push PushDirFilter result for this call onto stack
+    pushStatusStack.add(callPushDirFilter);
+  }
+
+  public Void visitDynamicParam(RexDynamicParam dynamicParam) {
+    return pushVariable();
+  }
+
+  public Void visitRangeRef(RexRangeRef rangeRef) {
+    return pushVariable();
+  }
+
+  public Void visitFieldAccess(RexFieldAccess fieldAccess) {
+    return pushVariable();
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
new file mode 100644
index 0000000..b8c9ebf
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
@@ -0,0 +1,296 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical.partition;
+
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import net.hydromatic.optiq.util.BitSets;
+
+import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
+import org.apache.drill.exec.expr.fn.interpreter.InterpreterEvaluator;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.physical.base.FileGroupScan;
+import org.apache.drill.exec.physical.base.GroupScan;
+import org.apache.drill.exec.planner.FileSystemPartitionDescriptor;
+import org.apache.drill.exec.planner.logical.DrillFilterRel;
+import org.apache.drill.exec.planner.logical.DrillOptiq;
+import org.apache.drill.exec.planner.logical.DrillParseContext;
+import org.apache.drill.exec.planner.logical.DrillProjectRel;
+import org.apache.drill.exec.planner.logical.DrillRel;
+import org.apache.drill.exec.planner.logical.DrillScanRel;
+import org.apache.drill.exec.planner.logical.RelOptHelper;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.store.dfs.FileSelection;
+import org.apache.drill.exec.store.dfs.FormatSelection;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelOptRuleOperand;
+import org.eigenbase.relopt.RelOptUtil;
+import org.eigenbase.rex.RexNode;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public abstract class PruneScanRule extends RelOptRule {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PruneScanRule.class);
+
+  public static final RelOptRule getFilterOnProject(QueryContext context){
+      return new PruneScanRule(
+          RelOptHelper.some(DrillFilterRel.class, RelOptHelper.some(DrillProjectRel.class,
RelOptHelper.any(DrillScanRel.class))),
+          "PruneScanRule:Filter_On_Project",
+          context) {
+
+      @Override
+        public boolean matches(RelOptRuleCall call) {
+          final DrillScanRel scan = (DrillScanRel) call.rel(2);
+          GroupScan groupScan = scan.getGroupScan();
+          // this rule is applicable only for dfs based partition pruning
+          return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown();
+        }
+
+      @Override
+      public void onMatch(RelOptRuleCall call) {
+        final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0);
+        final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1);
+        final DrillScanRel scanRel = (DrillScanRel) call.rel(2);
+        doOnMatch(call, filterRel, projectRel, scanRel);
+      };
+    };
+  }
+
+  public static final RelOptRule getFilterOnScan(QueryContext context){
+    return new PruneScanRule(
+          RelOptHelper.some(DrillFilterRel.class, RelOptHelper.any(DrillScanRel.class)),
+          "PruneScanRule:Filter_On_Scan", context) {
+
+      @Override
+        public boolean matches(RelOptRuleCall call) {
+          final DrillScanRel scan = (DrillScanRel) call.rel(1);
+          GroupScan groupScan = scan.getGroupScan();
+          // this rule is applicable only for dfs based partition pruning
+          return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown();
+        }
+
+      @Override
+      public void onMatch(RelOptRuleCall call) {
+        final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0);
+        final DrillScanRel scanRel = (DrillScanRel) call.rel(1);
+        doOnMatch(call, filterRel, null, scanRel);
+      }
+    };
+  }
+
+  final QueryContext context;
+
+  private PruneScanRule(RelOptRuleOperand operand, String id, QueryContext context) {
+    super(operand, id);
+    this.context = context;
+  }
+
+  protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, DrillProjectRel
projectRel, DrillScanRel scanRel) {
+    PlannerSettings settings = context.getPlannerSettings();
+    FileSystemPartitionDescriptor descriptor = new FileSystemPartitionDescriptor(settings.getFsPartitionColumnLabel());
+    final BufferAllocator allocator = context.getAllocator();
+
+
+    RexNode condition = null;
+    if(projectRel == null){
+      condition = filterRel.getCondition();
+    }else{
+      // get the filter as if it were below the projection.
+      condition = RelOptUtil.pushFilterPastProject(filterRel.getCondition(), projectRel);
+    }
+
+    Map<Integer, String> dirNames = Maps.newHashMap();
+    List<String> fieldNames = scanRel.getRowType().getFieldNames();
+    BitSet columnBitset = new BitSet();
+    BitSet dirBitset = new BitSet();
+    {
+      int colIndex = 0;
+      for(String field : fieldNames){
+        final Integer dirIndex = descriptor.getIdIfValid(field);
+        if(dirIndex != null){
+          dirNames.put(dirIndex, field);
+          dirBitset.set(dirIndex);
+          columnBitset.set(colIndex);
+        }
+        colIndex++;
+      }
+    }
+
+    if(dirBitset.isEmpty()){
+      return;
+    }
+
+    FindPartitionConditions c = new FindPartitionConditions(columnBitset, filterRel.getCluster().getRexBuilder());
+    c.analyze(condition);
+    RexNode pruneCondition = c.getFinalCondition();
+
+    if(pruneCondition == null){
+      return;
+    }
+
+    // set up the partitions
+    final FormatSelection origSelection = (FormatSelection)scanRel.getDrillTable().getSelection();
+    final List<String> files = origSelection.getAsFiles();
+    final String selectionRoot = origSelection.getSelection().selectionRoot;
+    List<PathPartition> partitions = Lists.newLinkedList();
+
+    // let's only deal with one batch of files for now.
+    if(files.size() > Character.MAX_VALUE){
+      return;
+    }
+
+    for(String f : files){
+      partitions.add(new PathPartition(descriptor.getMaxHierarchyLevel(), selectionRoot,
f));
+    }
+
+    final NullableBitVector output = new NullableBitVector(MaterializedField.create("", Types.optional(MinorType.BIT)),
allocator);
+    final VectorContainer container = new VectorContainer();
+
+    try{
+      final NullableVarCharVector[] vectors = new NullableVarCharVector[descriptor.getMaxHierarchyLevel()];
+      for(int dirIndex : BitSets.toIter(dirBitset)){
+        NullableVarCharVector vector = new NullableVarCharVector(MaterializedField.create(dirNames.get(dirIndex),
Types.optional(MinorType.VARCHAR)), allocator);
+        vector.allocateNew(5000, partitions.size());
+        vectors[dirIndex] = vector;
+        container.add(vector);
+      }
+
+      // populate partition vectors.
+      int record = 0;
+      for(Iterator<PathPartition> iter = partitions.iterator(); iter.hasNext(); record++){
+        final PathPartition partition = iter.next();
+        for(int dirIndex : BitSets.toIter(dirBitset)){
+          if(partition.dirs[dirIndex] == null){
+            vectors[dirIndex].getMutator().setNull(record);
+          }else{
+            byte[] bytes = partition.dirs[dirIndex].getBytes(Charsets.UTF_8);
+            vectors[dirIndex].getMutator().setSafe(record, bytes, 0, bytes.length);
+          }
+        }
+      }
+
+      for(NullableVarCharVector v : vectors){
+        if(v == null){
+          continue;
+        }
+        v.getMutator().setValueCount(partitions.size());
+      }
+
+
+      // materialize the expression
+      logger.debug("Attempting to prune {}", pruneCondition);
+      LogicalExpression expr = DrillOptiq.toDrill(new DrillParseContext(), scanRel, pruneCondition);
+      ErrorCollectorImpl errors = new ErrorCollectorImpl();
+      LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(expr, container,
errors, context.getFunctionRegistry());
+      if (errors.getErrorCount() != 0) {
+        logger.warn("Failure while materializing expression [{}].  Errors: {}", expr, errors);
+      }
+
+      output.allocateNew(partitions.size());
+      InterpreterEvaluator.evaluate(partitions.size(), context, container, output, materializedExpr);
+      record = 0;
+
+      List<String> newFiles = Lists.newArrayList();
+      for(Iterator<PathPartition> iter = partitions.iterator(); iter.hasNext(); record++){
+        PathPartition part = iter.next();
+        if(!output.getAccessor().isNull(record) && output.getAccessor().get(record)
== 1){
+          newFiles.add(part.file);
+        }
+      }
+
+      if(newFiles.isEmpty()){
+        newFiles.add(files.get(0));
+      }
+
+      if(newFiles.size() == files.size()){
+        return;
+      }
+
+      logger.debug("Pruned {} => {}", files, newFiles);
+
+
+      final FileSelection newFileSelection = new FileSelection(newFiles, origSelection.getSelection().selectionRoot,
true);
+      final FileGroupScan newScan = ((FileGroupScan)scanRel.getGroupScan()).clone(newFileSelection);
+      final DrillScanRel newScanRel =
+          new DrillScanRel(scanRel.getCluster(),
+              scanRel.getTraitSet().plus(DrillRel.DRILL_LOGICAL),
+              scanRel.getTable(),
+              newScan,
+              scanRel.getRowType(),
+              scanRel.getColumns());
+
+      RelNode inputRel = newScanRel;
+
+      if(projectRel != null){
+        inputRel = projectRel.copy(projectRel.getTraitSet(), Collections.singletonList(inputRel));
+      }
+
+      final RelNode newFilter = filterRel.copy(filterRel.getTraitSet(), Collections.singletonList(inputRel));
+      call.transformTo(newFilter);
+
+    }catch(Exception e){
+      logger.warn("Exception while trying to prune partition.", e);
+    }finally{
+      container.clear();
+      if(output !=null){
+        output.clear();
+      }
+    }
+  }
+
+  private static class PathPartition {
+    final String[] dirs;
+    final String file;
+
+    public PathPartition(int max, String selectionRoot, String file){
+      this.file = file;
+      int start = file.indexOf(selectionRoot) + selectionRoot.length();
+      String postPath = file.substring(start);
+      if(postPath.charAt(0) == '/'){
+        postPath = postPath.substring(1);
+      }
+      String[] mostDirs = postPath.split("/");
+      this.dirs = new String[max];
+      int maxLoop = Math.min(max, mostDirs.length - 1);
+      for(int i =0; i < maxLoop; i++){
+        this.dirs[i] = mostDirs[i];
+      }
+    }
+
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestPrune.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestPrune.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestPrune.java
new file mode 100644
index 0000000..d15555e
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestPrune.java
@@ -0,0 +1,47 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.expr;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.util.TestTools;
+import org.junit.Test;
+
+public class TestPrune extends BaseTestQuery {
+
+  String MULTILEVEL = TestTools.getWorkingPath() + "/../java-exec/src/test/resources/multilevel";
+
+  @Test
+  public void pruneCompound1() throws Exception {
+    test(String.format("select * from dfs.`%s/csv` where x is null and dir1 in ('Q1', 'Q2')",
MULTILEVEL));
+  }
+
+  @Test
+  public void pruneSimple1() throws Exception {
+    test(String.format("select * from dfs.`%s/csv` where dir1 in ('Q1', 'Q2')", MULTILEVEL));
+  }
+
+  @Test
+  public void pruneCompound2() throws Exception {
+    String query1 = String.format("select * from dfs_test.`%s/parquet` where (dir0=1995 and
o_totalprice < 40000) or (dir0=1996 and o_totalprice < 40000)", MULTILEVEL);
+    String query2 = String.format("select * from dfs_test.`%s/parquet` where dir0=1995 and
o_totalprice < 40000", MULTILEVEL);
+    String query3 = String.format("select * from dfs_test.`%s/parquet` where (dir0=1995 and
o_totalprice < 40000) or dir0=1996", MULTILEVEL);
+    String query4 = String.format("select * from dfs_test.`%s/parquet` where dir0=1995 or
dir0=1996", MULTILEVEL);
+    test(query3);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/48c9c01d/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/FilterSplitTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/FilterSplitTest.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/FilterSplitTest.java
new file mode 100644
index 0000000..7c85c19
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/FilterSplitTest.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import static org.junit.Assert.*;
+
+import java.util.BitSet;
+
+import net.hydromatic.optiq.impl.java.JavaTypeFactory;
+import net.hydromatic.optiq.jdbc.JavaTypeFactoryImpl;
+
+import org.apache.drill.exec.planner.logical.partition.FindPartitionConditions;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.rex.RexBuilder;
+import org.eigenbase.rex.RexNode;
+import org.eigenbase.sql.fun.SqlStdOperatorTable;
+import org.eigenbase.sql.type.SqlTypeName;
+import org.junit.Test;
+
+public class FilterSplitTest {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FilterSplitTest.class);
+
+  final JavaTypeFactory t = new JavaTypeFactoryImpl();
+  final RexBuilder builder = new RexBuilder(t);
+  final RelDataType intType = t.createSqlType(SqlTypeName.INTEGER);
+  final RelDataType sType = t.createSqlType(SqlTypeName.VARCHAR, 20);
+
+  @Test
+  public void simpleCompound() {
+    // a < 1 AND dir0 in (2,3)
+    RexNode n = and(
+          lt(c(0), lit(1)),
+          or(
+              eq(c(1), lit(2)),
+              eq(c(1), lit(3))
+              )
+        );
+
+    BitSet bs = new BitSet();
+    bs.set(1);
+    FindPartitionConditions c = new FindPartitionConditions(bs, builder);
+    c.analyze(n);
+
+    RexNode partNode = c.getFinalCondition();
+    assertEquals(n.toString(), "AND(<($0, 1), OR(=($1, 2), =($1, 3)))");
+    assertEquals(partNode.toString(), "OR(=($1, 2), =($1, 3))");
+  }
+
+  @Test
+  public void twoLevelDir() {
+    // (dir0 = 1 and dir1 = 2) OR (dir0 = 3 and dir1 = 4)
+    RexNode n = or(
+          and(
+              eq(c(1), lit(1)),
+              eq(c(2), lit(2))
+              ),
+          and(
+              eq(c(1), lit(3)),
+              eq(c(2), lit(4))
+              )
+
+        );
+
+    BitSet bs = new BitSet();
+    bs.set(1);
+    bs.set(2);
+    FindPartitionConditions c = new FindPartitionConditions(bs, builder);
+    c.analyze(n);
+
+    RexNode partNode = c.getFinalCondition();
+    assertEquals("OR(AND(=($1, 1), =($2, 2)), AND(=($1, 3), =($2, 4)))", n.toString());
+    assertEquals("OR(AND(=($1, 1), =($2, 2)), AND(=($1, 3), =($2, 4)))", partNode.toString());
+  }
+
+  @Test
+  public void badOr() {
+    // (dir0 = 1 and dir1 = 2) OR (a < 5)
+    RexNode n = or(
+          and(
+              eq(c(1), lit(1)),
+              eq(c(2), lit(2))
+              ),
+          lt(c(0), lit(5))
+
+        );
+
+    BitSet bs = new BitSet();
+    bs.set(1);
+    bs.set(2);
+    FindPartitionConditions c = new FindPartitionConditions(bs, builder);
+    c.analyze(n);
+
+    RexNode partNode = c.getFinalCondition();
+    assertEquals("OR(AND(=($1, 1), =($2, 2)), <($0, 5))", n.toString());
+    assertTrue(partNode == null);
+  }
+
+
+  @Test
+  public void badFunc() {
+    // (dir0 = 1 and dir1 = 2) OR (a < 5)
+    RexNode n = fn(
+        cs(0),
+        cs(1)
+        );
+
+    BitSet bs = new BitSet();
+    bs.set(1);
+    bs.set(2);
+    FindPartitionConditions c = new FindPartitionConditions(bs, builder);
+    c.analyze(n);
+
+    RexNode partNode = c.getFinalCondition();
+    assertEquals("||($0, $1)", n.toString());
+    assertTrue(partNode == null);
+  }
+
+
+  private RexNode and(RexNode...nodes){
+    return builder.makeCall(SqlStdOperatorTable.AND, nodes);
+  }
+
+  private RexNode fn(RexNode...nodes){
+    return builder.makeCall(SqlStdOperatorTable.CONCAT, nodes);
+  }
+
+  private RexNode or(RexNode...nodes){
+    return builder.makeCall(SqlStdOperatorTable.OR, nodes);
+  }
+
+  private RexNode lt(RexNode left, RexNode right){
+    return builder.makeCall(SqlStdOperatorTable.LESS_THAN, left, right);
+  }
+
+  private RexNode eq(RexNode left, RexNode right){
+    return builder.makeCall(SqlStdOperatorTable.EQUALS, left, right);
+  }
+
+  private RexNode lit(int value){
+    return builder.makeLiteral(value, intType, true);
+  }
+
+  private RexNode c(int index){
+    return builder.makeInputRef(intType, index);
+  }
+
+
+  private RexNode cs(int index){
+    return builder.makeInputRef(sType, index);
+  }
+
+  private RexNode str(String s){
+    return builder.makeLiteral(s);
+  }
+}


Mime
View raw message