tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject tajo git commit: TAJO-2013: FilterPushDownRule fails due to the unsupported index.
Date Thu, 10 Dec 2015 05:39:58 GMT
Repository: tajo
Updated Branches:
  refs/heads/branch-0.11.1 94d02005d -> 04d6d8e80


TAJO-2013: FilterPushDownRule fails due to the unsupported index.


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/04d6d8e8
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/04d6d8e8
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/04d6d8e8

Branch: refs/heads/branch-0.11.1
Commit: 04d6d8e8057d462223ab162c9c50b73506f40dae
Parents: 94d0200
Author: Jinho Kim <jhkim@apache.org>
Authored: Thu Dec 10 14:39:26 2015 +0900
Committer: Jinho Kim <jhkim@apache.org>
Committed: Thu Dec 10 14:39:26 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 +
 .../plan/rewrite/rules/FilterPushDownRule.java  | 95 ++++++++++----------
 2 files changed, 52 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/04d6d8e8/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index f7aba71..d55e4b8 100644
--- a/CHANGES
+++ b/CHANGES
@@ -24,6 +24,8 @@ Release 0.11.1 - unreleased
 
   BUG FIXES
 
+    TAJO-2013: FilterPushDownRule fails due to the unsupported index. (jinho)
+
     TAJO-2014: TestRpcClientManager fails occasionally. (jinho)
 
     TAJO-2000: BSTIndex can cause OOM. (jinho)

http://git-wip-us.apache.org/repos/asf/tajo/blob/04d6d8e8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
index e44bba5..1ac29ef 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.SessionVars;
 import org.apache.tajo.algebra.JoinType;
 import org.apache.tajo.catalog.*;
 import org.apache.tajo.datum.Datum;
@@ -57,7 +58,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
   private CatalogService catalog;
 
   static class FilterPushDownContext {
-    Set<EvalNode> pushingDownFilters = TUtil.newHashSet();
+    Set<EvalNode> pushingDownFilters = new HashSet<>();
+    LogicalPlanRewriteRuleContext rewriteRuleContext;
 
     public void clear() {
       pushingDownFilters.clear();
@@ -111,6 +113,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
         . It not, create new HavingNode and set parent's child.
      */
     FilterPushDownContext context = new FilterPushDownContext();
+    context.rewriteRuleContext = rewriteRuleContext;
     LogicalPlan plan = rewriteRuleContext.getPlan();
     catalog = rewriteRuleContext.getCatalog();
     for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) {
@@ -129,7 +132,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
   @Override
   public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock
block,
                                  SelectionNode selNode, Stack<LogicalNode> stack) throws
TajoException {
-    context.pushingDownFilters.addAll(TUtil.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(selNode.getQual())));
+    context.pushingDownFilters.addAll(new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(selNode.getQual()))));
 
     stack.push(selNode);
     visit(context, plan, block, selNode.getChild(), stack);
@@ -147,7 +150,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
     } else { // if there remain search conditions
 
       // check if it can be evaluated here
-      Set<EvalNode> matched = TUtil.newHashSet();
+      Set<EvalNode> matched = new HashSet<>();
       for (EvalNode eachEval : context.pushingDownFilters) {
         if (LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)) {
           matched.add(eachEval);
@@ -168,9 +171,9 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
   @Override
   public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock
block,
                                JoinNode joinNode, Stack<LogicalNode> stack) throws
TajoException {
-    Set<EvalNode> onPredicates = TUtil.newHashSet();
+    Set<EvalNode> onPredicates = new HashSet<>();
     if (joinNode.hasJoinQual()) {
-      onPredicates.addAll(TUtil.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual())));
+      onPredicates.addAll(new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))));
     }
     // clear join qual
     joinNode.clearJoinQual();
@@ -293,7 +296,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
                                                            final Set<EvalNode> onPredicates,
                                                            final Set<EvalNode> wherePredicates)
       throws TajoException {
-    Set<EvalNode> nonPushableQuals = TUtil.newHashSet();
+    Set<EvalNode> nonPushableQuals = new HashSet<>();
     // TODO: non-equi theta join quals must not be pushed until TAJO-742 is resolved.
     nonPushableQuals.addAll(extractNonEquiThetaJoinQuals(wherePredicates, block, joinNode));
     nonPushableQuals.addAll(extractNonEquiThetaJoinQuals(onPredicates, block, joinNode));
@@ -340,8 +343,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
                                                                 final Set<EvalNode>
onPredicates,
                                                                 final Set<EvalNode>
wherePredicates,
                                                                 final JoinNode joinNode)
throws TajoException {
-    Set<String> nullSupplyingTableNameSet = TUtil.newHashSet();
-    Set<String> preservedTableNameSet = TUtil.newHashSet();
+    Set<String> nullSupplyingTableNameSet = new HashSet<>();
+    Set<String> preservedTableNameSet = new HashSet<>();
     String leftRelation = PlannerUtil.getTopRelationInLineage(plan, joinNode.getLeftChild());
     String rightRelation = PlannerUtil.getTopRelationInLineage(plan, joinNode.getRightChild());
 
@@ -359,7 +362,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
       nullSupplyingTableNameSet.add(rightRelation);
     }
 
-    Set<EvalNode> nonPushableQuals = TUtil.newHashSet();
+    Set<EvalNode> nonPushableQuals = new HashSet<>();
     for (EvalNode eachQual : onPredicates) {
       for (String relName : preservedTableNameSet) {
         if (isEvalNeedRelation(eachQual, relName)) {
@@ -425,7 +428,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
       LogicalPlan.QueryBlock block,
       LogicalNode node, LogicalNode childNode) throws TajoException {
     // transformed -> pushingDownFilters
-    Map<EvalNode, EvalNode> transformedMap = TUtil.newHashMap();
+    Map<EvalNode, EvalNode> transformedMap = new HashMap<>();
 
     if (originEvals.isEmpty()) {
       return transformedMap;
@@ -483,7 +486,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
     }
 
     // node in column -> child out column
-    Map<String, String> columnMap = TUtil.newHashMap();
+    Map<String, String> columnMap = new HashMap<>();
 
     for (int i = 0; i < node.getInSchema().size(); i++) {
       String inColumnName = node.getInSchema().getColumn(i).getQualifiedName();
@@ -643,7 +646,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
   }
 
   private Collection<EvalNode> reverseTransform(BiMap<EvalNode, EvalNode> map,
Set<EvalNode> remainFilters) {
-    Set<EvalNode> reversed = TUtil.newHashSet();
+    Set<EvalNode> reversed = new HashSet<>();
     for (EvalNode evalNode : remainFilters) {
       reversed.add(map.get(evalNode));
     }
@@ -655,7 +658,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
       LogicalNode childNode, List<EvalNode> notMatched,
       Set<String> partitionColumns, int columnOffset) throws TajoException {
     // canonical name -> target
-    Map<String, Target> nodeTargetMap = TUtil.newHashMap();
+    Map<String, Target> nodeTargetMap = new HashMap<>();
     for (Target target : node.getTargets()) {
       nodeTargetMap.put(target.getCanonicalName(), target);
     }
@@ -780,8 +783,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
                                        HavingNode havingNode,
                                        GroupbyNode groupByNode) throws TajoException {
     // find aggregation column
-    Set<Column> groupingColumns = TUtil.newHashSet(groupByNode.getGroupingColumns());
-    Set<String> aggrFunctionOutColumns = TUtil.newHashSet();
+    Set<Column> groupingColumns = new HashSet<>(Arrays.asList(groupByNode.getGroupingColumns()));
+    Set<String> aggrFunctionOutColumns = new HashSet<>();
     for (Column column : groupByNode.getOutSchema().getRootColumns()) {
       if (!groupingColumns.contains(column)) {
         aggrFunctionOutColumns.add(column.getQualifiedName());
@@ -896,7 +899,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
     List<EvalNode> matched = TUtil.newList();
 
     // find partition column and check matching
-    Set<String> partitionColumns = TUtil.newHashSet();
+    Set<String> partitionColumns = new HashSet<>();
     TableDesc table = scanNode.getTableDesc();
     boolean hasQualifiedName = false;
     if (table.hasPartition()) {
@@ -905,7 +908,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
         hasQualifiedName = c.hasQualifier();
       }
     }
-    Set<EvalNode> partitionEvals = TUtil.newHashSet();
+    Set<EvalNode> partitionEvals = new HashSet<>();
     for (EvalNode eval : context.pushingDownFilters) {
       if (table.hasPartition()) {
         Set<Column> columns = EvalTreeUtil.findUniqueColumns(eval);
@@ -966,35 +969,37 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
       scanNode.setQual(qual);
 
       // Index path can be identified only after filters are pushed into each scan.
-      String databaseName, tableName;
-      databaseName = CatalogUtil.extractQualifier(table.getName());
-      tableName = CatalogUtil.extractSimpleName(table.getName());
-      Set<Predicate> predicates = TUtil.newHashSet();
-      for (EvalNode eval : PlannerUtil.getAllEqualEvals(qual)) {
-        BinaryEval binaryEval = (BinaryEval) eval;
-        // TODO: consider more complex predicates
-        if (binaryEval.getLeftExpr().getType() == EvalType.FIELD &&
-            binaryEval.getRightExpr().getType() == EvalType.CONST) {
-          predicates.add(new Predicate(binaryEval.getType(),
-              ((FieldEval) binaryEval.getLeftExpr()).getColumnRef(),
-              ((ConstEval)binaryEval.getRightExpr()).getValue()));
-        } else if (binaryEval.getLeftExpr().getType() == EvalType.CONST &&
-            binaryEval.getRightExpr().getType() == EvalType.FIELD) {
-          predicates.add(new Predicate(binaryEval.getType(),
-              ((FieldEval) binaryEval.getRightExpr()).getColumnRef(),
-              ((ConstEval)binaryEval.getLeftExpr()).getValue()));
+      if(context.rewriteRuleContext.getQueryContext().getBool(SessionVars.INDEX_ENABLED))
{
+        String databaseName, tableName;
+        databaseName = CatalogUtil.extractQualifier(table.getName());
+        tableName = CatalogUtil.extractSimpleName(table.getName());
+        Set<Predicate> predicates = new HashSet<>();
+        for (EvalNode eval : PlannerUtil.getAllEqualEvals(qual)) {
+          BinaryEval binaryEval = (BinaryEval) eval;
+          // TODO: consider more complex predicates
+          if (binaryEval.getLeftExpr().getType() == EvalType.FIELD &&
+              binaryEval.getRightExpr().getType() == EvalType.CONST) {
+            predicates.add(new Predicate(binaryEval.getType(),
+                ((FieldEval) binaryEval.getLeftExpr()).getColumnRef(),
+                ((ConstEval) binaryEval.getRightExpr()).getValue()));
+          } else if (binaryEval.getLeftExpr().getType() == EvalType.CONST &&
+              binaryEval.getRightExpr().getType() == EvalType.FIELD) {
+            predicates.add(new Predicate(binaryEval.getType(),
+                ((FieldEval) binaryEval.getRightExpr()).getColumnRef(),
+                ((ConstEval) binaryEval.getLeftExpr()).getValue()));
+          }
         }
-      }
 
-      // for every subset of the set of columns, find all matched index paths
-      for (Set<Predicate> subset : Sets.powerSet(predicates)) {
-        if (subset.size() == 0)
-          continue;
-        Column[] columns = extractColumns(subset);
-        if (catalog.existIndexByColumns(databaseName, tableName, columns)) {
-          IndexDesc indexDesc = catalog.getIndexByColumns(databaseName, tableName, columns);
-          block.addAccessPath(scanNode, new IndexScanInfo(
-              table.getStats(), indexDesc, getSimplePredicates(indexDesc, subset)));
+        // for every subset of the set of columns, find all matched index paths
+        for (Set<Predicate> subset : Sets.powerSet(predicates)) {
+          if (subset.size() == 0)
+            continue;
+          Column[] columns = extractColumns(subset);
+          if (catalog.existIndexByColumns(databaseName, tableName, columns)) {
+            IndexDesc indexDesc = catalog.getIndexByColumns(databaseName, tableName, columns);
+            block.addAccessPath(scanNode, new IndexScanInfo(
+                table.getStats(), indexDesc, getSimplePredicates(indexDesc, subset)));
+          }
         }
       }
     }
@@ -1023,7 +1028,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
 
   private static SimplePredicate[] getSimplePredicates(IndexDesc desc, Set<Predicate>
predicates) {
     SimplePredicate[] simplePredicates = new SimplePredicate[predicates.size()];
-    Map<Column, Datum> colToValue = TUtil.newHashMap();
+    Map<Column, Datum> colToValue = new HashMap<>();
     for (Predicate predicate : predicates) {
       colToValue.put(predicate.column, predicate.value);
     }


Mime
View raw message