hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zs...@apache.org
Subject svn commit: r781904 - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/ppd/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Date Fri, 05 Jun 2009 03:38:13 GMT
Author: zshao
Date: Fri Jun  5 03:38:13 2009
New Revision: 781904

URL: http://svn.apache.org/viewvc?rev=781904&view=rev
Log:
HIVE-544. Fix predicate push for exprFieldNodeDesc. (Prasad Chakka via zshao)

Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Jun  5 03:38:13 2009
@@ -223,6 +223,9 @@
     HIVE-540. Restore test.silent setting
     (Zheng Shao via namit)
 
+    HIVE-544. Fix predicate push for exprFieldNodeDesc.
+    (Prasad Chakka via zshao)
+
 Release 0.3.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java Fri Jun
 5 03:38:13 2009
@@ -53,7 +53,7 @@
     else
       setDone(true);
   }
-
+  
   public String getName() {
     return "LIM";
   }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
(original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
Fri Jun  5 03:38:13 2009
@@ -141,6 +141,43 @@
 
   }
 
+  public static class FieldExprProcessor implements NodeProcessor {
+
+    @Override
+    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+        Object... nodeOutputs) throws SemanticException {
+      ExprWalkerInfo ctx = (ExprWalkerInfo) procCtx;
+      String alias = null;
+      exprNodeFieldDesc expr = (exprNodeFieldDesc) nd;
+
+      boolean isCandidate = true;
+      assert(nd.getChildren().size() == 1);
+      exprNodeDesc ch = (exprNodeDesc) nd.getChildren().get(0);
+      exprNodeDesc newCh = ctx.getConvertedNode(ch);
+      if (newCh != null) {
+        expr.setDesc(newCh);
+        ch = newCh;
+      }
+      String chAlias = ctx.getAlias(ch);
+
+      isCandidate = isCandidate && ctx.isCandidate(ch);
+      // need to iterate through all children even if one is found to be not a candidate
+      // in case if the other children could be individually pushed up
+      if (isCandidate && chAlias != null) {
+        if (alias == null) {
+          alias = chAlias;
+        } else if (!chAlias.equalsIgnoreCase(alias)) {
+          isCandidate = false;
+        }
+      }
+
+      ctx.addAlias(expr, alias);
+      ctx.setIsCandidate(expr, isCandidate);
+      return isCandidate;
+    }
+
+  }
+
   /**
    * If all children are candidates and refer only to one table alias then this expr is a
candidate
    * else it is not a candidate but its children could be final candidates
@@ -271,6 +308,10 @@
     return new ColumnExprProcessor();
   }
 
+  private static NodeProcessor getFieldProcessor() {
+    return new FieldExprProcessor();
+  }
+
   public static ExprWalkerInfo extractPushdownPreds(OpWalkerInfo opContext, 
       Operator<? extends Serializable> op,
       exprNodeDesc pred) throws SemanticException {
@@ -297,7 +338,7 @@
     // generates the plan from the operator tree
     Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>();
     exprRules.put(new RuleRegExp("R1", exprNodeColumnDesc.class.getName() + "%"), getColumnProcessor());
-    exprRules.put(new RuleRegExp("R2", exprNodeFieldDesc.class.getName() + "%"), getFuncProcessor());
+    exprRules.put(new RuleRegExp("R2", exprNodeFieldDesc.class.getName() + "%"), getFieldProcessor());
     exprRules.put(new RuleRegExp("R3", exprNodeFuncDesc.class.getName() + "%"), getFuncProcessor());
     exprRules.put(new RuleRegExp("R4", exprNodeIndexDesc.class.getName() + "%"), getIndexProcessor());
     exprRules.put(new RuleRegExp("R5", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor());

Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q Fri Jun  5 03:38:13
2009
@@ -1,3 +1,20 @@
+set hive.optimize.ppd=false;
+
+EXPLAIN
+FROM src_thrift
+SELECT src_thrift.mstringstring['key_9'], lintstring.myint
+WHERE src_thrift.mstringstring['key_9'] IS NOT NULL
+      AND lintstring.myint IS NOT NULL
+      AND lintstring IS NOT NULL;
+
+FROM src_thrift
+SELECT src_thrift.mstringstring['key_9'], lintstring.myint
+WHERE src_thrift.mstringstring['key_9'] IS NOT NULL
+      OR lintstring.myint IS NOT NULL
+      OR lintstring IS NOT NULL;
+
+set hive.optimize.ppd=true;
+
 EXPLAIN
 FROM src_thrift
 SELECT src_thrift.mstringstring['key_9'], lintstring.myint

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out Fri Jun  5
03:38:13 2009
@@ -50,7 +50,74 @@
       OR lintstring.myint IS NOT NULL
       OR lintstring IS NOT NULL
 Input: default/src_thrift
-Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/115422682/57144029.10000
+Output: file:/data/users/pchakka/workspace/oshive/build/ql/tmp/249993417/10000
+NULL	[0]
+NULL	[1]
+NULL	[4]
+NULL	[9]
+NULL	[16]
+NULL	[25]
+NULL	[36]
+NULL	[49]
+NULL	[64]
+value_9	[81]
+query: EXPLAIN
+FROM src_thrift
+SELECT src_thrift.mstringstring['key_9'], lintstring.myint
+WHERE src_thrift.mstringstring['key_9'] IS NOT NULL
+      AND lintstring.myint IS NOT NULL
+      AND lintstring IS NOT NULL
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
(TOK_SELECT (TOK_SELEXPR ([ (. (TOK_TABLE_OR_COL src_thrift) mstringstring) 'key_9')) (TOK_SELEXPR
(. (TOK_TABLE_OR_COL lintstring) myint))) (TOK_WHERE (AND (AND (TOK_FUNCTION TOK_ISNOTNULL
([ (. (TOK_TABLE_OR_COL src_thrift) mstringstring) 'key_9')) (TOK_FUNCTION TOK_ISNOTNULL (.
(TOK_TABLE_OR_COL lintstring) myint))) (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL lintstring))))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        src_thrift 
+            Filter Operator
+              predicate:
+                  expr: ((mstringstring['key_9'] is not null and lintstring.myint is not
null) and lintstring is not null)
+                  type: boolean
+              Select Operator
+                expressions:
+                      expr: mstringstring
+                      type: map<string,string>
+                      expr: lintstring
+                      type: array<struct<myint:int,mystring:string,underscore_int:int>>
+                Filter Operator
+                  predicate:
+                      expr: ((0['key_9'] is not null and 1.myint is not null) and 1 is not
null)
+                      type: boolean
+                  Select Operator
+                    expressions:
+                          expr: 0['key_9']
+                          type: string
+                          expr: 1.myint
+                          type: array<int>
+                    File Output Operator
+                      compressed: false
+                      GlobalTableId: 0
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+query: FROM src_thrift
+SELECT src_thrift.mstringstring['key_9'], lintstring.myint
+WHERE src_thrift.mstringstring['key_9'] IS NOT NULL
+      OR lintstring.myint IS NOT NULL
+      OR lintstring IS NOT NULL
+Input: default/src_thrift
+Output: file:/data/users/pchakka/workspace/oshive/build/ql/tmp/1189169528/10000
 NULL	[0]
 NULL	[1]
 NULL	[4]



Mime
View raw message