hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1471700 - in /hive/branches/branch-0.11/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/ppd/ test/queries/clientpositive/ test/results/clientpositive/ test/results/compiler/...
Date Wed, 24 Apr 2013 21:36:36 GMT
Author: hashutosh
Date: Wed Apr 24 21:36:35 2013
New Revision: 1471700

URL: http://svn.apache.org/r1471700
Log:
HIVE-4365 : wrong result in left semi join (Navis via Ashutosh Chauhan)

Modified:
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
    hive/branches/branch-0.11/ql/src/test/queries/clientpositive/semijoin.q
    hive/branches/branch-0.11/ql/src/test/results/clientpositive/semijoin.q.out
    hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join1.q.xml
    hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join2.q.xml
    hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join3.q.xml
    hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join4.q.xml
    hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join5.q.xml
    hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join6.q.xml
    hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join7.q.xml
    hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join8.q.xml

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
(original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
Wed Apr 24 21:36:35 2013
@@ -80,6 +80,15 @@ public class ReduceSinkOperator extends 
   transient byte[] tagByte = new byte[1];
   transient protected int numDistributionKeys;
   transient protected int numDistinctExprs;
+  transient String inputAlias;  // input alias of this RS for join (used for PPD)
+
+  public void setInputAlias(String inputAlias) {
+    this.inputAlias = inputAlias;
+  }
+
+  public String getInputAlias() {
+    return inputAlias;
+  }
 
   @Override
   protected void initializeOp(Configuration hconf) throws HiveException {

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Wed Apr 24 21:36:35 2013
@@ -5976,6 +5976,7 @@ public class SemanticAnalyzer extends Ba
             reduceKeys.size(), numReds), new RowSchema(outputRS
             .getColumnInfos()), child), outputRS);
     rsOp.setColumnExprMap(colExprMap);
+    rsOp.setInputAlias(srcName);
     return rsOp;
   }
 

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
(original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
Wed Apr 24 21:36:35 2013
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.ql.ppd;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -35,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.Fi
 import org.apache.hadoop.hive.ql.exec.JoinOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorFactory;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
 import org.apache.hadoop.hive.ql.exec.RowSchema;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -501,12 +503,19 @@ public final class OpProcFactory {
         Object... nodeOutputs) throws SemanticException {
       LOG.info("Processing for " + nd.getName() + "("
           + ((Operator) nd).getIdentifier() + ")");
+      ReduceSinkOperator rs = (ReduceSinkOperator) nd;
       OpWalkerInfo owi = (OpWalkerInfo) procCtx;
-      Set<String> aliases = owi.getRowResolver(nd).getTableNames();
+
+      Set<String> aliases;
       boolean ignoreAliases = false;
-      if (aliases.size() == 1 && aliases.contains("")) {
-        // Reduce sink of group by operator
-        ignoreAliases = true;
+      if (rs.getInputAlias() != null) {
+        aliases = new HashSet<String>(Arrays.asList(rs.getInputAlias()));
+      } else {
+        aliases = owi.getRowResolver(nd).getTableNames();
+        if (aliases.size() == 1 && aliases.contains("")) {
+          // Reduce sink of group by operator
+          ignoreAliases = true;
+        }
       }
       boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, aliases, ignoreAliases);
       if (HiveConf.getBoolVar(owi.getParseContext().getConf(),

Modified: hive/branches/branch-0.11/ql/src/test/queries/clientpositive/semijoin.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/queries/clientpositive/semijoin.q?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/queries/clientpositive/semijoin.q (original)
+++ hive/branches/branch-0.11/ql/src/test/queries/clientpositive/semijoin.q Wed Apr 24 21:36:35
2013
@@ -77,7 +77,5 @@ select a.key from t3 a left semi join t1
 explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c
on a.value = c.value sort by a.key;
 select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.value
= c.value sort by a.key;
 
-
-
-
-
+explain select a.key from t3 a left semi join t2 b on a.value = b.value where a.key >
100;
+select a.key from t3 a left semi join t2 b on a.value = b.value where a.key > 100;

Modified: hive/branches/branch-0.11/ql/src/test/results/clientpositive/semijoin.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/clientpositive/semijoin.q.out?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/clientpositive/semijoin.q.out (original)
+++ hive/branches/branch-0.11/ql/src/test/results/clientpositive/semijoin.q.out Wed Apr 24
21:36:35 2013
@@ -2980,3 +2980,97 @@ POSTHOOK: Input: default@t3
 16
 18
 20
+PREHOOK: query: explain select a.key from t3 a left semi join t2 b on a.value = b.value where
a.key > 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select a.key from t3 a left semi join t2 b on a.value = b.value
where a.key > 100
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_LEFTSEMIJOIN (TOK_TABREF (TOK_TABNAME t3) a) (TOK_TABREF (TOK_TABNAME
t2) b) (= (. (TOK_TABLE_OR_COL a) value) (. (TOK_TABLE_OR_COL b) value)))) (TOK_INSERT (TOK_DESTINATION
(TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key))) (TOK_WHERE
(> (. (TOK_TABLE_OR_COL a) key) 100))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        a 
+          TableScan
+            alias: a
+            Filter Operator
+              predicate:
+                  expr: (key > 100)
+                  type: boolean
+              Reduce Output Operator
+                key expressions:
+                      expr: value
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: value
+                      type: string
+                tag: 0
+                value expressions:
+                      expr: key
+                      type: int
+        b 
+          TableScan
+            alias: b
+            Select Operator
+              expressions:
+                    expr: value
+                    type: string
+              outputColumnNames: value
+              Group By Operator
+                bucketGroup: false
+                keys:
+                      expr: value
+                      type: string
+                mode: hash
+                outputColumnNames: _col0
+                Reduce Output Operator
+                  key expressions:
+                        expr: _col0
+                        type: string
+                  sort order: +
+                  Map-reduce partition columns:
+                        expr: _col0
+                        type: string
+                  tag: 1
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Left Semi Join 0 to 1
+          condition expressions:
+            0 {VALUE._col0}
+            1 
+          handleSkewJoin: false
+          outputColumnNames: _col0
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: int
+            outputColumnNames: _col0
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+PREHOOK: query: select a.key from t3 a left semi join t2 b on a.value = b.value where a.key
> 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: select a.key from t3 a left semi join t2 b on a.value = b.value where a.key
> 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####

Modified: hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join1.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join1.q.xml?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join1.q.xml (original)
+++ hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join1.q.xml Wed Apr 24 21:36:35
2013
@@ -619,6 +619,9 @@
               </void> 
              </object> 
             </void> 
+            <void property="inputAlias"> 
+             <string>src2</string> 
+            </void> 
             <void property="operatorId"> 
              <string>RS_3</string> 
             </void> 
@@ -918,6 +921,9 @@
               </void> 
              </object> 
             </void> 
+            <void property="inputAlias"> 
+             <string>src1</string> 
+            </void> 
             <void property="operatorId"> 
              <string>RS_2</string> 
             </void> 

Modified: hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join2.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join2.q.xml?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join2.q.xml (original)
+++ hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join2.q.xml Wed Apr 24 21:36:35
2013
@@ -793,6 +793,9 @@
                   </void> 
                  </object> 
                 </void> 
+                <void property="inputAlias"> 
+                 <string>src3</string> 
+                </void> 
                 <void property="operatorId"> 
                  <string>RS_7</string> 
                 </void> 
@@ -2160,6 +2163,9 @@
               </void> 
              </object> 
             </void> 
+            <void property="inputAlias"> 
+             <string>src2</string> 
+            </void> 
             <void property="operatorId"> 
              <string>RS_4</string> 
             </void> 
@@ -2452,6 +2458,9 @@
               </void> 
              </object> 
             </void> 
+            <void property="inputAlias"> 
+             <string>src1</string> 
+            </void> 
             <void property="operatorId"> 
              <string>RS_3</string> 
             </void> 

Modified: hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join3.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join3.q.xml?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join3.q.xml (original)
+++ hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join3.q.xml Wed Apr 24 21:36:35
2013
@@ -744,6 +744,9 @@
               </void> 
              </object> 
             </void> 
+            <void property="inputAlias"> 
+             <string>src2</string> 
+            </void> 
             <void property="operatorId"> 
              <string>RS_4</string> 
             </void> 
@@ -1029,6 +1032,9 @@
               </void> 
              </object> 
             </void> 
+            <void property="inputAlias"> 
+             <string>src3</string> 
+            </void> 
             <void property="operatorId"> 
              <string>RS_5</string> 
             </void> 
@@ -1324,6 +1330,9 @@
               </void> 
              </object> 
             </void> 
+            <void property="inputAlias"> 
+             <string>src1</string> 
+            </void> 
             <void property="operatorId"> 
              <string>RS_3</string> 
             </void> 

Modified: hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join4.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join4.q.xml?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join4.q.xml (original)
+++ hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join4.q.xml Wed Apr 24 21:36:35
2013
@@ -482,6 +482,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>a</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_6</string> 
                     </void> 
@@ -1074,6 +1077,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>b</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_7</string> 
                     </void> 

Modified: hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join5.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join5.q.xml?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join5.q.xml (original)
+++ hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join5.q.xml Wed Apr 24 21:36:35
2013
@@ -482,6 +482,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>a</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_6</string> 
                     </void> 
@@ -1074,6 +1077,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>b</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_7</string> 
                     </void> 

Modified: hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join6.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join6.q.xml?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join6.q.xml (original)
+++ hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join6.q.xml Wed Apr 24 21:36:35
2013
@@ -482,6 +482,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>a</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_6</string> 
                     </void> 
@@ -1074,6 +1077,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>b</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_7</string> 
                     </void> 

Modified: hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join7.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join7.q.xml?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join7.q.xml (original)
+++ hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join7.q.xml Wed Apr 24 21:36:35
2013
@@ -630,6 +630,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>a</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_9</string> 
                     </void> 
@@ -1222,6 +1225,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>b</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_10</string> 
                     </void> 
@@ -1802,6 +1808,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>c</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_11</string> 
                     </void> 

Modified: hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join8.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join8.q.xml?rev=1471700&r1=1471699&r2=1471700&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join8.q.xml (original)
+++ hive/branches/branch-0.11/ql/src/test/results/compiler/plan/join8.q.xml Wed Apr 24 21:36:35
2013
@@ -482,6 +482,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>a</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_6</string> 
                     </void> 
@@ -1115,6 +1118,9 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="inputAlias"> 
+                     <string>b</string> 
+                    </void> 
                     <void property="operatorId"> 
                      <string>RS_7</string> 
                     </void> 



Mime
View raw message