hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1588401 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec: HashTableSinkOperator.java MapJoinOperator.java Operator.java mr/MapredLocalTask.java
Date Fri, 18 Apr 2014 02:19:41 GMT
Author: xuefu
Date: Fri Apr 18 02:19:41 2014
New Revision: 1588401

URL: http://svn.apache.org/r1588401
Log:
HIVE-6913: Hive unable to find the hashtable file during complex multi-staged map join (Brock
via Xuefu)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Fri Apr
18 02:19:41 2014
@@ -263,7 +263,9 @@ public class HashTableSinkOperator exten
   @Override
   public void closeOp(boolean abort) throws HiveException {
     try {
-      if (mapJoinTables != null) {
+      if (mapJoinTables == null) {
+        LOG.debug("mapJoinTables is null");
+      } else {
         flushToFile();
       }
       super.closeOp(abort);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Fri Apr 18
02:19:41 2014
@@ -248,8 +248,10 @@ public class MapJoinOperator extends Abs
           storage[pos] = null;
         }
       }
-    } catch (SerDeException e) {
-      throw new HiveException(e);
+    } catch (Exception e) {
+      String msg = "Unxpected exception: " + e.getMessage();
+      LOG.error(msg, e);
+      throw new HiveException(msg, e);
     }
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Fri Apr 18 02:19:41
2014
@@ -546,6 +546,7 @@ public abstract class Operator<T extends
         if(parent==null){
           continue;
         }
+        LOG.debug("allInitializedParentsAreClosed? parent.state = " + parent.state);
         if (!(parent.state == State.CLOSE || parent.state == State.UNINIT)) {
           return false;
         }
@@ -565,6 +566,7 @@ public abstract class Operator<T extends
 
     // check if all parents are finished
     if (!allInitializedParentsAreClosed()) {
+      LOG.debug("Not all parent operators are closed. Not closing.");
       return;
     }
 
@@ -585,6 +587,7 @@ public abstract class Operator<T extends
       }
 
       for (Operator<? extends OperatorDesc> op : childOperators) {
+        LOG.debug("Closing child = " + op);
         op.close(abort);
       }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java Fri Apr
18 02:19:41 2014
@@ -349,14 +349,15 @@ public class MapredLocalTask extends Tas
         setUpFetchOpContext(fetchOp, alias, bigTableBucket);
       }
 
+      // get the root operator
+      Operator<? extends OperatorDesc> forwardOp = work.getAliasToWork().get(alias);
       if (fetchOp.isEmptyTable()) {
         //generate empty hashtable for empty table
         this.generateDummyHashTable(alias, bigTableBucket);
+        forwardOp.close(false);
         continue;
       }
 
-      // get the root operator
-      Operator<? extends OperatorDesc> forwardOp = work.getAliasToWork().get(alias);
       // walk through the operator tree
       while (!forwardOp.getDone()) {
         InspectableObject row = fetchOp.getNextRow();
@@ -375,6 +376,9 @@ public class MapredLocalTask extends Tas
 
   private void initializeOperators(Map<FetchOperator, JobConf> fetchOpJobConfMap)
       throws HiveException {
+    for (Map.Entry<String, Operator<? extends OperatorDesc>> entry : work.getAliasToWork().entrySet())
{
+      LOG.debug("initializeOperators: " +  entry.getKey() + ", children = "  + entry.getValue().getChildOperators());
+    }
     // this mapper operator is used to initialize all the operators
     for (Map.Entry<String, FetchWork> entry : work.getAliasToFetchWork().entrySet())
{
       if (entry.getValue() == null) {
@@ -419,6 +423,7 @@ public class MapredLocalTask extends Tas
 
   private void generateDummyHashTable(String alias, String bigBucketFileName)
       throws HiveException,IOException {
+    LOG.debug("generating dummy for " + alias);
     // find the (byte)tag for the map join(HashTableSinkOperator)
     Operator<? extends OperatorDesc> parentOp = work.getAliasToWork().get(alias);
     Operator<? extends OperatorDesc> childOp = parentOp.getChildOperators().get(0);



Mime
View raw message