hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1545434 - in /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec: Utilities.java mr/ExecMapper.java mr/ExecReducer.java tez/MapRecordProcessor.java tez/ObjectCache.java tez/ReduceRecordProcessor.java
Date Mon, 25 Nov 2013 22:40:22 GMT
Author: gunther
Date: Mon Nov 25 22:40:22 2013
New Revision: 1545434

URL: http://svn.apache.org/r1545434
Log:
HIVE-5883: Plan is deserialized more often than necessary on Tez (in container reuse case)
(Gunther Hagleitner)

Modified:
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Mon Nov 25
22:40:22 2013
@@ -265,15 +265,31 @@ public final class Utilities {
     return w;
   }
 
+  public static void setMapWork(Configuration conf, MapWork work) {
+    setBaseWork(conf, MAP_PLAN_NAME, work);
+  }
+
   public static MapWork getMapWork(Configuration conf) {
     return (MapWork) getBaseWork(conf, MAP_PLAN_NAME);
   }
 
+  public static void setReduceWork(Configuration conf, ReduceWork work) {
+    setBaseWork(conf, REDUCE_PLAN_NAME, work);
+  }
+
   public static ReduceWork getReduceWork(Configuration conf) {
     return (ReduceWork) getBaseWork(conf, REDUCE_PLAN_NAME);
   }
 
   /**
+   * Pushes work into the global work map
+   */
+  public static void setBaseWork(Configuration conf, String name, BaseWork work) {
+    Path path = getPlanPath(conf, name);
+    gWorkMap.put(path, work);
+  }
+
+  /**
    * Returns the Map or Reduce plan
    * Side effect: the BaseWork returned is also placed in the gWorkMap
    * @param conf
@@ -288,8 +304,7 @@ public final class Utilities {
     try {
       path = getPlanPath(conf, name);
       assert path != null;
-      gWork = gWorkMap.get(path);
-      if (gWork == null) {
+      if (!gWorkMap.containsKey(path)) {
         Path localPath;
         if (ShimLoader.getHadoopShims().isLocalMode(conf)) {
           localPath = path;
@@ -334,6 +349,7 @@ public final class Utilities {
         gWorkMap.put(path, gWork);
       } else {
         LOG.debug("Found plan in cache.");
+        gWork = gWorkMap.get(path);
       }
       return gWork;
     } catch (FileNotFoundException fnf) {

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java Mon Nov
25 22:40:22 2013
@@ -110,6 +110,8 @@ public class ExecMapper extends MapReduc
       if (mrwork == null) {
         mrwork = Utilities.getMapWork(job);
         cache.cache(PLAN_KEY, mrwork);
+      } else {
+        Utilities.setMapWork(job, mrwork);
       }
       if (mrwork.getVectorMode()) {
         mo = new VectorMapOperator();

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java Mon Nov
25 22:40:22 2013
@@ -122,6 +122,8 @@ public class ExecReducer extends MapRedu
     if (gWork == null) {
       gWork = Utilities.getReduceWork(job);
       cache.cache(PLAN_KEY, gWork);
+    } else {
+      Utilities.setReduceWork(job, gWork);
     }
 
     reducer = gWork.getReducer();

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
(original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
Mon Nov 25 22:40:22 2013
@@ -95,6 +95,8 @@ public class MapRecordProcessor  extends
         for (String s: mapWork.getAliases()) {
           l4j.info("Alias: "+s);
         }
+      } else {
+        Utilities.setMapWork(jconf, mapWork);
       }
       if (mapWork.getVectorMode()) {
         mapOp = new VectorMapOperator();

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java Mon
Nov 25 22:40:22 2013
@@ -36,14 +36,16 @@ public class ObjectCache implements org.
 
   @Override
   public void cache(String key, Object value) {
-    LOG.info("Adding " + key + " to cache");
+    LOG.info("Adding " + key + " to cache with value " + value);
     registry.add(ObjectLifeCycle.VERTEX, key, value);
   }
 
   @Override
   public Object retrieve(String key) {
     Object o = registry.get(key);
-    LOG.info("Found " + key + " in cache with value: " + o);
+    if (o != null) {
+      LOG.info("Found " + key + " in cache with value: " + o);
+    }
     return o;
   }
 }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
(original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
Mon Nov 25 22:40:22 2013
@@ -104,6 +104,8 @@ public class ReduceRecordProcessor  exte
     if (redWork == null) {
       redWork = Utilities.getReduceWork(jconf);
       cache.cache(REDUCE_PLAN_KEY, redWork);
+    } else {
+      Utilities.setReduceWork(jconf, redWork);
     }
 
     reducer = redWork.getReducer();



Mime
View raw message