tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject tez git commit: TEZ-3852. Optimize ContainerContext.isSuperSet to speed container reuse decisions. Contributed by Jonathan Eagles
Date Thu, 12 Oct 2017 14:35:02 GMT
Repository: tez
Updated Branches:
  refs/heads/master c82b2eade -> 0207281b3


TEZ-3852. Optimize ContainerContext.isSuperSet to speed container reuse decisions. Contributed
by Jonathan Eagles


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0207281b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0207281b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0207281b

Branch: refs/heads/master
Commit: 0207281b3cea4f78b5a57fb81fef0cf14d8c24e1
Parents: c82b2ea
Author: Jason Lowe <jlowe@apache.org>
Authored: Thu Oct 12 09:33:14 2017 -0500
Committer: Jason Lowe <jlowe@apache.org>
Committed: Thu Oct 12 09:33:14 2017 -0500

----------------------------------------------------------------------
 .../apache/tez/dag/app/ContainerContext.java    | 52 ++++++++------------
 .../tez/dag/app/rm/TestContainerReuse.java      |  4 ++
 2 files changed, 24 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/0207281b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java
index f00b27b..d88daff 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java
@@ -129,26 +129,24 @@ public class ContainerContext {
   // classpath modification
   private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs,
       Map<String, LocalResource> reqLRs) {
-    Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs);
-    for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) {
-      LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey());
-      if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource))
{
+    for (Entry<String, LocalResource> reqLREntry : reqLRs.entrySet()) {
+      LocalResource requestedLocalResource = srcLRs.get(reqLREntry.getKey());
+      if (requestedLocalResource == null) {
+        LocalResource lr = reqLREntry.getValue();
+        if (!LocalResourceType.FILE.equals(lr.getType())) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Cannot match container: Additional local resource needed is not of
type FILE"
+                + ", resourceName: " + reqLREntry.getKey()
+                + ", resourceDetails: " + reqLREntry);
+          }
+          return false;
+        }
+      } else if(!reqLREntry.getValue().equals(requestedLocalResource)) {
         if (LOG.isDebugEnabled()) {
           LOG.debug("Cannot match container: Attempting to use same target resource name:
"
-              + srcLREntry.getKey()
+              + reqLREntry.getKey()
               + ", but with different source resources. Already localized: "
-              + srcLREntry.getValue() + ", requested: " + requestedLocalResource);
-        }
-        return false;
-      }
-    }
-    for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) {
-      LocalResource lr = additionalLREntry.getValue();
-      if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType()))
{
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Cannot match container: Additional local resource needed is not of type
FILE"
-              + ", resourceName: " + additionalLREntry.getKey()
-              + ", resourceDetails: " + additionalLREntry);
+              + requestedLocalResource + ", requested: " + reqLREntry.getValue());
         }
         return false;
       }
@@ -161,24 +159,14 @@ public class ContainerContext {
     for (Entry<K, V> oEntry : matchMap.entrySet()) {
       K oKey = oEntry.getKey();
       V oVal = oEntry.getValue();
-      if (srcMap.containsKey(oKey)) {
-        if (!oVal.equals(srcMap.get(oKey))) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Incompatible container context"
+      V srcVal = srcMap.get(oKey);
+      if (!oVal.equals(srcVal)) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Incompatible container context"
               + ", matchInfo=" + matchInfo
               + ", thisKey=" + oKey
-              + ", thisVal=" + srcMap.get(oKey)
+              + ", thisVal=" + srcVal
               + ", otherVal=" + oVal);
-          }
-          return false;
-        }
-      } else {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Incompatible container context"
-            + ", matchInfo=" + matchInfo
-            + ", thisKey=" + oKey
-            + ", thisVal=null"
-            + ", otherVal=" + oVal);
         }
         return false;
       }

http://git-wip-us.apache.org/repos/asf/tez/blob/0207281b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index 7e9e9ab..4e29dd5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -38,6 +38,7 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.tez.common.TezUtils;
 import org.apache.tez.serviceplugins.api.TaskScheduler;
 import org.mockito.ArgumentCaptor;
@@ -1127,8 +1128,11 @@ public class TestContainerReuse {
     String rsrc1 = "rsrc1";
     String rsrc2 = "rsrc2";
     LocalResource lr1 = mock(LocalResource.class);
+    doReturn(LocalResourceType.FILE).when(lr1).getType();
     LocalResource lr2 = mock(LocalResource.class);
+    doReturn(LocalResourceType.FILE).when(lr2).getType();
     LocalResource lr3 = mock(LocalResource.class);
+    doReturn(LocalResourceType.FILE).when(lr3).getType();
 
     AMContainerEventAssignTA assignEvent = null;
 


Mime
View raw message