hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oz...@apache.org
Subject hadoop git commit: YARN-3258. FairScheduler: Need to add more logging to investigate allocations. Contributed by Anubhav Dhoot.
Date Tue, 31 Mar 2015 08:43:28 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 312e50f9c -> 99b825569


YARN-3258. FairScheduler: Need to add more logging to investigate allocations. Contributed
by Anubhav Dhoot.

(cherry picked from commit b5a22e983832d4843b5df1d07858988e8bbf37e3)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99b82556
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99b82556
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99b82556

Branch: refs/heads/branch-2
Commit: 99b8255693f92db8c6cd6666e8c976e7f4263dbc
Parents: 312e50f
Author: Tsuyoshi Ozawa <ozawa@apache.org>
Authored: Tue Mar 31 17:42:44 2015 +0900
Committer: Tsuyoshi Ozawa <ozawa@apache.org>
Committed: Tue Mar 31 17:43:07 2015 +0900

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                              | 3 +++
 .../server/resourcemanager/scheduler/fair/FSAppAttempt.java  | 4 ++++
 .../server/resourcemanager/scheduler/fair/FSLeafQueue.java   | 8 +++++++-
 .../yarn/server/resourcemanager/scheduler/fair/FSQueue.java  | 8 ++++++++
 4 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b82556/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d2850ad..b76ef16 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -38,6 +38,9 @@ Release 2.8.0 - UNRELEASED
     YARN-2495. Allow admin specify labels from each NM (Distributed 
     configuration for node label). (Naganarasimha G R via wangda)
 
+    YARN-3258. FairScheduler: Need to add more logging to investigate
+    allocations. (Anubhav Dhoot via ozawa)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b82556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index dfde5ab..46617ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -570,6 +570,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
         // Check the AM resource usage for the leaf queue
         if (getLiveContainers().size() == 0 && !getUnmanagedAM()) {
           if (!getQueue().canRunAppAM(getAMResource())) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Skipping allocation because maxAMShare limit would " +
+                  "be exceeded");
+            }
             return Resources.none();
           }
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b82556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 3c97535..c49a323 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -284,6 +284,8 @@ public class FSLeafQueue extends FSQueue {
     if (LOG.isDebugEnabled()) {
       LOG.debug("The updated demand for " + getName() + " is " + demand
           + "; the max is " + maxRes);
+      LOG.debug("The updated fairshare for " + getName() + " is "
+          + getFairShare());
     }
   }
   
@@ -304,7 +306,7 @@ public class FSLeafQueue extends FSQueue {
     Resource assigned = Resources.none();
     if (LOG.isDebugEnabled()) {
       LOG.debug("Node " + node.getNodeName() + " offered to queue: " +
-          getName());
+          getName() + " fairShare: " + getFairShare());
     }
 
     if (!assignContainerPreCheck(node)) {
@@ -330,6 +332,10 @@ public class FSLeafQueue extends FSQueue {
 
         assigned = sched.assignContainer(node);
         if (!assigned.equals(Resources.none())) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Assigned container in queue:" + getName() + " " +
+                "container:" + assigned);
+          }
           break;
         }
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b82556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 1562bf6..ee47d9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -23,6 +23,8 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -41,6 +43,9 @@ import org.apache.hadoop.yarn.util.resource.Resources;
 @Private
 @Unstable
 public abstract class FSQueue implements Queue, Schedulable {
+  private static final Log LOG = LogFactory.getLog(
+      FSQueue.class.getName());
+
   private Resource fairShare = Resources.createResource(0, 0);
   private Resource steadyFairShare = Resources.createResource(0, 0);
   private final String name;
@@ -164,6 +169,9 @@ public abstract class FSQueue implements Queue, Schedulable {
   public void setFairShare(Resource fairShare) {
     this.fairShare = fairShare;
     metrics.setFairShare(fairShare);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("The updated fairShare for " + getName() + " is " + fairShare);
+    }
   }
 
   /** Get the steady fair share assigned to this Schedulable. */


Mime
View raw message