hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjs...@apache.org
Subject svn commit: r1587718 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-...
Date Tue, 15 Apr 2014 20:40:54 GMT
Author: zjshen
Date: Tue Apr 15 20:40:53 2014
New Revision: 1587718

URL: http://svn.apache.org/r1587718
Log:
YARN-1892. Improved some logs in the scheduler. Contributed by Jian He.
svn merge --ignore-ancestry -c 1587717 ../../trunk/

Modified:
    hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1587718&r1=1587717&r2=1587718&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Tue Apr 15 20:40:53 2014
@@ -58,6 +58,8 @@ Release 2.4.1 - UNRELEASED
 
   IMPROVEMENTS
 
+    YARN-1892. Improved some logs in the scheduler. (Jian He via zjshen)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1587718&r1=1587717&r2=1587718&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
Tue Apr 15 20:40:53 2014
@@ -343,10 +343,6 @@ public class NodeStatusUpdaterImpl exten
           + ", " + nodeHealthStatus.getHealthReport());
     }
     List<ContainerStatus> containersStatuses = getContainerStatuses();
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(this.nodeId + " sending out status for "
-          + containersStatuses.size() + " containers");
-    }
     NodeStatus nodeStatus =
         NodeStatus.newInstance(nodeId, responseId, containersStatuses,
           createKeepAliveApplicationList(), nodeHealthStatus);
@@ -373,7 +369,8 @@ public class NodeStatusUpdaterImpl exten
       }
     }
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Sending out container statuses: " + containerStatuses);
+      LOG.debug("Sending out " + containerStatuses.size()
+          + " container statuses: " + containerStatuses);
     }
     return containerStatuses;
   }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java?rev=1587718&r1=1587717&r2=1587718&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
Tue Apr 15 20:40:53 2014
@@ -255,13 +255,15 @@ public class SchedulerApplicationAttempt
       this.reservedContainers.put(priority, reservedContainers);
     }
     reservedContainers.put(node.getNodeID(), rmContainer);
-    
-    LOG.info("Application " + getApplicationId() 
-        + " reserved container " + rmContainer
-        + " on node " + node + ", currently has " + reservedContainers.size()
-        + " at priority " + priority 
-        + "; currentReservation " + currentReservation.getMemory());
-    
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Application attempt " + getApplicationAttemptId()
+          + " reserved container " + rmContainer + " on node " + node
+          + ". This attempt currently has " + reservedContainers.size()
+          + " reserved containers at priority " + priority
+          + "; currentReservation " + currentReservation.getMemory());
+    }
+
     return rmContainer;
   }
   

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1587718&r1=1587717&r2=1587718&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Tue Apr 15 20:40:53 2014
@@ -107,8 +107,6 @@ public class LeafQueue implements CSQueu
   private final Resource maximumAllocation;
   private final float minimumAllocationFactor;
 
-  private RMContainerTokenSecretManager containerTokenSecretManager;
-
   private Map<String, User> users = new HashMap<String, User>();
   
   private final QueueMetrics metrics;
@@ -149,7 +147,6 @@ public class LeafQueue implements CSQueu
         Resources.ratio(resourceCalculator, 
             Resources.subtract(maximumAllocation, minimumAllocation), 
             maximumAllocation);
-    this.containerTokenSecretManager = cs.getContainerTokenSecretManager();
 
     float capacity = 
       (float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
@@ -560,7 +557,7 @@ public class LeafQueue implements CSQueu
     return queueName + ": " + 
         "capacity=" + capacity + ", " + 
         "absoluteCapacity=" + absoluteCapacity + ", " + 
-        "usedResources=" + usedResources +  
+        "usedResources=" + usedResources +  ", " +
         "usedCapacity=" + getUsedCapacity() + ", " + 
         "absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " +
         "numApps=" + getNumApplications() + ", " + 
@@ -949,15 +946,16 @@ public class LeafQueue implements CSQueu
             Resources.add(usedResources, required), 
             clusterResource);
     if (potentialNewCapacity > absoluteMaxCapacity) {
-      LOG.info(getQueueName() + 
-          " usedResources: " + usedResources +
-          " clusterResources: " + clusterResource +
-          " currentCapacity " + 
-            Resources.divide(resourceCalculator, clusterResource, 
-                usedResources, clusterResource) + 
-          " required " + required +
-          " potentialNewCapacity: " + potentialNewCapacity + " ( " +
-          " max-capacity: " + absoluteMaxCapacity + ")");
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(getQueueName()
+            + " usedResources: " + usedResources
+            + " clusterResources: " + clusterResource
+            + " currentCapacity "
+            + Resources.divide(resourceCalculator, clusterResource,
+              usedResources, clusterResource) + " required " + required
+            + " potentialNewCapacity: " + potentialNewCapacity + " ( "
+            + " max-capacity: " + absoluteMaxCapacity + ")");
+      }
       return false;
     }
     return true;
@@ -1300,7 +1298,7 @@ public class LeafQueue implements CSQueu
       ResourceRequest request, NodeType type, RMContainer rmContainer) {
     if (LOG.isDebugEnabled()) {
       LOG.debug("assignContainers: node=" + node.getNodeName()
-        + " application=" + application.getApplicationId().getId()
+        + " application=" + application.getApplicationId()
         + " priority=" + priority.getPriority()
         + " request=" + request + " type=" + type);
     }
@@ -1352,14 +1350,10 @@ public class LeafQueue implements CSQueu
           allocatedContainer);
 
       LOG.info("assignedContainer" +
-          " application=" + application.getApplicationId() +
+          " application attempt=" + application.getApplicationAttemptId() +
           " container=" + container + 
-          " containerId=" + container.getId() + 
           " queue=" + this + 
-          " usedCapacity=" + getUsedCapacity() +
-          " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
-          " used=" + usedResources + 
-          " cluster=" + clusterResource);
+          " clusterResource=" + clusterResource);
 
       return container.getResource();
     } else {
@@ -1367,13 +1361,11 @@ public class LeafQueue implements CSQueu
       reserve(application, priority, node, rmContainer, container);
 
       LOG.info("Reserved container " + 
-          " application=" + application.getApplicationId() +
+          " application attempt=" + application.getApplicationAttemptId() +
           " resource=" + request.getCapability() + 
           " queue=" + this.toString() + 
-          " usedCapacity=" + getUsedCapacity() +
-          " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
-          " used=" + usedResources + 
-          " cluster=" + clusterResource);
+          " node=" + node +
+          " clusterResource=" + clusterResource);
 
       return request.getCapability();
     }
@@ -1440,11 +1432,7 @@ public class LeafQueue implements CSQueu
               application, container.getResource());
           LOG.info("completedContainer" +
               " container=" + container +
-              " resource=" + container.getResource() +
               " queue=" + this +
-              " usedCapacity=" + getUsedCapacity() +
-              " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
-              " used=" + usedResources +
               " cluster=" + clusterResource);
         }
       }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java?rev=1587718&r1=1587717&r2=1587718&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
Tue Apr 15 20:40:53 2014
@@ -231,12 +231,19 @@ public class FiCaSchedulerNode extends S
             " on node " + this);
       }
 
-      LOG.info("Updated reserved container " + 
-          reservedContainer.getContainer().getId() + " on node " + 
-          this + " for application " + application);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Updated reserved container "
+            + reservedContainer.getContainer().getId() + " on node " + this
+            + " for application attempt "
+            + application.getApplicationAttemptId());
+      }
     } else {
-      LOG.info("Reserved container " + reservedContainer.getContainer().getId() + 
-          " on node " + this + " for application " + application);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Reserved container "
+            + reservedContainer.getContainer().getId() + " on node " + this
+            + " for application attempt "
+            + application.getApplicationAttemptId());
+      }
     }
     this.reservedContainer = reservedContainer;
   }



Mime
View raw message