hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1130434 - in /hadoop/mapreduce/branches/MR-279: ./ yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/ha...
Date Thu, 02 Jun 2011 08:13:17 GMT
Author: acmurthy
Date: Thu Jun  2 08:13:17 2011
New Revision: 1130434

URL: http://svn.apache.org/viewvc?rev=1130434&view=rev
Log:
Fix calculation of max-capacity for a queue, also fixed a bug in registration of NodeManagers.


Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1130434&r1=1130433&r2=1130434&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Thu Jun  2 08:13:17 2011
@@ -5,6 +5,9 @@ Trunk (unreleased changes)
 
     MAPREDUCE-279
 
+    Fix calculation of max-capacity for a queue, also fixed a bug in 
+    registration of NodeManagers. (acmurthy) 
+  
     Making each node aggregate all its user-logs to a separate hdfs
     file. (vinodkv)
 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java?rev=1130434&r1=1130433&r2=1130434&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java
Thu Jun  2 08:13:17 2011
@@ -98,6 +98,8 @@ public class NodeManagerImpl implements 
     this.httpAddress = hostName + ":" + httpPort;
     Resources.addTo(availableResource, capability);
     this.node = node;
+    this.nodeHealthStatus.setIsNodeHealthy(true);
+    this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
   }
 
   /**

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1130434&r1=1130433&r2=1130434&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Thu Jun  2 08:13:17 2011
@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -441,7 +440,7 @@ implements ResourceScheduler, CapacitySc
   @Override
   public synchronized void nodeUpdate(NodeInfo nm, 
       Map<String,List<Container>> containers ) {
-    LOG.info("nodeUpdate: " + nm);
+    LOG.info("nodeUpdate: " + nm + " clusterResources: " + clusterResource);
 
 
     // Completed containers

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1130434&r1=1130433&r2=1130434&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Thu Jun  2 08:13:17 2011
@@ -603,14 +603,16 @@ public class LeafQueue implements Queue 
 
   private synchronized boolean assignToQueue(Resource clusterResource, 
       Resource required) {
-    // Check how of the parent's capacity we are currently using...
-    float newCapacity = 
+    // Check how of the cluster's absolute capacity we are currently using...
+    float potentialNewCapacity = 
       (float)(usedResources.getMemory() + required.getMemory()) / 
-      (clusterResource.getMemory() * parent.getAbsoluteCapacity());
-    if (newCapacity > absoluteMaxCapacity) {
+        clusterResource.getMemory();
+    if (potentialNewCapacity > absoluteMaxCapacity) {
       LOG.info(getQueueName() + 
-          " current-capacity (" + getUtilization() + ") +" +
-          " required (" + required.getMemory() + ")" +
+          " usedResources: " + usedResources.getMemory() + 
+          " currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory()
+ 
+          " required " + required.getMemory() +
+          " potentialNewCapacity: " + potentialNewCapacity + " ( " +
           " > max-capacity (" + absoluteMaxCapacity + ")");
       return false;
     }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1130434&r1=1130433&r2=1130434&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
Thu Jun  2 08:13:17 2011
@@ -554,12 +554,9 @@ public class ParentQueue implements Queu
   }
   
   private synchronized boolean assignToQueue(Resource clusterResource) {
-    // Check how of the parent's capacity we are currently using...
-    float parentAbsoluteCapacity = 
-          (parent == null) ? 100.0f : parent.getAbsoluteCapacity();
+    // Check how of the cluster's absolute capacity we are currently using...
     float currentCapacity = 
-      (float)(usedResources.getMemory()) / 
-      (clusterResource.getMemory() * parentAbsoluteCapacity);
+      (float)(usedResources.getMemory()) / clusterResource.getMemory();
     if (currentCapacity > absoluteMaxCapacity) {
       LOG.info(getQueueName() + 
           " current-capacity (" + currentCapacity + ") +" +



Mime
View raw message