hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r1085233 - in /hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src: main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/ main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/cap...
Date Fri, 25 Mar 2011 01:10:49 GMT
Author: mahadev
Date: Fri Mar 25 01:10:49 2011
New Revision: 1085233

URL: http://svn.apache.org/viewvc?rev=1085233&view=rev
Log:
Fix NodeManager expiry in the RM to notify scheduler and remove from data structures of RM
contributed by mahadev konar

Modified:
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java?rev=1085233&r1=1085232&r2=1085233&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
Fri Mar 25 01:10:49 2011
@@ -269,8 +269,15 @@ ResourceTracker, ResourceContext {
       nmExpiryQueue.add(status);
     }
   }
+  
   protected void expireNMs(List<NodeID> nodes) {
-    /* for now do nothing */
+    for (NodeID id: nodes) {
+      synchronized (nodeManagers) {
+        NodeInfo nInfo = nodeManagers.get(id).getNodeInfo();
+        nodeManagers.remove(id);
+        resourceListener.removeNode(nInfo);
+      }
+    }
   }
 
   /*

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1085233&r1=1085232&r2=1085233&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Fri Mar 25 01:10:49 2011
@@ -384,6 +384,7 @@ implements ResourceScheduler, CapacitySc
   public synchronized void removeNode(NodeInfo nodeInfo) {
     org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource(
         clusterResource, nodeInfo.getTotalCapability());
+    //TODO inform the applications that the containers are completed/failed
     nodes.remove(nodeInfo.getHostName());
   }
   

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1085233&r1=1085232&r2=1085233&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Fri Mar 25 01:10:49 2011
@@ -454,6 +454,7 @@ public class FifoScheduler implements Re
   public synchronized void removeNode(NodeInfo nodeInfo) {
     org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource(
         clusterResource, nodeInfo.getTotalCapability());
+    //TODO inform the the applications that the containers are completed/failed
     nodes.remove(nodeInfo.getHostName());
   }
   

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java?rev=1085233&r1=1085232&r2=1085233&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
Fri Mar 25 01:10:49 2011
@@ -1,20 +1,20 @@
 /**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.hadoop.yarn.server.resourcemanager.resourcetracker;
 
@@ -52,7 +52,7 @@ public class TestNMExpiry extends TestCa
   ContainerTokenSecretManager containerTokenSecretManager = 
     new ContainerTokenSecretManager();
   AtomicInteger test = new AtomicInteger();
-  Object notify = new Object();
+  AtomicInteger notify = new AtomicInteger();
 
   private static class VoidResourceListener implements ResourceListener {
     @Override
@@ -89,6 +89,7 @@ public class TestNMExpiry extends TestCa
             Thread.sleep(2000);
           } catch(InterruptedException ie){}
           synchronized(notify) {
+            notify.addAndGet(1);
             notify.notifyAll();
           }
         }
@@ -137,7 +138,9 @@ public class TestNMExpiry extends TestCa
     stopT = false;
     new TestThread().start();
     synchronized (notify) {
-      notify.wait(10000);
+      while (notify.get() == 0) {
+        notify.wait();
+      }
     }
     if (test.get() != 2) 
       assertTrue(false);



Mime
View raw message