Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9F4B098F4 for ; Tue, 5 Jun 2012 23:56:12 +0000 (UTC) Received: (qmail 68105 invoked by uid 500); 5 Jun 2012 23:56:12 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 68051 invoked by uid 500); 5 Jun 2012 23:56:12 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 68043 invoked by uid 99); 5 Jun 2012 23:56:12 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Jun 2012 23:56:12 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Jun 2012 23:56:08 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 86EAC2388ACC; Tue, 5 Jun 2012 23:55:47 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1346673 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ hadoop-yarn/hadoop-yarn-server/hadoop... Date: Tue, 05 Jun 2012 23:55:47 -0000 To: mapreduce-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120605235547.86EAC2388ACC@eris.apache.org> Author: vinodkv Date: Tue Jun 5 23:55:46 2012 New Revision: 1346673 URL: http://svn.apache.org/viewvc?rev=1346673&view=rev Log: MAPREDUCE-3873. Fixed NodeManagers' decommissioning at RM to accept IP addresses also. Contributed by xieguiming and vinodkv. svn merge --ignore-ancestry -c 1346671 ../../trunk/ Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1346673&r1=1346672&r2=1346673&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Jun 5 23:55:46 2012 @@ -45,6 +45,9 @@ Release 2.0.1-alpha - UNRELEASED MAPREDUCE-4313. TestTokenCache doesn't compile due TokenCache.getDelegationToken compilation error (bobby) + MAPREDUCE-3873. Fixed NodeManagers' decommissioning at RM to accept IP + addresses also. (xieguiming via vinodkv) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java?rev=1346673&r1=1346672&r2=1346673&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java Tue Jun 5 23:55:46 2012 @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHa import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -37,6 +38,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.service.AbstractService; +@SuppressWarnings("unchecked") public class NodesListManager extends AbstractService implements EventHandler { @@ -112,8 +114,10 @@ public class NodesListManager extends Ab synchronized (hostsReader) { Set hostsList = hostsReader.getHosts(); Set excludeList = hostsReader.getExcludedHosts(); - return ((hostsList.isEmpty() || hostsList.contains(hostName)) && - !excludeList.contains(hostName)); + String ip = NetUtils.normalizeHostName(hostName); + return (hostsList.isEmpty() || hostsList.contains(hostName) || hostsList + .contains(ip)) + && !(excludeList.contains(hostName) || excludeList.contains(ip)); } } Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java?rev=1346673&r1=1346672&r2=1346673&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java Tue Jun 5 23:55:46 2012 @@ -28,14 +28,14 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; @@ -52,8 +52,6 @@ public class TestResourceTrackerService "test.build.data", "/tmp"), "decommision"); private File hostFile = new File(TEMP_DIR + File.separator + "hostFile.txt"); private MockRM rm; - private static final RecordFactory recordFactory = RecordFactoryProvider - .getRecordFactory(null); /** * decommissioning using a include hosts file @@ -61,9 +59,9 @@ public class TestResourceTrackerService @Test public void testDecommissionWithIncludeHosts() throws Exception { - writeToHostsFile("host1", "host2"); + writeToHostsFile("localhost", "host1", "host2"); Configuration conf = new Configuration(); - conf.set("yarn.resourcemanager.nodes.include-path", hostFile + conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile .getAbsolutePath()); rm = new MockRM(conf); @@ -71,17 +69,22 @@ public class TestResourceTrackerService MockNM nm1 = rm.registerNode("host1:1234", 5120); MockNM nm2 = rm.registerNode("host2:5678", 10240); + MockNM nm3 = rm.registerNode("localhost:4433", 1024); ClusterMetrics metrics = ClusterMetrics.getMetrics(); assert(metrics != null); - int initialMetricCount = metrics.getNumDecommisionedNMs(); + int metricCount = metrics.getNumDecommisionedNMs(); HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); nodeHeartbeat = nm2.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); + nodeHeartbeat = nm3.nodeHeartbeat(true); + Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); - writeToHostsFile("host1"); + // To test that IPs also work + String ip = NetUtils.normalizeHostName("localhost"); + writeToHostsFile("host1", ip); rm.getNodesListManager().refreshNodes(); @@ -94,7 +97,12 @@ public class TestResourceTrackerService Assert.assertTrue("Node is not decommisioned.", NodeAction.SHUTDOWN .equals(nodeHeartbeat.getNodeAction())); - checkDecommissionedNMCount(rm, ++initialMetricCount); + checkDecommissionedNMCount(rm, ++metricCount); + + nodeHeartbeat = nm3.nodeHeartbeat(true); + Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); + Assert.assertEquals(metricCount, ClusterMetrics.getMetrics() + .getNumDecommisionedNMs()); } /** @@ -103,7 +111,7 @@ public class TestResourceTrackerService @Test public void testDecommissionWithExcludeHosts() throws Exception { Configuration conf = new Configuration(); - conf.set("yarn.resourcemanager.nodes.exclude-path", hostFile + conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile .getAbsolutePath()); writeToHostsFile(""); @@ -112,16 +120,18 @@ public class TestResourceTrackerService MockNM nm1 = rm.registerNode("host1:1234", 5120); MockNM nm2 = rm.registerNode("host2:5678", 10240); + MockNM nm3 = rm.registerNode("localhost:4433", 1024); - int initialMetricCount = ClusterMetrics.getMetrics() - .getNumDecommisionedNMs(); + int metricCount = ClusterMetrics.getMetrics().getNumDecommisionedNMs(); HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); nodeHeartbeat = nm2.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); - writeToHostsFile("host2"); + // To test that IPs also work + String ip = NetUtils.normalizeHostName("localhost"); + writeToHostsFile("host2", ip); rm.getNodesListManager().refreshNodes(); @@ -130,14 +140,19 @@ public class TestResourceTrackerService nodeHeartbeat = nm2.nodeHeartbeat(true); Assert.assertTrue("The decommisioned metrics are not updated", NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction())); - checkDecommissionedNMCount(rm, ++initialMetricCount); + checkDecommissionedNMCount(rm, ++metricCount); + + nodeHeartbeat = nm3.nodeHeartbeat(true); + Assert.assertTrue("The decommisioned metrics are not updated", + NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction())); + checkDecommissionedNMCount(rm, ++metricCount); } @Test public void testNodeRegistrationFailure() throws Exception { writeToHostsFile("host1"); Configuration conf = new Configuration(); - conf.set("yarn.resourcemanager.nodes.include-path", hostFile + conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile .getAbsolutePath()); rm = new MockRM(conf); rm.start(); @@ -191,7 +206,7 @@ public class TestResourceTrackerService @Test public void testUnhealthyNodeStatus() throws Exception { Configuration conf = new Configuration(); - conf.set("yarn.resourcemanager.nodes.exclude-path", hostFile + conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile .getAbsolutePath()); rm = new MockRM(conf);