hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ka...@apache.org
Subject [09/50] [abbrv] hadoop git commit: YARN-4611. Fix scheduler load simulator to support multi-layer network location. Contributed by Ming Ma.
Date Mon, 25 Jan 2016 17:25:44 GMT
YARN-4611. Fix scheduler load simulator to support multi-layer network
location. Contributed by Ming Ma.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1708a4cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1708a4cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1708a4cd

Branch: refs/heads/YARN-1011
Commit: 1708a4cd2377c56e2cb5738720b7eaf10baf13c8
Parents: 890a2eb
Author: Xuan <xgong@apache.org>
Authored: Wed Jan 20 15:41:50 2016 -0800
Committer: Xuan <xgong@apache.org>
Committed: Wed Jan 20 15:41:50 2016 -0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java  | 11 +++++++++--
 .../hadoop/yarn/sls/nodemanager/TestNMSimulator.java     |  2 +-
 .../org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java   |  5 +++++
 hadoop-yarn-project/CHANGES.txt                          |  3 +++
 4 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
index 922370c..d950aee 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.NodeBase;
 import org.apache.hadoop.tools.rumen.JobTraceReader;
 import org.apache.hadoop.tools.rumen.LoggedJob;
 import org.apache.hadoop.tools.rumen.LoggedTask;
@@ -43,9 +44,15 @@ import org.codehaus.jackson.map.ObjectMapper;
 @Unstable
 public class SLSUtils {
 
+  // hostname includes the network path and the host name. for example
+  // "/default-rack/hostFoo" or "/coreSwitchA/TORSwitchB/hostBar".
+  // the function returns two Strings, the first element is the network
+  // location without "/", the second element is the host name. for example,
+  // {"default-rack", "hostFoo"} or "coreSwitchA/TORSwitchB", "hostBar"
   public static String[] getRackHostName(String hostname) {
-    hostname = hostname.substring(1);
-    return hostname.split("/");
+    NodeBase node = new NodeBase(hostname);
+    return new String[] {node.getNetworkLocation().substring(1),
+        node.getName()};
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
index 2381549..f9a3932 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
@@ -51,7 +51,7 @@ public class TestNMSimulator {
   public void testNMSimulator() throws Exception {
     // Register one node
     NMSimulator node1 = new NMSimulator();
-    node1.init("rack1/node1", GB * 10, 10, 0, 1000, rm);
+    node1.init("/rack1/node1", GB * 10, 10, 0, 1000, rm);
     node1.middleStep();
 
     int numClusterNodes = rm.getResourceScheduler().getNumClusterNodes();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
index 498615a..f4eda67 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
@@ -29,6 +29,11 @@ public class TestSLSUtils {
     String rackHostname[] = SLSUtils.getRackHostName(str);
     Assert.assertEquals(rackHostname[0], "rack1");
     Assert.assertEquals(rackHostname[1], "node1");
+
+    str = "/rackA/rackB/node1";
+    rackHostname = SLSUtils.getRackHostName(str);
+    Assert.assertEquals(rackHostname[0], "rackA/rackB");
+    Assert.assertEquals(rackHostname[1], "node1");
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ee78d22..212ffa1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -131,6 +131,9 @@ Release 2.9.0 - UNRELEASED
     YARN-3446. FairScheduler headroom calculation should exclude nodes in the 
     blacklist. (Zhihai Xu via kasha)
 
+    YARN-4611. Fix scheduler load simulator to support multi-layer network
+    location. (Ming Ma via xgong)
+
 Release 2.8.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES


Mime
View raw message