hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject hadoop git commit: YARN-8333. Load balance YARN services using RegistryDNS multiple A records. Contributed by Eric Yang
Date Thu, 31 May 2018 13:47:35 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 6468071f1 -> 6bc92e304


YARN-8333. Load balance YARN services using RegistryDNS multiple A records. Contributed by
Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bc92e30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bc92e30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bc92e30

Branch: refs/heads/trunk
Commit: 6bc92e304fe05e80f13830104d1fd2c59da8344b
Parents: 6468071
Author: Billie Rinaldi <billie@apache.org>
Authored: Thu May 31 06:46:34 2018 -0700
Committer: Billie Rinaldi <billie@apache.org>
Committed: Thu May 31 06:46:34 2018 -0700

----------------------------------------------------------------------
 .../server/dns/BaseServiceRecordProcessor.java  | 20 +++++++
 .../dns/ContainerServiceRecordProcessor.java    |  3 +-
 .../registry/server/dns/TestRegistryDNS.java    | 62 ++++++++++++++++----
 .../markdown/yarn-service/ServiceDiscovery.md   | 14 ++++-
 4 files changed, 84 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bc92e30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
index 51ae99a..f30c0c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
@@ -290,6 +290,26 @@ public abstract class BaseServiceRecordProcessor
           domain));
     }
 
+    /**
+     * Return the DNS name constructed from the component name.
+     *
+     * @return the DNS naem.
+     * @throws PathNotFoundException
+     * @throws TextParseException
+     */
+    protected Name getComponentName()
+        throws PathNotFoundException, TextParseException {
+      String service = RegistryPathUtils.lastPathEntry(
+          RegistryPathUtils.parentOf(RegistryPathUtils.parentOf(getPath())));
+      String component = getRecord().get("yarn:component").toLowerCase();
+      String user = RegistryPathUtils.getUsername(getPath());
+      return Name.fromString(MessageFormat.format("{0}.{1}.{2}.{3}",
+          component,
+          service,
+          user,
+          domain));
+    }
+
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bc92e30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java
index 2e95f54..e40a177 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java
@@ -242,7 +242,8 @@ public class ContainerServiceRecordProcessor extends
       }
       try {
         this.setTarget(InetAddress.getByName(ip));
-        this.setNames(new Name[] {getContainerName(), getContainerIDName()});
+        this.setNames(new Name[] {getContainerName(), getContainerIDName(),
+            getComponentName()});
       } catch (Exception e) {
         throw new IllegalStateException(e);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bc92e30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
index bce73ad..01adc45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -115,32 +115,47 @@ public class TestRegistryDNS extends Assert {
       + "}\n";
   static final String CONTAINER_RECORD = "{\n"
       + "  \"type\" : \"JSONServiceRecord\",\n"
-      + "  \"description\" : \"COMP-NAME\",\n"
+      + "  \"description\" : \"httpd-1\",\n"
       + "  \"external\" : [ ],\n"
       + "  \"internal\" : [ ],\n"
       + "  \"yarn:id\" : \"container_e50_1451931954322_0016_01_000002\",\n"
       + "  \"yarn:persistence\" : \"container\",\n"
       + "  \"yarn:ip\" : \"172.17.0.19\",\n"
-      + "  \"yarn:hostname\" : \"0a134d6329ba\"\n"
+      + "  \"yarn:hostname\" : \"host1\",\n"
+      + "  \"yarn:component\" : \"httpd\"\n"
+      + "}\n";
+
+  static final String CONTAINER_RECORD2 = "{\n"
+      + "  \"type\" : \"JSONServiceRecord\",\n"
+      + "  \"description\" : \"httpd-2\",\n"
+      + "  \"external\" : [ ],\n"
+      + "  \"internal\" : [ ],\n"
+      + "  \"yarn:id\" : \"container_e50_1451931954322_0016_01_000003\",\n"
+      + "  \"yarn:persistence\" : \"container\",\n"
+      + "  \"yarn:ip\" : \"172.17.0.20\",\n"
+      + "  \"yarn:hostname\" : \"host2\",\n"
+      + "  \"yarn:component\" : \"httpd\"\n"
       + "}\n";
 
   private static final String CONTAINER_RECORD_NO_IP = "{\n"
       + "  \"type\" : \"JSONServiceRecord\",\n"
-      + "  \"description\" : \"COMP-NAME\",\n"
+      + "  \"description\" : \"httpd-1\",\n"
       + "  \"external\" : [ ],\n"
       + "  \"internal\" : [ ],\n"
       + "  \"yarn:id\" : \"container_e50_1451931954322_0016_01_000002\",\n"
-      + "  \"yarn:persistence\" : \"container\"\n"
+      + "  \"yarn:persistence\" : \"container\",\n"
+      + "  \"yarn:component\" : \"httpd\"\n"
       + "}\n";
 
   private static final String CONTAINER_RECORD_YARN_PERSISTANCE_ABSENT = "{\n"
       + "  \"type\" : \"JSONServiceRecord\",\n"
-      + "  \"description\" : \"COMP-NAME\",\n"
+      + "  \"description\" : \"httpd-1\",\n"
       + "  \"external\" : [ ],\n"
       + "  \"internal\" : [ ],\n"
       + "  \"yarn:id\" : \"container_e50_1451931954322_0016_01_000003\",\n"
       + "  \"yarn:ip\" : \"172.17.0.19\",\n"
-      + "  \"yarn:hostname\" : \"0a134d6329bb\"\n"
+      + "  \"yarn:hostname\" : \"0a134d6329bb\",\n"
+      + "  \"yarn:component\" : \"httpd\""
       + "}\n";
 
   @Before
@@ -229,7 +244,7 @@ public class TestRegistryDNS extends Assert {
     assertEquals("wrong result", "172.17.0.19",
         ((ARecord) recs[0]).getAddress().getHostAddress());
 
-    recs = assertDNSQuery("comp-name.test1.root.dev.test.", 1);
+    recs = assertDNSQuery("httpd-1.test1.root.dev.test.", 1);
     assertTrue("not an ARecord", recs[0] instanceof ARecord);
   }
 
@@ -268,7 +283,7 @@ public class TestRegistryDNS extends Assert {
         ((ARecord) recs[0]).getAddress().getHostAddress());
     assertEquals("wrong ttl", 30L, recs[0].getTTL());
 
-    recs = assertDNSQuery("comp-name.test1.root.dev.test.", 1);
+    recs = assertDNSQuery("httpd-1.test1.root.dev.test.", 1);
     assertTrue("not an ARecord", recs[0] instanceof ARecord);
 
     assertEquals("wrong ttl", 30L, recs[0].getTTL());
@@ -286,7 +301,7 @@ public class TestRegistryDNS extends Assert {
     // start assessing whether correct records are available
     Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
     assertEquals("wrong result",
-        "comp-name.test1.root.dev.test.",
+        "httpd-1.test1.root.dev.test.",
         ((PTRRecord) recs[0]).getTarget().toString());
   }
 
@@ -312,7 +327,7 @@ public class TestRegistryDNS extends Assert {
     // start assessing whether correct records are available
     Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
     assertEquals("wrong result",
-        "comp-name.test1.root.dev.test.",
+        "httpd-1.test1.root.dev.test.",
         ((PTRRecord) recs[0]).getTarget().toString());
   }
 
@@ -490,7 +505,7 @@ public class TestRegistryDNS extends Assert {
     assertEquals("wrong result", "172.17.0.19",
         ((AAAARecord) recs[0]).getAddress().getHostAddress());
 
-    recs = assertDNSQuery("comp-name.test1.root.dev.test.", Type.AAAA, 1);
+    recs = assertDNSQuery("httpd-1.test1.root.dev.test.", Type.AAAA, 1);
     assertTrue("not an ARecord", recs[0] instanceof AAAARecord);
   }
 
@@ -565,13 +580,13 @@ public class TestRegistryDNS extends Assert {
     assertEquals("wrong result", "172.17.0.19",
         ((ARecord) recs[0]).getAddress().getHostAddress());
 
-    recs = assertDNSQuery("comp-name.test1.root.dev.test.", 1);
+    recs = assertDNSQuery("httpd-1.test1.root.dev.test.", 1);
     assertTrue("not an ARecord", recs[0] instanceof ARecord);
 
     // lookup dyanmic reverse records
     recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
     assertEquals("wrong result",
-        "comp-name.test1.root.dev.test.",
+        "httpd-1.test1.root.dev.test.",
         ((PTRRecord) recs[0]).getTarget().toString());
 
     // now lookup static reverse records
@@ -649,6 +664,27 @@ public class TestRegistryDNS extends Assert {
         assertDNSQueryNotNull("mail.yahoo.com.", Type.CNAME);
   }
 
+  @Test
+  public void testMultiARecord() throws Exception {
+    ServiceRecord record = getMarshal().fromBytes("somepath",
+        CONTAINER_RECORD.getBytes());
+    ServiceRecord record2 = getMarshal().fromBytes("somepath",
+        CONTAINER_RECORD2.getBytes());
+    getRegistryDNS().register(
+        "/registry/users/root/services/org-apache-slider/test1/components/"
+            + "ctr-e50-1451931954322-0016-01-000002",
+        record);
+    getRegistryDNS().register(
+        "/registry/users/root/services/org-apache-slider/test1/components/"
+            + "ctr-e50-1451931954322-0016-01-000003",
+        record2);
+
+    // start assessing whether correct records are available
+    Record[] recs =
+        assertDNSQuery("httpd.test1.root.dev.test.", 2);
+    assertTrue("not an ARecord", recs[0] instanceof ARecord);
+    assertTrue("not an ARecord", recs[1] instanceof ARecord);
+  }
   public RegistryDNS getRegistryDNS() {
     return registryDNS;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bc92e30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
index f351e23..7ee16dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
@@ -65,6 +65,18 @@ Note that YARN service framework assigns `COMPONENT_INSTANCE_NAME` for
each cont
 assigned `0` since it is the first and only instance for the `hbasemaster` component. In
case of `regionserver` component, it can have multiple containers
  and so be named as such: `regionserver-0`, `regionserver-1`, `regionserver-2` ... etc
 
+Each YARN service component also has Multi-A Records for container fault tolerance or load
balancing via RegistryDNS.  The naming format is defined as:
+```
+${COMPONENT_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}
+```
+
+For example, a component named www for application app launched by Chuck with 3 containers
will have DNS records that look like:
+```
+www.app.chuck.example.com IN A 123.123.123.1
+www.app.chuck.example.com IN A 123.123.123.1
+www.app.chuck.example.com IN A 123.123.123.1
+```
+
 `Disclaimer`: The DNS implementation is still experimental. It should not be used as a fully-functional
DNS.
 
 
@@ -140,4 +152,4 @@ You can edit the `/etc/resolv.conf` to make your system use the registry
DNS suc
 ```
 nameserver 192.168.154.3
 ```
-Alternatively, if you have a corporate DNS in your organization, you can configure zone forwarding
so that the Registry DNS resolves hostnames for the domain used by the cluster.
\ No newline at end of file
+Alternatively, if you have a corporate DNS in your organization, you can configure zone forwarding
so that the Registry DNS resolves hostnames for the domain used by the cluster.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message