hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [1/6] YARN-913 service registry: YARN-2652 add hadoop-yarn-registry package under hadoop-yarn
Date Wed, 08 Oct 2014 19:54:48 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 7ec65c861 -> 088ae9c5b


http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
new file mode 100644
index 0000000..bb95340
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.secure;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.PathPermissionException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException;
+import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
+import org.apache.hadoop.registry.client.impl.RegistryOperationsClient;
+import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
+import org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions;
+import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
+import org.apache.hadoop.registry.server.services.RegistryAdminService;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.security.auth.login.LoginException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
+
+/**
+ * Verify that the {@link RMRegistryOperationsService} works securely
+ */
+public class TestSecureRMRegistryOperations extends AbstractSecureRegistryTest {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestSecureRMRegistryOperations.class);
+  private Configuration secureConf;
+  private Configuration zkClientConf;
+  private UserGroupInformation zookeeperUGI;
+
+  @Before
+  public void setupTestSecureRMRegistryOperations() throws Exception {
+    startSecureZK();
+    secureConf = new Configuration();
+    secureConf.setBoolean(KEY_REGISTRY_SECURE, true);
+
+    // create client conf containing the ZK quorum
+    zkClientConf = new Configuration(secureZK.getConfig());
+    zkClientConf.setBoolean(KEY_REGISTRY_SECURE, true);
+    assertNotEmpty(zkClientConf.get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM));
+
+    // ZK is in charge
+    secureConf.set(KEY_REGISTRY_SYSTEM_ACCOUNTS, "sasl:zookeeper@");
+    zookeeperUGI = loginUGI(ZOOKEEPER, keytab_zk);
+  }
+
+  @After
+  public void teardownTestSecureRMRegistryOperations() {
+  }
+
+  /**
+   * Create the RM registry operations as the current user
+   * @return the service
+   * @throws LoginException
+   * @throws FileNotFoundException
+   */
+  public RMRegistryOperationsService startRMRegistryOperations() throws
+      LoginException, IOException, InterruptedException {
+    // kerberos
+    secureConf.set(KEY_REGISTRY_CLIENT_AUTH,
+        REGISTRY_CLIENT_AUTH_KERBEROS);
+    secureConf.set(KEY_REGISTRY_CLIENT_JAAS_CONTEXT, ZOOKEEPER_CLIENT_CONTEXT);
+
+    RMRegistryOperationsService registryOperations = zookeeperUGI.doAs(
+        new PrivilegedExceptionAction<RMRegistryOperationsService>() {
+          @Override
+          public RMRegistryOperationsService run() throws Exception {
+            RMRegistryOperationsService operations
+                = new RMRegistryOperationsService("rmregistry", secureZK);
+            addToTeardown(operations);
+            operations.init(secureConf);
+            LOG.info(operations.bindingDiagnosticDetails());
+            operations.start();
+            return operations;
+          }
+        });
+
+    return registryOperations;
+  }
+
+  /**
+   * test that ZK can write as itself
+   * @throws Throwable
+   */
+  @Test
+  public void testZookeeperCanWriteUnderSystem() throws Throwable {
+
+    RMRegistryOperationsService rmRegistryOperations =
+        startRMRegistryOperations();
+    RegistryOperations operations = rmRegistryOperations;
+    operations.mknode(PATH_SYSTEM_SERVICES + "hdfs",
+        false);
+    ZKPathDumper pathDumper = rmRegistryOperations.dumpPath(true);
+    LOG.info(pathDumper.toString());
+  }
+
+  @Test
+  public void testAnonReadAccess() throws Throwable {
+    RMRegistryOperationsService rmRegistryOperations =
+        startRMRegistryOperations();
+    describe(LOG, "testAnonReadAccess");
+    RegistryOperations operations =
+        RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
+    addToTeardown(operations);
+    operations.start();
+
+    assertFalse("RegistrySecurity.isClientSASLEnabled()==true",
+        RegistrySecurity.isClientSASLEnabled());
+    assertFalse("ZooKeeperSaslClient.isEnabled()==true",
+        ZooKeeperSaslClient.isEnabled());
+    operations.list(PATH_SYSTEM_SERVICES);
+  }
+
+  @Test
+  public void testAnonNoWriteAccess() throws Throwable {
+    RMRegistryOperationsService rmRegistryOperations =
+        startRMRegistryOperations();
+    describe(LOG, "testAnonNoWriteAccess");
+    RegistryOperations operations =
+        RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
+    addToTeardown(operations);
+    operations.start();
+
+    String servicePath = PATH_SYSTEM_SERVICES + "hdfs";
+    expectMkNodeFailure(operations, servicePath);
+  }
+
+  @Test
+  public void testAnonNoWriteAccessOffRoot() throws Throwable {
+    RMRegistryOperationsService rmRegistryOperations =
+        startRMRegistryOperations();
+    describe(LOG, "testAnonNoWriteAccessOffRoot");
+    RegistryOperations operations =
+        RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
+    addToTeardown(operations);
+    operations.start();
+    assertFalse("mknode(/)", operations.mknode("/", false));
+    expectMkNodeFailure(operations, "/sub");
+    expectDeleteFailure(operations, PATH_SYSTEM_SERVICES, true);
+  }
+
+  /**
+   * Expect a mknode operation to fail
+   * @param operations operations instance
+   * @param path path
+   * @throws IOException An IO failure other than those permitted
+   */
+  public void expectMkNodeFailure(RegistryOperations operations,
+      String path) throws IOException {
+    try {
+      operations.mknode(path, false);
+      fail("should have failed to create a node under " + path);
+    } catch (PathPermissionException expected) {
+      // expected
+    } catch (NoPathPermissionsException expected) {
+      // expected
+    }
+  }
+
+  /**
+   * Expect a delete operation to fail
+   * @param operations operations instance
+   * @param path path
+   * @param recursive
+   * @throws IOException An IO failure other than those permitted
+   */
+  public void expectDeleteFailure(RegistryOperations operations,
+      String path, boolean recursive) throws IOException {
+    try {
+      operations.delete(path, recursive);
+      fail("should have failed to delete the node " + path);
+    } catch (PathPermissionException expected) {
+      // expected
+    } catch (NoPathPermissionsException expected) {
+      // expected
+    }
+  }
+
+  @Test
+  public void testAlicePathRestrictedAnonAccess() throws Throwable {
+    RMRegistryOperationsService rmRegistryOperations =
+        startRMRegistryOperations();
+    String aliceHome = rmRegistryOperations.initUserRegistry(ALICE);
+    describe(LOG, "Creating anonymous accessor");
+    RegistryOperations anonOperations =
+        RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
+    addToTeardown(anonOperations);
+    anonOperations.start();
+    anonOperations.list(aliceHome);
+    expectMkNodeFailure(anonOperations, aliceHome + "/anon");
+    expectDeleteFailure(anonOperations, aliceHome, true);
+  }
+
+  @Test
+  public void testUserZookeeperHomePathAccess() throws Throwable {
+    RMRegistryOperationsService rmRegistryOperations =
+        startRMRegistryOperations();
+    final String home = rmRegistryOperations.initUserRegistry(ZOOKEEPER);
+    describe(LOG, "Creating ZK client");
+
+    RegistryOperations operations = zookeeperUGI.doAs(
+        new PrivilegedExceptionAction<RegistryOperations>() {
+          @Override
+          public RegistryOperations run() throws Exception {
+            RegistryOperations operations =
+                RegistryOperationsFactory.createKerberosInstance(zkClientConf,
+                    ZOOKEEPER_CLIENT_CONTEXT);
+            addToTeardown(operations);
+            operations.start();
+
+            return operations;
+          }
+        });
+    operations.list(home);
+    String path = home + "/subpath";
+    operations.mknode(path, false);
+    operations.delete(path, true);
+  }
+
+  @Test
+  public void testUserHomedirsPermissionsRestricted() throws Throwable {
+    // test that the /users/$user permissions are restricted
+    RMRegistryOperationsService rmRegistryOperations =
+        startRMRegistryOperations();
+    // create Alice's dir, so it should have an ACL for Alice
+    final String home = rmRegistryOperations.initUserRegistry(ALICE);
+    List<ACL> acls = rmRegistryOperations.zkGetACLS(home);
+    ACL aliceACL = null;
+    for (ACL acl : acls) {
+      LOG.info(RegistrySecurity.aclToString(acl));
+      Id id = acl.getId();
+      if (id.getScheme().equals(ZookeeperConfigOptions.SCHEME_SASL)
+          && id.getId().startsWith(ALICE)) {
+
+        aliceACL = acl;
+        break;
+      }
+    }
+    assertNotNull(aliceACL);
+    assertEquals(RegistryAdminService.USER_HOMEDIR_ACL_PERMISSIONS,
+        aliceACL.getPerms());
+  }
+
+  @Test
+  public void testDigestAccess() throws Throwable {
+    RMRegistryOperationsService registryAdmin =
+        startRMRegistryOperations();
+    String id = "username";
+    String pass = "password";
+    registryAdmin.addWriteAccessor(id, pass);
+    List<ACL> clientAcls = registryAdmin.getClientAcls();
+    LOG.info("Client ACLS=\n{}", RegistrySecurity.aclsToString(clientAcls));
+
+    String base = "/digested";
+    registryAdmin.mknode(base, false);
+    List<ACL> baseACLs = registryAdmin.zkGetACLS(base);
+    String aclset = RegistrySecurity.aclsToString(baseACLs);
+    LOG.info("Base ACLs=\n{}", aclset);
+    ACL found = null;
+    for (ACL acl : baseACLs) {
+      if (ZookeeperConfigOptions.SCHEME_DIGEST.equals(acl.getId().getScheme())) {
+        found = acl;
+        break;
+      }
+    }
+    assertNotNull("Did not find digest entry in ACLs " + aclset, found);
+    zkClientConf.set(KEY_REGISTRY_USER_ACCOUNTS,
+        "sasl:somebody@EXAMPLE.COM, sasl:other");
+    RegistryOperations operations =
+        RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
+            id,
+            pass);
+    addToTeardown(operations);
+    operations.start();
+    RegistryOperationsClient operationsClient =
+        (RegistryOperationsClient) operations;
+    List<ACL> digestClientACLs = operationsClient.getClientAcls();
+    LOG.info("digest client ACLs=\n{}",
+        RegistrySecurity.aclsToString(digestClientACLs));
+    operations.stat(base);
+    operations.mknode(base + "/subdir", false);
+    ZKPathDumper pathDumper = registryAdmin.dumpPath(true);
+    LOG.info(pathDumper.toString());
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testNoDigestAuthMissingId() throws Throwable {
+    RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
+        "",
+        "pass");
+  }
+
+  @Test(expected = ServiceStateException.class)
+  public void testNoDigestAuthMissingId2() throws Throwable {
+    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_DIGEST);
+    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_ID, "");
+    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_PASSWORD, "pass");
+    RegistryOperationsFactory.createInstance("DigestRegistryOperations",
+        zkClientConf);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testNoDigestAuthMissingPass() throws Throwable {
+    RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
+        "id",
+        "");
+  }
+
+  @Test(expected = ServiceStateException.class)
+  public void testNoDigestAuthMissingPass2() throws Throwable {
+    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_DIGEST);
+    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_ID, "id");
+    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_PASSWORD, "");
+    RegistryOperationsFactory.createInstance("DigestRegistryOperations",
+        zkClientConf);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
new file mode 100644
index 0000000..2dad4bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.secure;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
+import org.apache.hadoop.registry.client.impl.zk.CuratorService;
+import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
+import org.apache.zookeeper.CreateMode;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.security.auth.login.LoginContext;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
+
+/**
+ * Verify that the Mini ZK service can be started up securely
+ */
+public class TestSecureRegistry extends AbstractSecureRegistryTest {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestSecureRegistry.class);
+
+  @Before
+  public void beforeTestSecureZKService() throws Throwable {
+      enableKerberosDebugging();
+  }
+
+  @After
+  public void afterTestSecureZKService() throws Throwable {
+    disableKerberosDebugging();
+    RegistrySecurity.clearZKSaslClientProperties();
+  }
+
+  @Test
+  public void testCreateSecureZK() throws Throwable {
+    startSecureZK();
+    secureZK.stop();
+  }
+
+  @Test
+  public void testInsecureClientToZK() throws Throwable {
+    startSecureZK();
+    userZookeeperToCreateRoot();
+    RegistrySecurity.clearZKSaslClientProperties();
+
+    CuratorService curatorService =
+        startCuratorServiceInstance("insecure client", false);
+
+    curatorService.zkList("/");
+    curatorService.zkMkPath("", CreateMode.PERSISTENT, false,
+        RegistrySecurity.WorldReadWriteACL);
+  }
+
+  /**
+   * test that ZK can write as itself
+   * @throws Throwable
+   */
+  @Test
+  public void testZookeeperCanWrite() throws Throwable {
+
+    System.setProperty("curator-log-events", "true");
+    startSecureZK();
+    CuratorService curator = null;
+    LoginContext login = login(ZOOKEEPER_LOCALHOST,
+        ZOOKEEPER_CLIENT_CONTEXT,
+        keytab_zk);
+    try {
+      logLoginDetails(ZOOKEEPER, login);
+      RegistrySecurity.setZKSaslClientProperties(ZOOKEEPER,
+                                                ZOOKEEPER_CLIENT_CONTEXT);
+      curator = startCuratorServiceInstance("ZK", true);
+      LOG.info(curator.toString());
+
+      addToTeardown(curator);
+      curator.zkMkPath("/", CreateMode.PERSISTENT, false,
+          RegistrySecurity.WorldReadWriteACL);
+      curator.zkList("/");
+      curator.zkMkPath("/zookeeper", CreateMode.PERSISTENT, false,
+          RegistrySecurity.WorldReadWriteACL);
+    } finally {
+      logout(login);
+      ServiceOperations.stop(curator);
+    }
+  }
+
+  /**
+   * Start a curator service instance
+   * @param name name
+   * @param secure flag to indicate the cluster is secure
+   * @return an inited and started curator service
+   */
+  protected CuratorService startCuratorServiceInstance(String name,
+      boolean secure) {
+    Configuration clientConf = new Configuration();
+    clientConf.set(KEY_REGISTRY_ZK_ROOT, "/");
+    clientConf.setBoolean(KEY_REGISTRY_SECURE, secure);
+    describe(LOG, "Starting Curator service");
+    CuratorService curatorService = new CuratorService(name, secureZK);
+    curatorService.init(clientConf);
+    curatorService.start();
+    LOG.info("Curator Binding {}",
+        curatorService.bindingDiagnosticDetails());
+    return curatorService;
+  }
+
+  /**
+   * have the ZK user create the root dir.
+   * This logs out the ZK user after and stops its curator instance,
+   * to avoid contamination
+   * @throws Throwable
+   */
+  public void userZookeeperToCreateRoot() throws Throwable {
+
+    System.setProperty("curator-log-events", "true");
+    CuratorService curator = null;
+    LoginContext login = login(ZOOKEEPER_LOCALHOST,
+        ZOOKEEPER_CLIENT_CONTEXT,
+        keytab_zk);
+    try {
+      logLoginDetails(ZOOKEEPER, login);
+      RegistrySecurity.setZKSaslClientProperties(ZOOKEEPER,
+          ZOOKEEPER_CLIENT_CONTEXT);
+      curator = startCuratorServiceInstance("ZK", true);
+      LOG.info(curator.toString());
+
+      addToTeardown(curator);
+      curator.zkMkPath("/", CreateMode.PERSISTENT, false,
+          RegistrySecurity.WorldReadWriteACL);
+      ZKPathDumper pathDumper = curator.dumpPath(true);
+      LOG.info(pathDumper.toString());
+    } finally {
+      logout(login);
+      ServiceOperations.stop(curator);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/log4j.properties
new file mode 100644
index 0000000..629e1d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/log4j.properties
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+# packages under test
+log4j.logger.org.apache.hadoop.yarn.registry=DEBUG
+log4j.logger.org.apache.hadoop.service=DEBUG
+
+log4j.logger.org.apache.hadoop.security.UserGroupInformation=DEBUG
+
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=INFO
+log4j.logger.org.apache.zookeeper.ClientCnxn=DEBUG
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
+log4j.logger.org.apache.hadoop.metrics2=ERROR
+log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
+log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
+log4j.logger.org.apache.hadoop.security.token.delegation=WARN
+log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
+
+# curator noise
+log4j.logger.org.apache.curator.framework.imps=WARN
+log4j.logger.org.apache.curator.framework.state.ConnectionStateManager=ERROR
+
+log4j.logger.org.apache.directory.api.ldap=ERROR
+log4j.logger.org.apache.directory.server=ERROR
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
index 9f07b19..a60e4a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
@@ -58,3 +58,5 @@ MapReduce NextGen aka YARN aka MRv2
   * {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduce_Compatibility_Hadoop1_Hadoop2.html}Backward Compatibility between Apache Hadoop 1.x and 2.x for MapReduce}}
 
   * {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html}Encrypted Shuffle}}
+
+  * {{{./registry/index.html}Registry}}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
new file mode 100644
index 0000000..a9ea24f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
@@ -0,0 +1,28 @@
+<!---
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+# YARN Service Registry
+
+The Service registry is a service which can be deployed in a Hadoop cluster
+to allow deployed applications to register themselves and the means of
+communicating with them. Client applications can then locate services
+and use the binding information to connect with the services's network-accessible
+endpoints, be they REST, IPC, Web UI, Zookeeper quorum+path or some other protocol.
+
+* [Architecture](yarn-registry.html)
+* [Using the YARN Service registry](using-the-yarn-service-registry.html)
+* [Security](registry-security.html)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-security.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-security.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-security.md
new file mode 100644
index 0000000..7278534
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-security.md
@@ -0,0 +1,120 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Registry Security
+
+This document describes how security is implemented in the service registry
+
+In a non-Kerberos-enabled Hadoop cluster, the Registry does not offer any
+security at all: the registry is world writeable.
+
+This document is therefore relevant only to secure clusters.
+
+## Security Model
+
+The security model of the registry is designed to meet the following goals
+a secur
+1. Deliver functional security on e ZK installation.
+1. Allow the RM to create per-user regions of the registration space
+1. Allow applications belonging to a user to write registry entries
+into their part of the space. These may be short-lived or long-lived
+YARN applications,  or they may be be static applications.
+1. Prevent other users from writing into another user's part of the registry.
+1. Allow system services to register to a `/services` section of the registry.
+1. Provide read access to clients of a registry.
+1. Permit future support of DNS
+1. Permit the future support of registering data private to a user.
+This allows a service to publish binding credentials (keys &c) for clients to use.
+1. Not require a ZK keytab on every user's home directory in a YARN cluster.
+This implies that kerberos credentials cannot be used by YARN applications.
+
+
+ZK security uses an ACL model, documented in
+[Zookeeper and SASL](https://cwiki.apache.org/confluence/display/ZOOKEEPER/Zookeeper+and+SASL)
+In which different authentication schemes may be used to restrict access
+to different znodes. This permits the registry to use a mixed
+Kerberos + Private password model.
+
+* The YARN-based registry (the `RMRegistryOperationsService`), uses kerberos
+as the authentication mechanism for YARN itself.
+* The registry configures the base of the registry to be writeable only by
+itself and other hadoop system accounts holding the relevant kerberos credentials.
+* The user specific parts of the tree are also configured to allow the same
+system accounts to write and manipulate that part of the tree.
+* User accounts are created with a `(username,password)` keypair granted
+write access to their part of the tree.
+* The secret part of the keypair is stored in the users' home directory
+on HDFS, using the Hadoop Credentials API.
+* Initially, the entire registry tree will be world readable.
+
+
+What are the limitations of such a scheme?
+
+1. It is critical that the user-specific registry keypair is kept a secret.
+This relies on filesystem security to keep the file readable only
+ by the (authenticated) user.
+1. As the [ZK Documentation says](http://zookeeper.apache.org/doc/r3.4.6/zookeeperProgrammers.html#sc_ZooKeeperAccessControl),
+*" Authentication is done by sending the username:password in clear text"
+1. While it is possible to change the password for an account,
+this involves a recursive walk down the registry tree, and will stop all
+running services from being able to authenticate for write access until they
+reload the key.
+1. A world-readable registry tree is exposing information about the cluster.
+There is some mitigation here in that access may be restricted by IP Address.
+1. There's also the need to propagate information from the registry down to
+the clients for setting up ACLs.
+
+
+
+## ACL Configuration propagation
+
+The registry manager cannot rely on clients consistently setting
+ZK permissions. At the very least, they cannot relay on client applications
+unintentionally wrong values for the accounts of the system services
+
+*Solution*: Initially, a registry permission is used here.
+
+### Automatic domain extension
+
+In a kerberos domain, it is possible for a kerberized client to determine the
+realm of a cluster at run time from the local
+user's kerberos credentials as used to talk to YARN or HDFS.
+
+This can be used to auto-generate account names with the correct realm for the
+system accounts hence aid having valid constants.
+
+This allows the registry to support a default configuration value for
+`hadoop.registry.system.accounts` of:
+
+      "sasl:yarn@, sasl:mapred@, sasl:hdfs@, sasl:hadoop@";
+
+#### In-registry publishing of core binding data
+
+Another strategy could be to have a `ServiceRecord` at the root
+of the registry that actually defines the registry —including listing
+those default binding values in the `data` field..
+
+### Auditing
+
+Something (perhaps the RM) could scan a user's portion of the registry and
+detect some ACL problems: IP/world access too lax, admin account settings wrong.
+It cannot view or fix the ACL permissions unless it has the `ADMIN` permission,
+though that situation can at least be detected. Given the RM must have `DELETE`
+permissions further up the stack, it would be in a position to delete the errant
+part of the tree —though this could be a destructive overreaction.
+
+## Further Reading
+
+* [Zookeeper and SASL](https://cwiki.apache.org/confluence/display/ZOOKEEPER/Zookeeper+and+SASL)
+* [Up and Running with Secure Zookeeper](https://github.com/ekoontz/zookeeper/wiki)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/using-the-yarn-service-registry.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/using-the-yarn-service-registry.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/using-the-yarn-service-registry.md
new file mode 100644
index 0000000..dcae372
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/using-the-yarn-service-registry.md
@@ -0,0 +1,150 @@
+<!---
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+# Using the YARN Service Registry
+
+The YARN service registry can be used in a numbe of ways :-
+
+1. To register dynamic YARN-deployed applications with entries that match the lifespan of the YARN application. Service Records can be set to be deleted on the completion of the YARN application, the application attempt, or an individual container.
+1. To look up static or dynamic applications and the mechanisms to communicate with them. Those mechanisms can incude: HTTP(S) URLs, Zookeeper paths, hostnames and ports and even paths in a Hadoop filesystem to configuration data.
+1. On a secure cluster, to verify that a service binding has been published by a specific user, or a system account. This can be done simply by looking at the path under which an entry has been placed.
+1. To register static applications. These will remain in the registry until deleted. They can be updated as required.
+
+A user of the registry may be both a publisher of entries —Service Records— and a consumer of other services located via their service records. Different parts of a distributed application may also use it for different purposes -central manager services can publish bindings for use by the worker services, which can then look up the bindings to communicate with that manager even if it was restarted on different nodes in the cluster.
+
+The registry cannot be used:-
+
+* To subscribe to service records or registry paths and listen for changes.
+* To directly share arbitrary data from a server for their clients. Such data must be published by some other means, a means which the registry entry can publish.
+* To share secrets between processes. The registry is world readable.
+
+
+## Registry Application Design Patterns
+
+
+### Short-lived YARN Application Masters registering their public service endpoints.
+
+1. A YARN application is deployed. In a secure cluster, it is given the kerberos token to write to the registry.
+2. When launched, it creates a service record at a known path
+3. This record MAY have application attempt persistence policy of and an ID of the application attempt
+
+		yarn:persistence = "application_attempt"
+		yarn:id = ${application_attemptId}
+	This means that the record will be deleted when the application attempt completes, even if a new attempt is created. Every Application attempt will have to re-register the endpoint —which may be needed to locate the service anyway.
+4. Alternatively, the record MAY have the persistence policy of "application":
+
+		yarn:persistence = "application_attempt"
+		yarn:id = application_attemptId
+	This means that the record will persist even between application attempts, albeit with out of date endpoint information.
+5. Client applications look up the service by way of the path.
+
+The choice of path is an application specific one. For services with a yarn application name guaranteed to be unique, we recommend a convention of:
+
+	/users/${username}/applications/${service-class}/${instance-name}
+
+Alternatively, the application Id can be used in the path:
+
+	/users/${username}/applications/${service-class}/${applicationId}
+
+The latter makes mapping a YARN application listing entry to a service record trivial.
+
+Client applications may locate the service
+
+* By enumerating all instances of a service class and selecting one by specific critera.
+* From a supplied service class and instance name
+* If listed by application ID, from the service class and application ID.
+
+After locating a service record, the client may enumerate the `external` bindings and locate the entry with the desired API.
+
+
+### YARN Containers registering their public service endpoints
+
+Here all containers in a YARN application are publishing service endpoints for public consumption.
+
+1. The deployed containers are passed the base path under which they should register themselves.
+2. Long-lived containers must be passed an `id:password` pair which gives them the right to update these entries without the kerberos credentials of the user. This allows the containers to update their entries even after the user tokens granting the AM write access to a registry path expire.
+3. The containers instantiate a registry operations instance with the id:pass pair.
+4. They then a register service record on a path consisting of:
+
+		${base-path} + "/" + RegistryPathUtils.encodeYarnID(containerId)
+
+	This record should have the container persistence policy an ID of the container
+
+		yarn:persistence = "container"
+		yarn:id = containerId
+
+	When the container is terminated, the entry will be automatically deleted.
+
+5. The exported service endpoints of this container-deployed service should be listed in the  `external` endpoint list of the service record.
+6. Clients may enumerate all containers exported by a YARN application by listing the entries under `${base-path}`.
+
+
+### Registering Static cluster services.
+
+Services which are generally fixed in a cluster, but which need to publish binding and configuration information may be published in the registry. Example: an Apache Oozie service.
+Services external to the cluster to which deployed applications may also be published. Example: An Amazon Dynamo instance.
+
+
+These services can be registered under paths which belong to the users running the service, such as `/users/oozie` or `/users/hbase`. Client applications would use this path. While this can authenticate the validity of the service record, it does rely on the client applications knowing the username a service is deployed on, or being configured with the full path.
+
+The alternative is for the services to be deployed under a static services path, under `/services`. For example, `/services/oozie` could contain the registration of the Oozie service. As the permissions for this path are restricted to pre-configured system accounts, the presence of a service registration does, on a secure cluster, confirm that it was registered by the cluster administration tools.
+
+1. The service is deployed by some management tool, or directly by the cluster operator.
+2. The deployed application can register itself under its own user name if given the binding information for the registry.
+3. If the application is to be registered under `/services` and it has been deployed by one of the system user accounts —it may register itself directly.
+4. If the application does not have the permissions to do so, the cluster administration tools must register the service instead.
+5. Client applications may locate a service by resolving its well known/configured path.
+5. If a service is stopped, the administration tools may delete the entry, or retain the entry but delete all it service endpoints. This is a proposed convention to indicate "the service is known but not currently reachable".
+6. When a service is restarted, its binding information may be updated, or its entire registry entry recreated.
+
+
+### YARN containers locating their Application Master
+
+Here YARN containers register with their AM to receive work, usually by some heartbeat mechanism where they report in regularly. If the AM is configured for containers to outlive the application attempt, when an AM fails the containers keep running. These containers will need to bind to any restarted AM. They may also wish to conclude that if an AM does not restart, that they should eventually time out and terminate themselves. Such a policy helps the application react to network partitions.
+
+1. The YARN AM publishes its service endpoints such as the FQDN and socket port neede for IPC communications, or an HTTP/HTTPS URL needed for a REST channel. These are published in the `internal` endpoint list, with the specific API the containers use
+2. The YARN containers are launched with the path to the service record (somehow) passed to them. Environment variables or command line parameters are two viable mechanisms. Shared secrets may also be passed this way, so allowing the containers to authenticate themselves with the AM. An alternative means of publishing secrets is saving information to the filesystem in a path specified as one of the endpoints.
+2. The YARN containers look up the service registry to identify the communications binding.
+3. If the registered service entry cannot be found, the container MAY do one of: exit. spin with some (jittered) retry period, polling for the entry, until the entry reappears. This implies that the AM has been found.
+4. If the service entry is found, the client should attempt to communicate with the AM on its channel. Shared authentication details may be used to validate the client with the server and vice versa.
+5. The client report in to the AM until the connections start failing to connect or authenticate, or when a long lived connection is broken and cannot be restarted.
+6. A this point the client may revert to step (3). Again, some backoff policy with some jitter helps stop a newly-restarted AM being overloaded. Containers may also with to have some timeout after which they conclude that the AM is not coming back and exit.
+6. We recommend that alongside the functional commands that an AM may issue to a client, a "terminate" command can be issued to a container. This allows the system to handle the specific situation of the YARN Node Manager terminating while spawned containers keep running.
+
+### YARN Applications and containers publishing their management and metrics bindings
+
+Management ports and bindings are simply another endpoint to publish. These should be published as *internal* endpoints, as they are not intended for public consumption. By convention, the name of the management protocol shoudl be used as the endpoint's declared API: `JMX`, `ganglia`, etc.
+
+
+### Client application enumerating services by endpoint APIs
+
+A client application wishes to locate all services implementing a specific API, such as `"org.apache.hbase"`
+
+1. The client starts from a path in the registry
+1. The client calls `registryOperations.list(path)` to list all nodes directly under that path, getting a relative list of child nodes.
+1. the client enumerates the child record statuses by calling `stat()` on each child.
+1. For all status entries, if the size of the entry is > the value of `ServiceRecordHeader.getLength()`, it MAY contain a service record.
+1. The contents can be retrieved using the `resolve()` operation. If successful, it does contain a service record —so the client can enumerate the `external` endpoints and locate the one with the desired API.
+1. The `children` field of each `RegistryPathStatus` status entry should be examined. If it is >= 0, the enumeration should be performed recursively on the path of that entry.
+1. The operation ultimately completes with a list of all entries.
+
+This algorithm describes a depth first search of the registry tree. Variations are of course possible, including breadth first search, or immediately halting the search as soon as a single entry point. There is also the option of parallel searches of different subtrees —which may reduce search time, albeit at the price of a higher client load on the registry infrastructure.
+
+A Utility class `RegistryUtils` provides static utility methods for common registry operations,
+in particular, `RegistryUtils.listServiceRecords(registryOperations, path)`
+performs the listing and collection of all immediate child record entries of
+a specified path.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
new file mode 100644
index 0000000..a2a5009
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
@@ -0,0 +1,889 @@
+<!---
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+## The YARN Service Registry
+
+# Introduction and concepts
+
+This document describes a YARN service registry built to address a problem:
+*how can clients talk to YARN-deployed services and the components which form
+such services?*
+
+Service registration and discovery is a long-standing problem in distributed
+computing, dating back to Xerox's Grapevine Service. This proposal is for a
+registry for locating distributed applications deployed by YARN, and
+determining the binding information needed to communicate with these
+applications.
+
+### Definitions
+**Service**: a potentially-distributed application deployed in —or reachable
+from— a Hadoop YARN cluster. Examples: Apache HBase, Apache hcatalog, Apache
+Storm. Services may be short-lived or long-lived.
+
+**Service Class:** the name of a type of service, used as a path in a registry
+and matching the DNS-compatible path naming scheme. Examples:
+`org-apache-hbase`, `org-apache-hcatalog`
+
+**Component**: a distributed element of a service. Examples: HBase master
+nodes, HBase region servers and HBase REST servers.
+
+**Service Instance:** A single instance of an application. Example, an HBase
+cluster `demo1`. A service instance is running if the instances the components
+which for the service are running. This does not imply "live" in the
+distributed computing sense, merely that the process are running.
+
+**Component Instance**: a single instance of a component within a service
+instance. Examples: an HBase master node on host `rack1server6` or a region
+server on host `rack3server40`.
+
+**Endpoint**: one means of binding with a service instance or a component
+instance. Examples: HBase's Apache Zookeeper binding, a Java JMX port on a
+region server, a Web UI on an HBase Master, and the REST API of an HBase REST
+component instance. Endpoints may be *internal* —for use within the service
+instance, or *external*: for use by clients of a service instance.
+
+**Service Record**: a record in the registry describing a service instance or a
+component instance, including listing its endpoints.
+
+**YARN Resource Manager, "RM":** the YARN component which allows client
+applications to submit work (including requests to deploy service instances) to
+a YARN cluster. The RM retains state on all running applications.
+
+**YARN Application**: An application deployed via YARN. Every application
+instance has a unique application ID.
+
+**YARN Application Master, "AM":** the application-specific component which is
+scheduled and deployed by the RM. It has the duty of maintaining the internal
+state of the application, including requesting and managing all other component
+instances of this application instance. The YARN RM will detect the failure of
+the AM, and respond by rescheduling it.
+
+**YARN Container:** An allocation of resources, including CPU and RAM, for a
+component instance. The AM has the responsibility of requesting the containers
+its components need, and building the commands to instantiate its component
+instances onto allocated containers. Every allocated container has a unique
+container ID.
+
+## The binding problem
+Hadoop YARN allows applications to run on the Hadoop cluster. Some of these are
+batch jobs or queries that can managed via Yarn’s existing API using its
+application ID. In addition YARN can deploy ong-lived services instances such a
+pool of Apache Tomcat web servers or an Apache HBase cluster. YARN will deploy
+them across the cluster depending on the individual each component requirements
+and server availability. These service instances need to be discovered by
+clients; traditionally their IP added is registered in DNS or in some
+configuration file —but that is not feasible in YARN-deployed applications when
+neither the hostname nor network ports can be known in advance.
+
+As a result there is no easy way for clients to interact with dynamically
+deployed applications.
+
+YARN supports a rudimentary registry which allows YARN Application Masters to
+register a web URL and an IPC address. but is not sufficient for our purposes
+since it It does not allow any other *endpoints* to be registered —such as REST
+URLs, or zookeeper path or the endpoints of the tasks that the Application
+Master executes. Further, information that can be registered is mapped to the
+YARN application instance —a unique instance ID that changes whenever a YARN
+application is started. This makes it impossible to resolve binding information
+via a static reference to a named service, or to even probe for the existence
+of a service instance which is not currently live.
+
+# Use Cases
+
+## Service Name Examples
+
+
+Core Hadoop services.
+
+These may be deployed statically, dynamically via an account with the
+permissions to write to the `/services` path, or even registrations of remote
+services accessible from within the Hadoop cluster
+
+        /services/hdfs
+        /services/yarn
+        /services/oozie
+
+Yarn-deployed services belonging to individual users.
+
+        /users/joe/org-apache-hbase/demo1
+        /users/joe/org-apache-hbase/demo1/components/regionserver1
+
+## Registration Use Cases
+
+1. A Hadoop core service that is not running under YARN example: HDFS) can be
+registered in for discovery. This could be done by the service or by management
+tools..
+
+2. A long-lived application deployed by YARN registers itself for discovery by
+clients. The registration data is intended to outlive the application master,
+and perhaps the lifespan of a single deployment of the service instance.
+
+3. Component instances of a service register themselves, publishing internal
+binding information, such as JMX ports.
+
+4. A YARN-deployed application can bind to dependent service instances both
+static and dynamic. Example: a Tomcat web pool binding to the dynamic HBase
+service instance "/users/joe/services/hbase/demo1".
+
+5. Component Instances use the registry to bind to an internal endpoint of
+their application master, to which they heartbeat regularly.
+
+## Unsupported Registration use cases:
+
+1. A short-lived Yarn application is registered automatically in the registry,
+including all its containers. and unregistered when the job terminates.
+Short-lived applications with many containers will place excessive load on a
+registry. All YARN applications will be given the option of registering, but it
+will not be automatic —and application authors must be advised against
+registering short-lived containers.
+
+## Lookup Use Cases
+
+1. A client application looks up a dynamically deployed service instance whose
+user, service class and instance name is known, e.g.
+`/users/joe/services/hbase/demo1`, and retrieves the information needed to
+connect to the service
+
+2. A client application looks up a statically deployed Hadoop service Example:
+`/services/hdfs`.
+
+3. An Application Master enumerates all registered component instances,
+discovers their listed JMX ports, and, initializes own web UI, offers links to these
+endpoints.
+
+4. A user connects to a private HBase service instance at
+`/users/joe/services/hbase/demo1`.
+
+5. A user connects to the cluster’s HBase service at `/services/hbase`.
+
+6. A user looks up the binding information to a remote Hadoop cluster's
+filesystem at `/net/cluster4/services/hdfs`. The registration information
+includes the `webhdfs://` URL for the remote filesystem.
+
+7. A user lists their HBase service instances:
+
+        ls /users/joe/services/hbase
+
+8. User finds all Hbase services in the cluster:
+
+        find -endpointField.api=org.apache.hbase
+
+9. Possibly in future: looking up a service via DNS.
+
+This registry proposal is intended to support these use cases by providing a
+means for applications to register their service endpoints, and for clients to
+locate them.
+
+# Key Requirements of a Service Registry
+
+Allow dynamic registration of service instances
+
+  * YARN deployed services instances must be able register their bindings and be
+  discovered by clients.
+
+  * Core Hadoop service instances must be able to register their service
+  endpoints.
+
+  * The binding must be upgradable if the service moves or in case if HA fails
+  over.
+
+  * A service instance must be able to publish a variety of endpoints for a
+  service: Web UI, RPC, REST, Zookeeper, others. Furthermore one must also be
+  able register certificates and other public security information may be
+  published as part of a binding.
+
+Registry service properties:
+
+  * The registry must be highly available.
+
+  * Scale: many services and many clients in a large cluster. This will limit
+  how much data a service can publish.
+
+  * Ubiquity: we need this in every YARN cluster, whether physical, virtual or
+  in-cloud.
+
+  * Must support hierarchical namespace and names. The name convention must
+  match that of DNS so that we have the option of accessing the namespace via
+  DNS protocol at a later phase of the project.
+
+  * Registry API Language/Protocols
+
+  * Cross-language: independent of any language; client language != service
+
+  * REST API for reading registry data
+
+Access Control:
+
+  * Read access for all
+
+  * Write is restricted so that squatting and impersonation can be avoided.
+
+Remote accessibility: supports remote access even on clusters which are
+  only reachable via Apache Knox, or hosted in cloud environments.
+
+## Non-Requirements
+
+* The registry is not intended for liveness detection, leader-election or
+  perform other "shared consensual state" actions for an application itself,
+  other than potentially sharing binding information between component
+  instances.
+
+* The registry is not intended to be a store for arbitrary application state,
+  or for publishing configuration data other than binding information to
+  endpoints offered by a service and its components. Such use would overload
+  the registry and rapidly reach limits of what Zookeeper permits.
+
+# Architecture
+
+We propose a base registry service that binds string-names to records
+describing service and component instances. We plan to use ZK as the base name
+service since it supports many of the properties, We pick a part of the ZK
+namespace to be the root of the service registry ( default: `yarnRegistry`).
+
+On top this base implementation we build our registry service API and the
+naming conventions that Yarn will use for its services. The registry will be
+accessed by the registry API, not directly via ZK - ZK is just an
+implementation choice (although unlikely to change in the future).
+
+1. Services are registered by binding a **_path_** to a value called a
+**_Service Record_**. Paths are hierarchical and use `/` as the root as well as
+the separator.
+
+2. Service records are registered as persistent znodes. This ensures that the
+record remains present during planned and unplanned outages of the service, on
+the assumption that client code is resilient to transient outages.
+
+3. Each service instance’s service record lists the endpoints for its various
+protocols exported by that service instance.
+
+4. For each protocol endpoint it must contain
+
+     1. The *protocol* name including: Web, REST, IPC, zookeeper. (type:string)
+
+     2. Its *address*: the specific details used to locate this endpoint
+
+     3. Its *addressType*. This is the format of the binding string. (URL, ZK path,
+        hostname:port pair). For the predefined protocols, we will define what
+        format the binding string MUST be. Example: `protocol==REST` means binding
+        type is `URL`, `protocol==IPC` binding uses the addresstype `host/port`.
+
+     4. The *api*. This is the API offered by the endpoint, and is application
+        specific. examples: `org.apache.hadoop.namenode`,
+        `org.apache.hadoop.webhdfs`
+
+5. Endpoints may be *external* —for use by programs other than the service
+itself, and *internal* —for connecting components within the service instance.
+They will be listed in different sections of the Service Record to distinguish
+them.
+
+6. Core services will be registered using the following convention:
+`/services/{servicename}` e.g. `/services/hdfs`.
+
+7. Yarn services SHOULD be registered using the following convention:
+
+        /users/{username}/{serviceclass}/{instancename}
+
+6. Component instances SHOULD be registered under
+
+        /users/{username}/{serviceclass}/{instancename}/components/{componentname}
+
+
+8. Each of the user's services which follows this convention must have unique
+service class names,
+
+9. Each component instance must have a name that is unique for that service
+instance. For a YARN-deployed application, this can be trivially
+derived from the container ID.
+
+The requirements for unique names ensures that the path to a service instance
+or component instance is guaranteed to be unique, and that all instances of a
+specific service class can be enumerated by listing all children of the service
+class path.
+
+
+# Registry Model
+
+Service entries MUST be persistent —it is the responsibility of YARN and other
+tools to determine when a service entry is to be deleted.
+
+## Path Elements
+
+All path elements MUST match that of a lower-case entry in a hostname path as
+defined in RFC1123; the regular expression is:
+
+    ([a-z0-9]|([a-z0-9][a-z0-9\-]*[a-z0-9]))
+
+This policy will ensure that were the registry hierarchy ever to exported by a
+DNS service, all service classes and names would be valid.
+
+A complication arises with user names, as platforms may allow user names with
+spaces, high unicode and other characters in them. Such paths must be converted
+to valid DNS hostname entries using the punycode convention used for
+internationalized DNS.
+
+## Service Record
+
+A Service Record has some basic information and possibly empty lists of
+internal and external endpoints.
+
+### Service Record:
+
+A Service Record contains some basic informations and two lists of endpoints:
+one list for users of a service, one list for internal use within the
+application.
+
+
+    <table>
+      <tr>
+        <td>Name</td>
+        <td>Description</td>
+      </tr>
+      <tr>
+        <td>description: String</td>
+        <td>Human-readable description.</td>
+      </tr>
+      <tr>
+        <td>external: List[Endpoint]</td>
+        <td>A list of service endpoints for external callers.</td>
+      </tr>
+      <tr>
+        <td>internal: List[Endpoint]</td>
+        <td>A list of service endpoints for internal use within the service instance.</td>
+      </tr>
+    </table>
+
+### YARN Persistence policies
+
+The YARN Resource Manager integration integrates cleanup of service records
+as an application, attempt or container is completed.
+
+This allows service to register entries which have a lifespan bound to one of
+these aspects of YARN applications' lifecycles. This is a feature which is only
+supported when the RM has enabled its support, and would not apply to
+any use of the registry without the RM's participation.
+
+The attributes, `yarn:id` and `yarn:persistence` specify which records
+*and any child entries* may be deleted as the associated YARN components complete.
+
+
+The `yarn:id` field defines the application, attempt or container ID to match;
+the `yarn:persistence` attribute defines the trigger for record cleanup, and
+implicitly the type of the contents of the `yarn:id` field.
+
+These attributes use the prefix "`yarn:`" to indicate that their reliance on
+the YARN layer of the Hadoop cluster to implement the policy. If the registry
+were to run standalone —which is entirely possible— all records would be
+implicitly persistent.
+
+<table>
+  <tr>
+    <td>Name</td>
+    <td>Description</td>
+    <td>contents of `yarn:id` field</td>
+  </tr>
+  <tr>
+    <td>permanent</td>
+    <td>The record persists until removed manually.</td>
+    <td>(unused)</td>
+  </tr>
+  <tr>
+    <td>application</td>
+    <td>Remove when the YARN application defined in the id field terminates.</td>
+    <td>application ID</td>
+  </tr>
+  <tr>
+    <td>application-attempt</td>
+    <td>Remove when the current YARN application attempt finishes.</td>
+    <td>application attempt ID</td>
+  </tr>
+  <tr>
+    <td>container</td>
+    <td>Remove when the YARN container in the ID field finishes</td>
+    <td>container ID</td>
+  </tr>
+</table>
+
+
+The policies which clean up when an application, application attempt or
+container terminates require the `yarn:id` field to match that of the
+application, attempt or container. If the wrong ID is set, the cleanup does not
+take place —and if set to a different application or container, will be cleaned
+up according the lifecycle of that application.
+
+### Endpoint:
+
+<table>
+  <tr>
+    <td>Name</td>
+    <td>Description</td>
+  </tr>
+  <tr>
+    <td>addresses: List[List[String]]</td>
+    <td>a list of address tuples whose format depends on the address type</td>
+  </tr>
+  <tr>
+    <td>addressType: String</td>
+    <td>format of the binding</td>
+  </tr>
+  <tr>
+    <td>protocol: String</td>
+    <td>Protocol. Examples:
+`http`, `https`, `hadoop-rpc`, `zookeeper`, `web`, `REST`, `SOAP`, ...</td>
+  </tr>
+  <tr>
+    <td>api: String</td>
+    <td>API implemented at the end of the binding</td>
+  </tr>
+</table>
+
+
+All string fields have a limit on size, to dissuade services from hiding
+complex JSON structures in the text description.
+
+### Field: Address Type
+
+The addressType field defines the string format of entries.
+
+Having separate types is that tools (such as a web viewer) can process binding
+strings without having to recognize the protocol.
+
+<table>
+  <tr>
+    <td>Format</td>
+    <td>binding format</td>
+  </tr>
+  <tr>
+    <td>`url`</td>
+    <td>`[URL]`</td>
+  </tr>
+  <tr>
+    <td>`hostname`</td>
+    <td>`[hostname]`</td>
+  </tr>
+  <tr>
+    <td>`inetaddress`</td>
+    <td>`[hostname, port]`</td>
+  </tr>
+  <tr>
+    <td>`path`</td>
+    <td>`[/path/to/something]`</td>
+  </tr>
+  <tr>
+    <td>`zookeeper`</td>
+    <td>`[quorum-entry, path]`</td>
+  </tr>
+</table>
+
+
+An actual zookeeper binding consists of a list of `hostname:port` bindings –the
+quorum— and the path within. In the proposed schema, every quorum entry will be
+listed as a triple of `[hostname, port, path]`. Client applications do not
+expect the path to de be different across the quorum. The first entry in the
+list of quorum hosts MUST define the path to be used by all clients. Later
+entries SHOULD list the same path, though clients MUST ignore these.
+
+New Address types may be defined; if not standard please prefix with the
+character sequence `"x-"`.
+
+#### **Field: API**
+
+APIs may be unique to a service class, or may be common across by service
+classes. They MUST be given unique names. These MAY be based on service
+packages but MAY be derived from other naming schemes:
+
+### Examples of Service Entries
+
+Here is an example of a service entry for a YARN-deployed tomcat application.
+
+After creation and registration of the application, the registry looks as follows:
+
+    /users
+      /devteam
+       /org-apache-tomcat
+         /test1
+           /components
+             /container-1408631738011-0001-01-000002
+             /container-1408631738011-0001-01-000001
+
+The `/users/devteam/org-apache-tomcat/tomcat-test` service record describes the
+overall application. It exports the URL to a load balancer.
+
+    {
+      "description" : "tomcat-based web application",
+      "registrationTime" : 1408638082444,
+      "external" : [ {
+        "api" : "www",
+        "addressType" : "uri",
+        "protocolType" : "REST",
+        "addresses" : [ [ "http://loadbalancer/" ] [ "http://loadbalancer2/" ] ]
+      } ],
+      "internal" : [ ]
+    }
+
+The service instance is built from two component instances, each described with
+their container ID converted into a DNS-compatible hostname. The entries are
+marked as ephemeral. If the entries were set within the container, then when
+that container is released or if the component fails, the entries will be
+automatically removed. Accordingly, it's persistence policy is declared to be
+"3", container. The `yarn:id` field identifies the container whose completion
+will trigger the deletion of this entry
+
+    /users/devteam/org-apache-tomcat/test1/components/container-1408631738011-0001-01-000001
+
+    {
+      "registrationTime" : 1408638082445,
+      "yarn:id" : "container_1408631738011_0001_01_000001",
+      "yarn:persistence" : "3",
+      "description" : null,
+      "external" : [ {
+        "api" : "www",
+        "addressType" : "uri",
+        "protocolType" : "REST",
+        "addresses" : [ [ "http://rack4server3:43572" ] ]
+      } ],
+      "internal" : [ {
+        "api" : "jmx",
+        "addressType" : "host/port",
+        "protocolType" : "JMX",
+        "addresses" : [ [ "rack4server3", "43573" ] ]
+      } ]
+    }
+
+The component instances list their endpoints: the public REST API as an
+external endpoint, the JMX addresses as internal.
+
+    /users/devteam/org-apache-tomcat/test1/components/container-1408631738011-0001-01-000002
+
+    {
+      "registrationTime" : 1408638082445,
+      "yarn:id" : "container_1408631738011_0001_01_000002",
+      "yarn:persistence" : "3",
+      "description" : null,
+      "external" : [ {
+        "api" : "www",
+        "addressType" : "uri",
+        "protocolType" : "REST",
+        "addresses" : [ [ "http://rack1server28:35881" ] ]
+      } ],
+      "internal" : [ {
+        "api" : "jmx",
+        "addressType" : "host/port",
+        "protocolType" : "JMX",
+        "addresses" : [ [ "rack1server28", "35882" ] ]
+      } ]
+    }
+
+This information could be used by the (hypothetical) load balancer to enumerate
+the components and build a list of component instances to dispatch requests to.
+Similarly, a management application could enumerate all available component
+instances and their JMX ports, then connect to each to collect performance
+metrics.
+
+# Registry API
+
+Here is the registry API as seen from a Java application. The API is a thin
+layer above the ZK operations, essentially building up paths, reading, writing
+and updating entries, and enumerating children. The REST API is implemented
+inside a server and use this same API to implement its REST API.
+
+The exceptions that are listed are only a subset of possible exception —the
+interface merely lists those that have special meaning.
+
+All write operations must assume that they are communicating with a registry
+service with the consistency view of a Zookeeper client; read-only clients must
+assume that their view may be somewhat out of date.
+
+All clients must recognize that the registry is a shared resource and that
+it may change during a sequence of actions.
+
+### Registry Operations
+
+    public interface RegistryOperations extends Service {
+
+      /**
+       * Create a path.
+       *
+       * It is not an error if the path exists already, be it empty or not.
+       *
+       * The createParents flag also requests creating the parents.
+       * As entries in the registry can hold data while still having
+       * child entries, it is not an error if any of the parent path
+       * elements have service records.
+       *
+       * @param path path to create
+       * @param createParents also create the parents.
+       * @throws PathNotFoundException parent path is not in the registry.
+       * @throws InvalidPathnameException path name is invalid.
+       * @throws IOException Any other IO Exception.
+       * @return true if the path was created, false if it existed.
+       */
+      boolean mknode(String path, boolean createParents)
+          throws PathNotFoundException,
+          InvalidPathnameException,
+          IOException;
+
+      /**
+       * Set a service record to an entry
+       * @param path path to service record
+       * @param record service record service record to create/update
+       * @param createFlags creation flags
+       * @throws PathNotFoundException the parent path does not exist
+       * @throws FileAlreadyExistsException path exists but create flags
+       * do not include "overwrite"
+       * @throws InvalidPathnameException path name is invalid.
+       * @throws IOException Any other IO Exception.
+       */
+      void bind(String path, ServiceRecord record, int createFlags)
+          throws PathNotFoundException,
+          FileAlreadyExistsException,
+          InvalidPathnameException,
+          IOException;
+
+      /**
+       * Resolve the record at a path
+       * @param path path to service record
+       * @return the record
+       * @throws PathNotFoundException path is not in the registry.
+       * @throws InvalidPathnameException the path is invalid.
+       * @throws IOException Any other IO Exception
+       */
+
+      ServiceRecord resolve(String path) throws PathNotFoundException,
+          InvalidPathnameException,
+          IOException;
+
+      /**
+       * Get the status of a path
+       * @param path path to query
+       * @return the status of the path
+       * @throws PathNotFoundException path is not in the registry.
+       * @throws InvalidPathnameException the path is invalid.
+       * @throws IOException Any other IO Exception
+       */
+      RegistryPathStatus stat(String path)
+          throws PathNotFoundException,
+          InvalidPathnameException,
+          IOException;
+
+      /**
+       * Probe for a path existing.
+       * This is equivalent to {@link #stat(String)} with
+       * any failure downgraded to a
+       * @param path path to query
+       * @return true if the path was found
+       * @throws IOException
+       */
+      boolean exists(String path) throws IOException;
+
+     /**
+       * List all entries under a registry path
+       * @param path path to query
+       * @return a possibly empty list of the full path names of
+       * child entries
+       * @throws PathNotFoundException
+       * @throws InvalidPathnameException
+       * @throws IOException
+       */
+       List<String> list(String path) throws
+          PathNotFoundException,
+          InvalidPathnameException,
+          IOException;
+
+      /**
+       * Delete a path.
+       *
+       * If the operation returns without an error then the entry has been
+       * deleted.
+       * @param path path delete recursively
+       * @param recursive recursive flag
+       * @throws PathNotFoundException path is not in the registry.
+       * @throws InvalidPathnameException the path is invalid.
+       * @throws PathIsNotEmptyDirectoryException path has child entries, but
+       * recursive is false.
+       * @throws IOException Any other IO Exception
+       *
+       */
+      void delete(String path, boolean recursive)
+          throws PathNotFoundException,
+          PathIsNotEmptyDirectoryException,
+          InvalidPathnameException,
+          IOException;
+
+      /**
+       * Add a new write access entry to be added to node permissions in all
+       * future write operations of a session connected to a secure registry.
+       *
+       * This does not grant the session any more rights: if it lacked any write
+       * access, it will still be unable to manipulate the registry.
+       *
+       * In an insecure cluster, this operation has no effect.
+       * @param id ID to use
+       * @param pass password
+       * @return true if the accessor was added: that is, the registry connection
+       * uses permissions to manage access
+       * @throws IOException on any failure to build the digest
+       */
+      boolean addWriteAccessor(String id, String pass) throws IOException;
+
+      /**
+       * Clear all write accessors.
+       *
+       * At this point all standard permissions/ACLs are retained,
+       * including any set on behalf of the user
+       * Only  accessors added via {@link #addWriteAccessor(String, String)}
+       * are removed.
+       */
+      public void clearWriteAccessors();
+    }
+
+
+### `RegistryPathStatus`
+
+The `RegistryPathStatus` class summarizes the contents of a node in the registry.
+
+    public class RegistryPathStatus {
+
+      /**
+       * Short path in the registry to this entry
+       */
+      public String path;
+
+      /**
+       * Timestamp
+       */
+      public long time;
+
+      /**
+       * Entry size in bytes, as returned by the storage infrastructure.
+       * In zookeeper, even "empty" nodes have a non-zero size.
+       */
+      public long size;
+
+      /**
+       * Number of child nodes
+       */
+      public int children;
+    }
+
+
+## Security
+
+The registry will allow a service instance can only be registered under the
+path where it has permissions. Yarn will create directories with appropriate
+permissions for users where Yarn deployed services can be registered by a user.
+of the user account of the service instance. The admin will also create
+directories (such as `/services`) with appropriate permissions (where core Hadoop
+services can register themselves.
+
+There will no attempt to restrict read access to registry information. The
+services will protect inappropriate access by clients by requiring
+authentication and authorization. There is a *scope* field in a service record
+, but this is just a marker to say "internal API only", rather than a direct
+security restriction. (this is why "internal" and "external" are proposed, not
+"public" and "private").
+
+Rationale: the endpoints being registered would be discoverable through port
+scanning anyway. Having everything world-readable allows the REST API to have a
+simpler access model —and is consistent with DNS.
+
+On a secure cluster, ZK token renewal may become an issue for long-lived
+services —if their token expires their session may expire. Renewal of such
+tokens is not part of the API implementation —we may need to add a means to
+update the tokens of an instance of the registry operations class.
+
+### Security Policy Summary
+
+In an a non-Kerberos Zookeeper Cluster, no security policy is implemented.
+
+The registry is designed to be secured *on a kerberos-managed cluster*.
+
+* The registry root grants full rights to "system accounts":
+`mapred`, `hdfs`, `yarn` : `"rwcda"`; all other accounts, and anonymous access
+is read-only.
+
+* The permissions are similarly restricted for `/users`, and `/services/`
+
+* installations may extend or change these system accounts.
+
+* When an application belonging to a user is scheduled, YARN
+SHALL create an entry for that user `/users/${username}`.
+
+* This node will have full access to the system; the user the access rights:
+`"crd"`. That is, they may create or delete child nodes, but not write to
+their home node, —or alter its permissions.
+
+* Applications wishing to write to the registry must use a SASL connection
+to authenticate via Zookeeper,
+
+* Applications creating nodes in the user path MUST include the site-specified
+system accounts in the ACL list, with full access.
+
+* Applications creating nodes in the user path MUST include an ACL Which
+
+* Applications creating nodes in the user path MUST declare their own
+user identity as a `sasl:user@REALM` entry.
+
+* Applications creating nodes the user path MAY add extra `digest:` ACL tokens
+so as to give their services the ability to manipulate portions of the
+registry *without needing kerberos credentials*.
+
+The digest-driven authentication avoid the problem of credential renewal in
+long-lived applications. An YARN application may be passed the token to
+connect with the ZK service when launched. It can then create or update an
+entry, including a secret digest ACL in the permissions of nodes it creates.
+As a result, even after the credentials expire, it retains *some* access.
+
+Note that for this to be successful, the client will need to fall back
+session to *not* use SASL, instead using authentication id:pass credentials.
+
+
+## Out of cluster and cross-cluster access
+
+1. A client should be able to access the registry of another cluster in order
+to access services of that cluster. Detail of this need to further fleshed out.
+
+2. Firewall services such as Apache Knox can examine the internal set of
+published services, and publish a subset of their endpoints. They MAY implement
+a future REST API.
+
+# Limits
+
+**Entry Size**
+
+Zookeeper has a default limit of 1MB/node. If all endpoints of a service or
+component are stored in JSON attached to that node, then there is a total limit
+of 1MB of all endpoint registration data.
+
+To prevent this becoming a problem, the client API should implement strict
+limits on the maximum length of fields, with low limits on the addressType,
+protocol, and api fields, something longer on the description and addresses
+elements —along with a limit on the number of elements in the addresses field.
+
+**Name size**
+
+To support DNS in future, there must be a limit of 63 bytes on all path
+elements. For non-ASCII User names, this restriction implies that a shorter
+path may be a limit.
+
+**Rate of Update**
+
+A rapid rate of entry change is considered antisocial in a ZK cluster.
+Implementations may throttle update operations.
+
+**Rate of Polling**
+
+Clients which poll the registry may be throttled.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 13553d5..52049cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -100,5 +100,6 @@
     <module>hadoop-yarn-applications</module>
     <module>hadoop-yarn-site</module>
     <module>hadoop-yarn-client</module>
+    <module>hadoop-yarn-registry</module>
   </modules>
 </project>


Mime
View raw message