hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [32/50] [abbrv] hadoop git commit: YARN-6669. Implemented Kerberos security for YARN service framework. (Contributed by Jian He)
Date Thu, 07 Dec 2017 00:54:17 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
index e18bcae..173001b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
@@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.service.exceptions.SliderException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
@@ -43,9 +45,7 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
-import java.net.ServerSocket;
-import java.net.URL;
-import java.net.URLDecoder;
+import java.net.*;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -57,6 +57,11 @@ import java.util.Map;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPOutputStream;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic
+    .HADOOP_SECURITY_DNS_INTERFACE_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic
+    .HADOOP_SECURITY_DNS_NAMESERVER_KEY;
+
 /**
  * These are slider-specific Util methods
  */
@@ -542,4 +547,24 @@ public final class ServiceUtils {
   public static String createDescriptionTag(String description) {
     return "Description: " + description;
   }
+
+  // Copied from SecurityUtil because it is not public
+  public static String getLocalHostName(@Nullable Configuration conf)
+      throws UnknownHostException {
+    if (conf != null) {
+      String dnsInterface = conf.get(HADOOP_SECURITY_DNS_INTERFACE_KEY);
+      String nameServer = conf.get(HADOOP_SECURITY_DNS_NAMESERVER_KEY);
+
+      if (dnsInterface != null) {
+        return DNS.getDefaultHost(dnsInterface, nameServer, true);
+      } else if (nameServer != null) {
+        throw new IllegalArgumentException(HADOOP_SECURITY_DNS_NAMESERVER_KEY +
+            " requires " + HADOOP_SECURITY_DNS_INTERFACE_KEY + ". Check your" +
+            "configuration.");
+      }
+    }
+
+    // Fallback to querying the default hostname as we did before.
+    return InetAddress.getLocalHost().getCanonicalHostName();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
new file mode 100644
index 0000000..14cdf68
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
@@ -0,0 +1,14 @@
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+org.apache.hadoop.yarn.service.ClientAMSecurityInfo

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
index 704b097..e74ca81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
@@ -111,6 +111,27 @@ public final class RegistryOperationsFactory {
   }
 
   /**
+   * Create a kerberos registry service client
+   * @param conf configuration
+   * @param jaasClientEntry the name of the login config entry
+   * @param principal principal of the client.
+   * @param keytab location to the keytab file
+   * @return a registry service client instance
+   */
+  public static RegistryOperations createKerberosInstance(Configuration conf,
+      String jaasClientEntry, String principal, String keytab) {
+    Preconditions.checkArgument(conf != null, "Null configuration");
+    conf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_KERBEROS);
+    conf.set(KEY_REGISTRY_CLIENT_JAAS_CONTEXT, jaasClientEntry);
+    RegistryOperationsClient operations =
+        new RegistryOperationsClient("KerberosRegistryOperations");
+    operations.setKerberosPrincipalAndKeytab(principal, keytab);
+    operations.init(conf);
+    return operations;
+  }
+
+
+  /**
    * Create and initialize an operations instance authenticated with write
    * access via an <code>id:password</code> pair.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
index 8713920..c81a0ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
@@ -127,6 +127,7 @@ public class CuratorService extends CompositeService
     } else {
       this.bindingSource = this;
     }
+    registrySecurity = new RegistrySecurity("registry security");
   }
 
   /**
@@ -152,8 +153,7 @@ public class CuratorService extends CompositeService
     registryRoot = conf.getTrimmed(KEY_REGISTRY_ZK_ROOT,
         DEFAULT_ZK_REGISTRY_ROOT);
 
-    // create and add the registy service
-    registrySecurity = new RegistrySecurity("registry security");
+    // add the registy service
     addService(registrySecurity);
 
     if (LOG.isDebugEnabled()) {
@@ -163,6 +163,10 @@ public class CuratorService extends CompositeService
     super.serviceInit(conf);
   }
 
+  public void setKerberosPrincipalAndKeytab(String principal, String keytab) {
+    registrySecurity.setKerberosPrincipalAndKeytab(principal, keytab);
+  }
+
   /**
    * Start the service.
    * This is where the curator instance is started.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
index 23fadb5..ff6e8aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
@@ -23,6 +23,7 @@ import com.google.common.base.Splitter;
 import com.google.common.collect.Lists;
 import org.apache.commons.lang.StringUtils;
 import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.util.KerberosUtil;
@@ -31,6 +32,7 @@ import org.apache.hadoop.service.ServiceStateException;
 import org.apache.hadoop.util.ZKUtil;
 import org.apache.zookeeper.Environment;
 import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Id;
 import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
@@ -44,9 +46,11 @@ import java.lang.reflect.InvocationTargetException;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Locale;
+import java.util.Map;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 import static org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions.*;
@@ -170,13 +174,17 @@ public class RegistrySecurity extends AbstractService {
   /**
    * Client context
    */
-  private String jaasClientContext;
+  private String jaasClientEntry;
 
   /**
    * Client identity
    */
   private String jaasClientIdentity;
 
+  private String principal;
+
+  private String keytab;
+
   /**
    * Create an instance
    * @param name service name
@@ -238,6 +246,8 @@ public class RegistrySecurity extends AbstractService {
 
       systemACLs.addAll(buildACLs(system, kerberosRealm, ZooDefs.Perms.ALL));
 
+      LOG.info("Registry default system acls: " + System.lineSeparator() +
+          systemACLs);
       // user accounts (may be empty, but for digest one user AC must
       // be built up
       String user = getConfig().get(KEY_REGISTRY_USER_ACCOUNTS,
@@ -252,6 +262,7 @@ public class RegistrySecurity extends AbstractService {
           userACLs.add(self);
         }
       }
+      LOG.info("Registry User ACLs " + System.lineSeparator()+ userACLs);
 
       // here check for UGI having secure on or digest + ID
       switch (access) {
@@ -262,13 +273,12 @@ public class RegistrySecurity extends AbstractService {
           }
           UserGroupInformation currentUser =
               UserGroupInformation.getCurrentUser();
-          jaasClientContext = getOrFail(KEY_REGISTRY_CLIENT_JAAS_CONTEXT,
+          jaasClientEntry = getOrFail(KEY_REGISTRY_CLIENT_JAAS_CONTEXT,
               DEFAULT_REGISTRY_CLIENT_JAAS_CONTEXT);
           jaasClientIdentity = currentUser.getShortUserName();
           if (LOG.isDebugEnabled()) {
             LOG.debug("Auth is SASL user=\"{}\" JAAS context=\"{}\"",
-                jaasClientIdentity,
-                jaasClientContext);
+                jaasClientIdentity, jaasClientEntry);
           }
           break;
 
@@ -738,9 +748,81 @@ public class RegistrySecurity extends AbstractService {
           break;
 
         case sasl:
-          // bind to the current identity and context within the JAAS file
-          setZKSaslClientProperties(jaasClientIdentity, jaasClientContext);
+          JaasConfiguration jconf =
+              new JaasConfiguration(jaasClientEntry, principal, keytab);
+          javax.security.auth.login.Configuration.setConfiguration(jconf);
+          setSystemPropertyIfUnset(ZooKeeperSaslClient.ENABLE_CLIENT_SASL_KEY,
+              "true");
+          setSystemPropertyIfUnset(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
+              jaasClientEntry);
+          LOG.info(
+              "Enabling ZK sasl client: jaasClientEntry = " + jaasClientEntry
+                  + ", principal = " + principal + ", keytab = " + keytab);
+      }
+    }
+  }
+
+  public void setKerberosPrincipalAndKeytab(String principal, String keytab) {
+    this.principal = principal;
+    this.keytab = keytab;
+  }
+
+  /**
+   * Creates a programmatic version of a jaas.conf file. This can be used
+   * instead of writing a jaas.conf file and setting the system property,
+   * "java.security.auth.login.config", to point to that file. It is meant to be
+   * used for connecting to ZooKeeper.
+   */
+  @InterfaceAudience.Private
+  public static class JaasConfiguration extends
+      javax.security.auth.login.Configuration {
+
+    private final javax.security.auth.login.Configuration baseConfig =
+        javax.security.auth.login.Configuration.getConfiguration();
+    private static AppConfigurationEntry[] entry;
+    private String entryName;
+
+    /**
+     * Add an entry to the jaas configuration with the passed in name,
+     * principal, and keytab. The other necessary options will be set for you.
+     *
+     * @param entryName The name of the entry (e.g. "Client")
+     * @param principal The principal of the user
+     * @param keytab The location of the keytab
+     */
+    public JaasConfiguration(String entryName, String principal, String keytab) {
+      this.entryName = entryName;
+      Map<String, String> options = new HashMap<String, String>();
+      options.put("keyTab", keytab);
+      options.put("principal", principal);
+      options.put("useKeyTab", "true");
+      options.put("storeKey", "true");
+      options.put("useTicketCache", "false");
+      options.put("refreshKrb5Config", "true");
+      String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
+      if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
+        options.put("debug", "true");
+      }
+      entry = new AppConfigurationEntry[]{
+          new AppConfigurationEntry(getKrb5LoginModuleName(),
+              AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
+              options)};
+    }
+
+    @Override
+    public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
+      return (entryName.equals(name)) ? entry : ((baseConfig != null)
+          ? baseConfig.getAppConfigurationEntry(name) : null);
+    }
+
+    private String getKrb5LoginModuleName() {
+      String krb5LoginModuleName;
+      if (System.getProperty("java.vendor").contains("IBM")) {
+        krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
+      } else {
+        krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
       }
+      return krb5LoginModuleName;
     }
   }
 
@@ -899,7 +981,7 @@ public class RegistrySecurity extends AbstractService {
              .append("; ");
       builder.append(KEY_REGISTRY_CLIENT_JAAS_CONTEXT)
              .append("=")
-             .append(jaasClientContext)
+             .append(jaasClientEntry)
              .append("; ");
       builder.append(describeProperty(PROP_ZK_SASL_CLIENT_USERNAME));
       builder.append(describeProperty(PROP_ZK_SASL_CLIENT_CONTEXT));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index d7ebece..358a963 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -412,6 +412,10 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
       // Single reverse zone
     } else {
       Name reverseLookupZoneName = getReverseZoneName(conf);
+      if (reverseLookupZoneName == null) {
+        // reverse lookup disabled
+        return;
+      }
       Zone reverseLookupZone = configureZone(reverseLookupZoneName, conf);
       zones.put(reverseLookupZone.getOrigin(), reverseLookupZone);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java
deleted file mode 100644
index e11890f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.registry.server.integration;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.curator.framework.api.BackgroundCallback;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.registry.client.impl.zk.RegistryBindingSource;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.server.services.DeleteCompletionCallback;
-import org.apache.hadoop.registry.server.services.RegistryAdminService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.Future;
-
-/**
- * Handle RM events by updating the registry
- * <p>
- * These actions are all implemented as event handlers to operations
- * which come from the RM.
- * <p>
- * This service is expected to be executed by a user with the permissions
- * to manipulate the entire registry,
- */
-@InterfaceAudience.LimitedPrivate("YARN")
-@InterfaceStability.Evolving
-public class RMRegistryOperationsService extends RegistryAdminService {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(RMRegistryOperationsService.class);
-
-  private PurgePolicy purgeOnCompletionPolicy = PurgePolicy.PurgeAll;
-
-  public RMRegistryOperationsService(String name) {
-    this(name, null);
-  }
-
-  public RMRegistryOperationsService(String name,
-      RegistryBindingSource bindingSource) {
-    super(name, bindingSource);
-  }
-
-
-  /**
-   * Extend the parent service initialization by verifying that the
-   * service knows —in a secure cluster— the realm in which it is executing.
-   * It needs this to properly build up the user names and hence their
-   * access rights.
-   *
-   * @param conf configuration of the service
-   * @throws Exception
-   */
-  @Override
-  protected void serviceInit(Configuration conf) throws Exception {
-    super.serviceInit(conf);
-
-    verifyRealmValidity();
-  }
-
-  public PurgePolicy getPurgeOnCompletionPolicy() {
-    return purgeOnCompletionPolicy;
-  }
-
-  public void setPurgeOnCompletionPolicy(PurgePolicy purgeOnCompletionPolicy) {
-    this.purgeOnCompletionPolicy = purgeOnCompletionPolicy;
-  }
-
-  public void onApplicationAttemptRegistered(ApplicationAttemptId attemptId,
-      String host, int rpcport, String trackingurl) throws IOException {
-
-  }
-
-  public void onApplicationLaunched(ApplicationId id) throws IOException {
-
-  }
-
-  /**
-   * Actions to take as an AM registers itself with the RM.
-   * @param attemptId attempt ID
-   * @throws IOException problems
-   */
-  public void onApplicationMasterRegistered(ApplicationAttemptId attemptId) throws
-      IOException {
-  }
-
-  /**
-   * Actions to take when the AM container is completed
-   * @param containerId  container ID
-   * @throws IOException problems
-   */
-  public void onAMContainerFinished(ContainerId containerId) throws
-      IOException {
-    LOG.info("AM Container {} finished, purging application attempt records",
-        containerId);
-
-    // remove all application attempt entries
-    purgeAppAttemptRecords(containerId.getApplicationAttemptId());
-
-    // also treat as a container finish to remove container
-    // level records for the AM container
-    onContainerFinished(containerId);
-  }
-
-  /**
-   * remove all application attempt entries
-   * @param attemptId attempt ID
-   */
-  protected void purgeAppAttemptRecords(ApplicationAttemptId attemptId) {
-    purgeRecordsAsync("/",
-        attemptId.toString(),
-        PersistencePolicies.APPLICATION_ATTEMPT);
-  }
-
-  /**
-   * Actions to take when an application attempt is completed
-   * @param attemptId  application  ID
-   * @throws IOException problems
-   */
-  public void onApplicationAttemptUnregistered(ApplicationAttemptId attemptId)
-      throws IOException {
-    LOG.info("Application attempt {} unregistered, purging app attempt records",
-        attemptId);
-    purgeAppAttemptRecords(attemptId);
-  }
-
-  /**
-   * Actions to take when an application is completed
-   * @param id  application  ID
-   * @throws IOException problems
-   */
-  public void onApplicationCompleted(ApplicationId id)
-      throws IOException {
-    LOG.info("Application {} completed, purging application-level records",
-        id);
-    purgeRecordsAsync("/",
-        id.toString(),
-        PersistencePolicies.APPLICATION);
-  }
-
-  public void onApplicationAttemptAdded(ApplicationAttemptId appAttemptId) {
-  }
-
-  /**
-   * This is the event where the user is known, so the user directory
-   * can be created
-   * @param applicationId application  ID
-   * @param user username
-   * @throws IOException problems
-   */
-  public void onStateStoreEvent(ApplicationId applicationId, String user) throws
-      IOException {
-    initUserRegistryAsync(user);
-  }
-
-  /**
-   * Actions to take when the AM container is completed
-   * @param id  container ID
-   * @throws IOException problems
-   */
-  public void onContainerFinished(ContainerId id) throws IOException {
-    LOG.info("Container {} finished, purging container-level records",
-        id);
-    purgeRecordsAsync("/",
-        id.toString(),
-        PersistencePolicies.CONTAINER);
-  }
-
-  /**
-   * Queue an async operation to purge all matching records under a base path.
-   * <ol>
-   *   <li>Uses a depth first search</li>
-   *   <li>A match is on ID and persistence policy, or, if policy==-1, any match</li>
-   *   <li>If a record matches then it is deleted without any child searches</li>
-   *   <li>Deletions will be asynchronous if a callback is provided</li>
-   * </ol>
-   * @param path base path
-   * @param id ID for service record.id
-   * @param persistencePolicyMatch ID for the persistence policy to match:
-   * no match, no delete.
-   * @return a future that returns the #of records deleted
-   */
-  @VisibleForTesting
-  public Future<Integer> purgeRecordsAsync(String path,
-      String id,
-      String persistencePolicyMatch) {
-
-    return purgeRecordsAsync(path,
-        id, persistencePolicyMatch,
-        purgeOnCompletionPolicy,
-        new DeleteCompletionCallback());
-  }
-
-  /**
-   * Queue an async operation to purge all matching records under a base path.
-   * <ol>
-   *   <li>Uses a depth first search</li>
-   *   <li>A match is on ID and persistence policy, or, if policy==-1, any match</li>
-   *   <li>If a record matches then it is deleted without any child searches</li>
-   *   <li>Deletions will be asynchronous if a callback is provided</li>
-   * </ol>
-   * @param path base path
-   * @param id ID for service record.id
-   * @param persistencePolicyMatch ID for the persistence policy to match:
-   * no match, no delete.
-   * @param purgePolicy how to react to children under the entry
-   * @param callback an optional callback
-   * @return a future that returns the #of records deleted
-   */
-  @VisibleForTesting
-  public Future<Integer> purgeRecordsAsync(String path,
-      String id,
-      String persistencePolicyMatch,
-      PurgePolicy purgePolicy,
-      BackgroundCallback callback) {
-    LOG.info(" records under {} with ID {} and policy {}: {}",
-        path, id, persistencePolicyMatch);
-    return submit(
-        new AsyncPurge(path,
-            new SelectByYarnPersistence(id, persistencePolicyMatch),
-            purgePolicy,
-            callback));
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java
index e160d4a..829ef68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.registry.server.services;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.api.BackgroundCallback;
 import org.apache.curator.framework.api.CuratorEvent;
-import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,7 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger;
  */
 public class DeleteCompletionCallback implements BackgroundCallback {
   private static final Logger LOG =
-      LoggerFactory.getLogger(RMRegistryOperationsService.class);
+      LoggerFactory.getLogger(DeleteCompletionCallback.class);
 
   private AtomicInteger events = new AtomicInteger(0);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java
index 5b34f60..0d4a467 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java
@@ -23,7 +23,8 @@ import org.apache.hadoop.registry.client.api.RegistryOperations;
 import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
 import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
 import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
+
+import org.apache.hadoop.registry.server.services.RegistryAdminService;
 import org.junit.Before;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -31,22 +32,16 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.net.URISyntaxException;
 
-/**
- * Abstract registry tests .. inits the field {@link #registry}
- * before the test with an instance of {@link RMRegistryOperationsService};
- * and {@link #operations} with the same instance cast purely
- * to the type {@link RegistryOperations}.
- *
- */
+
 public class AbstractRegistryTest extends AbstractZKRegistryTest {
   private static final Logger LOG =
       LoggerFactory.getLogger(AbstractRegistryTest.class);
-  protected RMRegistryOperationsService registry;
+  protected RegistryAdminService registry;
   protected RegistryOperations operations;
 
   @Before
   public void setupRegistry() throws IOException {
-    registry = new RMRegistryOperationsService("yarnRegistry");
+    registry = new RegistryAdminService("yarnRegistry");
     operations = registry;
     registry.init(createRegistryConfiguration());
     registry.start();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java
deleted file mode 100644
index 451a69b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java
+++ /dev/null
@@ -1,369 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.registry.integration;
-
-import org.apache.curator.framework.api.BackgroundCallback;
-import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
-import org.apache.hadoop.registry.AbstractRegistryTest;
-import org.apache.hadoop.registry.client.api.BindFlags;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
-import org.apache.hadoop.registry.client.impl.CuratorEventCatcher;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.RegistryPathStatus;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.registry.server.services.DeleteCompletionCallback;
-import org.apache.hadoop.registry.server.services.RegistryAdminService;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-
-import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.inetAddrEndpoint;
-import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.restEndpoint;
-
-public class TestRegistryRMOperations extends AbstractRegistryTest {
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(TestRegistryRMOperations.class);
-
-  /**
-   * trigger a purge operation
-   * @param path path
-   * @param id yarn ID
-   * @param policyMatch policy to match ID on
-   * @param purgePolicy policy when there are children under a match
-   * @return the number purged
-   * @throws IOException
-   */
-  public int purge(String path,
-      String id,
-      String policyMatch,
-      RegistryAdminService.PurgePolicy purgePolicy) throws
-      IOException,
-      ExecutionException,
-      InterruptedException {
-    return purge(path, id, policyMatch, purgePolicy, null);
-  }
-
-  /**
-   *
-   * trigger a purge operation
-   * @param path pathn
-   * @param id yarn ID
-   * @param policyMatch policy to match ID on
-   * @param purgePolicy policy when there are children under a match
-   * @param callback optional callback
-   * @return the number purged
-   * @throws IOException
-   */
-  public int purge(String path,
-      String id,
-      String policyMatch,
-      RegistryAdminService.PurgePolicy purgePolicy,
-      BackgroundCallback callback) throws
-      IOException,
-      ExecutionException,
-      InterruptedException {
-
-    Future<Integer> future = registry.purgeRecordsAsync(path,
-        id, policyMatch, purgePolicy, callback);
-    try {
-      return future.get();
-    } catch (ExecutionException e) {
-      if (e.getCause() instanceof IOException) {
-        throw (IOException) e.getCause();
-      } else {
-        throw e;
-      }
-    }
-  }
-
-  @Test
-  public void testPurgeEntryCuratorCallback() throws Throwable {
-
-    String path = "/users/example/hbase/hbase1/";
-    ServiceRecord written = buildExampleServiceEntry(
-        PersistencePolicies.APPLICATION_ATTEMPT);
-    written.set(YarnRegistryAttributes.YARN_ID,
-        "testAsyncPurgeEntry_attempt_001");
-
-    operations.mknode(RegistryPathUtils.parentOf(path), true);
-    operations.bind(path, written, 0);
-
-    ZKPathDumper dump = registry.dumpPath(false);
-    CuratorEventCatcher events = new CuratorEventCatcher();
-
-    LOG.info("Initial state {}", dump);
-
-    // container query
-    String id = written.get(YarnRegistryAttributes.YARN_ID, "");
-    int opcount = purge("/",
-        id,
-        PersistencePolicies.CONTAINER,
-        RegistryAdminService.PurgePolicy.PurgeAll,
-        events);
-    assertPathExists(path);
-    assertEquals(0, opcount);
-    assertEquals("Event counter", 0, events.getCount());
-
-    // now the application attempt
-    opcount = purge("/",
-        id,
-        PersistencePolicies.APPLICATION_ATTEMPT,
-        RegistryAdminService.PurgePolicy.PurgeAll,
-        events);
-
-    LOG.info("Final state {}", dump);
-
-    assertPathNotFound(path);
-    assertEquals("wrong no of delete operations in " + dump, 1, opcount);
-    // and validate the callback event
-    assertEquals("Event counter", 1, events.getCount());
-  }
-
-  @Test
-  public void testAsyncPurgeEntry() throws Throwable {
-
-    String path = "/users/example/hbase/hbase1/";
-    ServiceRecord written = buildExampleServiceEntry(
-        PersistencePolicies.APPLICATION_ATTEMPT);
-    written.set(YarnRegistryAttributes.YARN_ID,
-        "testAsyncPurgeEntry_attempt_001");
-
-    operations.mknode(RegistryPathUtils.parentOf(path), true);
-    operations.bind(path, written, 0);
-
-    ZKPathDumper dump = registry.dumpPath(false);
-
-    LOG.info("Initial state {}", dump);
-
-    DeleteCompletionCallback deletions = new DeleteCompletionCallback();
-    int opcount = purge("/",
-        written.get(YarnRegistryAttributes.YARN_ID, ""),
-        PersistencePolicies.CONTAINER,
-        RegistryAdminService.PurgePolicy.PurgeAll,
-        deletions);
-    assertPathExists(path);
-
-    dump = registry.dumpPath(false);
-
-    assertEquals("wrong no of delete operations in " + dump, 0,
-        deletions.getEventCount());
-    assertEquals("wrong no of delete operations in " + dump, 0, opcount);
-
-
-    // now app attempt
-    deletions = new DeleteCompletionCallback();
-    opcount = purge("/",
-        written.get(YarnRegistryAttributes.YARN_ID, ""),
-        PersistencePolicies.APPLICATION_ATTEMPT,
-        RegistryAdminService.PurgePolicy.PurgeAll,
-        deletions);
-
-    dump = registry.dumpPath(false);
-    LOG.info("Final state {}", dump);
-
-    assertPathNotFound(path);
-    assertEquals("wrong no of delete operations in " + dump, 1,
-        deletions.getEventCount());
-    assertEquals("wrong no of delete operations in " + dump, 1, opcount);
-    // and validate the callback event
-
-  }
-
-  @Test
-  public void testPutGetContainerPersistenceServiceEntry() throws Throwable {
-
-    String path = ENTRY_PATH;
-    ServiceRecord written = buildExampleServiceEntry(
-        PersistencePolicies.CONTAINER);
-
-    operations.mknode(RegistryPathUtils.parentOf(path), true);
-    operations.bind(path, written, BindFlags.CREATE);
-    ServiceRecord resolved = operations.resolve(path);
-    validateEntry(resolved);
-    assertMatches(written, resolved);
-  }
-
-  /**
-   * Create a complex example app
-   * @throws Throwable
-   */
-  @Test
-  public void testCreateComplexApplication() throws Throwable {
-    String appId = "application_1408631738011_0001";
-    String cid = "container_1408631738011_0001_01_";
-    String cid1 = cid + "000001";
-    String cid2 = cid + "000002";
-    String appPath = USERPATH + "tomcat";
-
-    ServiceRecord webapp = createRecord(appId,
-        PersistencePolicies.APPLICATION, "tomcat-based web application",
-        null);
-    webapp.addExternalEndpoint(restEndpoint("www",
-        new URI("http", "//loadbalancer/", null)));
-
-    ServiceRecord comp1 = createRecord(cid1, PersistencePolicies.CONTAINER,
-        null,
-        null);
-    comp1.addExternalEndpoint(restEndpoint("www",
-        new URI("http", "//rack4server3:43572", null)));
-    comp1.addInternalEndpoint(
-        inetAddrEndpoint("jmx", "JMX", "rack4server3", 43573));
-
-    // Component 2 has a container lifespan
-    ServiceRecord comp2 = createRecord(cid2, PersistencePolicies.CONTAINER,
-        null,
-        null);
-    comp2.addExternalEndpoint(restEndpoint("www",
-        new URI("http", "//rack1server28:35881", null)));
-    comp2.addInternalEndpoint(
-        inetAddrEndpoint("jmx", "JMX", "rack1server28", 35882));
-
-    operations.mknode(USERPATH, false);
-    operations.bind(appPath, webapp, BindFlags.OVERWRITE);
-    String componentsPath = appPath + RegistryConstants.SUBPATH_COMPONENTS;
-    operations.mknode(componentsPath, false);
-    String dns1 = RegistryPathUtils.encodeYarnID(cid1);
-    String dns1path = componentsPath + dns1;
-    operations.bind(dns1path, comp1, BindFlags.CREATE);
-    String dns2 = RegistryPathUtils.encodeYarnID(cid2);
-    String dns2path = componentsPath + dns2;
-    operations.bind(dns2path, comp2, BindFlags.CREATE);
-
-    ZKPathDumper pathDumper = registry.dumpPath(false);
-    LOG.info(pathDumper.toString());
-
-    logRecord("tomcat", webapp);
-    logRecord(dns1, comp1);
-    logRecord(dns2, comp2);
-
-    ServiceRecord dns1resolved = operations.resolve(dns1path);
-    assertEquals("Persistence policies on resolved entry",
-        PersistencePolicies.CONTAINER,
-        dns1resolved.get(YarnRegistryAttributes.YARN_PERSISTENCE, ""));
-
-    Map<String, RegistryPathStatus> children =
-        RegistryUtils.statChildren(operations, componentsPath);
-    assertEquals(2, children.size());
-    Collection<RegistryPathStatus>
-        componentStats = children.values();
-    Map<String, ServiceRecord> records =
-        RegistryUtils.extractServiceRecords(operations,
-            componentsPath, componentStats);
-    assertEquals(2, records.size());
-    ServiceRecord retrieved1 = records.get(dns1path);
-    logRecord(retrieved1.get(YarnRegistryAttributes.YARN_ID, ""), retrieved1);
-    assertMatches(dns1resolved, retrieved1);
-    assertEquals(PersistencePolicies.CONTAINER,
-        retrieved1.get(YarnRegistryAttributes.YARN_PERSISTENCE, ""));
-
-    // create a listing under components/
-    operations.mknode(componentsPath + "subdir", false);
-
-    // this shows up in the listing of child entries
-    Map<String, RegistryPathStatus> childrenUpdated =
-        RegistryUtils.statChildren(operations, componentsPath);
-    assertEquals(3, childrenUpdated.size());
-
-    // the non-record child this is not picked up in the record listing
-    Map<String, ServiceRecord> recordsUpdated =
-
-        RegistryUtils.extractServiceRecords(operations,
-            componentsPath,
-            childrenUpdated);
-    assertEquals(2, recordsUpdated.size());
-
-    // now do some deletions.
-
-    // synchronous delete container ID 2
-
-    // fail if the app policy is chosen
-    assertEquals(0, purge("/", cid2, PersistencePolicies.APPLICATION,
-        RegistryAdminService.PurgePolicy.FailOnChildren));
-    // succeed for container
-    assertEquals(1, purge("/", cid2, PersistencePolicies.CONTAINER,
-        RegistryAdminService.PurgePolicy.FailOnChildren));
-    assertPathNotFound(dns2path);
-    assertPathExists(dns1path);
-
-    // expect a skip on children to skip
-    assertEquals(0,
-        purge("/", appId, PersistencePolicies.APPLICATION,
-            RegistryAdminService.PurgePolicy.SkipOnChildren));
-    assertPathExists(appPath);
-    assertPathExists(dns1path);
-
-    // attempt to delete app with policy of fail on children
-    try {
-      int p = purge("/",
-          appId,
-          PersistencePolicies.APPLICATION,
-          RegistryAdminService.PurgePolicy.FailOnChildren);
-      fail("expected a failure, got a purge count of " + p);
-    } catch (PathIsNotEmptyDirectoryException expected) {
-      // expected
-    }
-    assertPathExists(appPath);
-    assertPathExists(dns1path);
-
-
-    // now trigger recursive delete
-    assertEquals(1,
-        purge("/", appId, PersistencePolicies.APPLICATION,
-            RegistryAdminService.PurgePolicy.PurgeAll));
-    assertPathNotFound(appPath);
-    assertPathNotFound(dns1path);
-
-  }
-
-  @Test
-  public void testChildDeletion() throws Throwable {
-    ServiceRecord app = createRecord("app1",
-        PersistencePolicies.APPLICATION, "app",
-        null);
-    ServiceRecord container = createRecord("container1",
-        PersistencePolicies.CONTAINER, "container",
-        null);
-
-    operations.bind("/app", app, BindFlags.OVERWRITE);
-    operations.bind("/app/container", container, BindFlags.OVERWRITE);
-
-    try {
-      int p = purge("/",
-          "app1",
-          PersistencePolicies.APPLICATION,
-          RegistryAdminService.PurgePolicy.FailOnChildren);
-      fail("expected a failure, got a purge count of " + p);
-    } catch (PathIsNotEmptyDirectoryException expected) {
-      // expected
-    }
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
deleted file mode 100644
index 41760d6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.registry.secure;
-
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.PathPermissionException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.ServiceStateException;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException;
-import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
-import org.apache.hadoop.registry.client.impl.RegistryOperationsClient;
-import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
-import org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions;
-import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
-import org.apache.hadoop.registry.server.services.RegistryAdminService;
-import org.apache.zookeeper.client.ZooKeeperSaslClient;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Id;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.security.auth.login.LoginException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.List;
-
-import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
-
-/**
- * Verify that the {@link RMRegistryOperationsService} works securely
- */
-public class TestSecureRMRegistryOperations extends AbstractSecureRegistryTest {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestSecureRMRegistryOperations.class);
-  private Configuration secureConf;
-  private Configuration zkClientConf;
-  private UserGroupInformation zookeeperUGI;
-
-  @Before
-  public void setupTestSecureRMRegistryOperations() throws Exception {
-    startSecureZK();
-    secureConf = new Configuration();
-    secureConf.setBoolean(KEY_REGISTRY_SECURE, true);
-
-    // create client conf containing the ZK quorum
-    zkClientConf = new Configuration(secureZK.getConfig());
-    zkClientConf.setBoolean(KEY_REGISTRY_SECURE, true);
-    assertNotEmpty(zkClientConf.get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM));
-
-    // ZK is in charge
-    secureConf.set(KEY_REGISTRY_SYSTEM_ACCOUNTS, "sasl:zookeeper@");
-    zookeeperUGI = loginUGI(ZOOKEEPER, keytab_zk);
-  }
-
-  @After
-  public void teardownTestSecureRMRegistryOperations() {
-  }
-
-  /**
-   * Create the RM registry operations as the current user
-   * @return the service
-   * @throws LoginException
-   * @throws FileNotFoundException
-   */
-  public RMRegistryOperationsService startRMRegistryOperations() throws
-      LoginException, IOException, InterruptedException {
-    // kerberos
-    secureConf.set(KEY_REGISTRY_CLIENT_AUTH,
-        REGISTRY_CLIENT_AUTH_KERBEROS);
-    secureConf.set(KEY_REGISTRY_CLIENT_JAAS_CONTEXT, ZOOKEEPER_CLIENT_CONTEXT);
-
-    RMRegistryOperationsService registryOperations = zookeeperUGI.doAs(
-        new PrivilegedExceptionAction<RMRegistryOperationsService>() {
-          @Override
-          public RMRegistryOperationsService run() throws Exception {
-            RMRegistryOperationsService operations
-                = new RMRegistryOperationsService("rmregistry", secureZK);
-            addToTeardown(operations);
-            operations.init(secureConf);
-            LOG.info(operations.bindingDiagnosticDetails());
-            operations.start();
-            return operations;
-          }
-        });
-
-    return registryOperations;
-  }
-
-  /**
-   * test that ZK can write as itself
-   * @throws Throwable
-   */
-  @Test
-  public void testZookeeperCanWriteUnderSystem() throws Throwable {
-
-    RMRegistryOperationsService rmRegistryOperations =
-        startRMRegistryOperations();
-    RegistryOperations operations = rmRegistryOperations;
-    operations.mknode(PATH_SYSTEM_SERVICES + "hdfs",
-        false);
-    ZKPathDumper pathDumper = rmRegistryOperations.dumpPath(true);
-    LOG.info(pathDumper.toString());
-  }
-
-  @Test
-  public void testAnonReadAccess() throws Throwable {
-    RMRegistryOperationsService rmRegistryOperations =
-        startRMRegistryOperations();
-    describe(LOG, "testAnonReadAccess");
-    RegistryOperations operations =
-        RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
-    addToTeardown(operations);
-    operations.start();
-
-    assertFalse("RegistrySecurity.isClientSASLEnabled()==true",
-        RegistrySecurity.isClientSASLEnabled());
-    operations.list(PATH_SYSTEM_SERVICES);
-  }
-
-  @Test
-  public void testAnonNoWriteAccess() throws Throwable {
-    RMRegistryOperationsService rmRegistryOperations =
-        startRMRegistryOperations();
-    describe(LOG, "testAnonNoWriteAccess");
-    RegistryOperations operations =
-        RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
-    addToTeardown(operations);
-    operations.start();
-
-    String servicePath = PATH_SYSTEM_SERVICES + "hdfs";
-    expectMkNodeFailure(operations, servicePath);
-  }
-
-  @Test
-  public void testAnonNoWriteAccessOffRoot() throws Throwable {
-    RMRegistryOperationsService rmRegistryOperations =
-        startRMRegistryOperations();
-    describe(LOG, "testAnonNoWriteAccessOffRoot");
-    RegistryOperations operations =
-        RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
-    addToTeardown(operations);
-    operations.start();
-    assertFalse("mknode(/)", operations.mknode("/", false));
-    expectMkNodeFailure(operations, "/sub");
-    expectDeleteFailure(operations, PATH_SYSTEM_SERVICES, true);
-  }
-
-  /**
-   * Expect a mknode operation to fail
-   * @param operations operations instance
-   * @param path path
-   * @throws IOException An IO failure other than those permitted
-   */
-  public void expectMkNodeFailure(RegistryOperations operations,
-      String path) throws IOException {
-    try {
-      operations.mknode(path, false);
-      fail("should have failed to create a node under " + path);
-    } catch (PathPermissionException expected) {
-      // expected
-    } catch (NoPathPermissionsException expected) {
-      // expected
-    }
-  }
-
-  /**
-   * Expect a delete operation to fail
-   * @param operations operations instance
-   * @param path path
-   * @param recursive
-   * @throws IOException An IO failure other than those permitted
-   */
-  public void expectDeleteFailure(RegistryOperations operations,
-      String path, boolean recursive) throws IOException {
-    try {
-      operations.delete(path, recursive);
-      fail("should have failed to delete the node " + path);
-    } catch (PathPermissionException expected) {
-      // expected
-    } catch (NoPathPermissionsException expected) {
-      // expected
-    }
-  }
-
-  @Test
-  public void testAlicePathRestrictedAnonAccess() throws Throwable {
-    RMRegistryOperationsService rmRegistryOperations =
-        startRMRegistryOperations();
-    String aliceHome = rmRegistryOperations.initUserRegistry(ALICE);
-    describe(LOG, "Creating anonymous accessor");
-    RegistryOperations anonOperations =
-        RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
-    addToTeardown(anonOperations);
-    anonOperations.start();
-    anonOperations.list(aliceHome);
-    expectMkNodeFailure(anonOperations, aliceHome + "/anon");
-    expectDeleteFailure(anonOperations, aliceHome, true);
-  }
-
-  @Test
-  public void testUserZookeeperHomePathAccess() throws Throwable {
-    RMRegistryOperationsService rmRegistryOperations =
-        startRMRegistryOperations();
-    final String home = rmRegistryOperations.initUserRegistry(ZOOKEEPER);
-    describe(LOG, "Creating ZK client");
-
-    RegistryOperations operations = zookeeperUGI.doAs(
-        new PrivilegedExceptionAction<RegistryOperations>() {
-          @Override
-          public RegistryOperations run() throws Exception {
-            RegistryOperations operations =
-                RegistryOperationsFactory.createKerberosInstance(zkClientConf,
-                    ZOOKEEPER_CLIENT_CONTEXT);
-            addToTeardown(operations);
-            operations.start();
-
-            return operations;
-          }
-        });
-    operations.list(home);
-    String path = home + "/subpath";
-    operations.mknode(path, false);
-    operations.delete(path, true);
-  }
-
-  @Test
-  public void testUserHomedirsPermissionsRestricted() throws Throwable {
-    // test that the /users/$user permissions are restricted
-    RMRegistryOperationsService rmRegistryOperations =
-        startRMRegistryOperations();
-    // create Alice's dir, so it should have an ACL for Alice
-    final String home = rmRegistryOperations.initUserRegistry(ALICE);
-    List<ACL> acls = rmRegistryOperations.zkGetACLS(home);
-    ACL aliceACL = null;
-    for (ACL acl : acls) {
-      LOG.info(RegistrySecurity.aclToString(acl));
-      Id id = acl.getId();
-      if (id.getScheme().equals(ZookeeperConfigOptions.SCHEME_SASL)
-          && id.getId().startsWith(ALICE)) {
-
-        aliceACL = acl;
-        break;
-      }
-    }
-    assertNotNull(aliceACL);
-    assertEquals(RegistryAdminService.USER_HOMEDIR_ACL_PERMISSIONS,
-        aliceACL.getPerms());
-  }
-
-  @Test
-  public void testDigestAccess() throws Throwable {
-    RMRegistryOperationsService registryAdmin =
-        startRMRegistryOperations();
-    String id = "username";
-    String pass = "password";
-    registryAdmin.addWriteAccessor(id, pass);
-    List<ACL> clientAcls = registryAdmin.getClientAcls();
-    LOG.info("Client ACLS=\n{}", RegistrySecurity.aclsToString(clientAcls));
-
-    String base = "/digested";
-    registryAdmin.mknode(base, false);
-    List<ACL> baseACLs = registryAdmin.zkGetACLS(base);
-    String aclset = RegistrySecurity.aclsToString(baseACLs);
-    LOG.info("Base ACLs=\n{}", aclset);
-    ACL found = null;
-    for (ACL acl : baseACLs) {
-      if (ZookeeperConfigOptions.SCHEME_DIGEST.equals(acl.getId().getScheme())) {
-        found = acl;
-        break;
-      }
-    }
-    assertNotNull("Did not find digest entry in ACLs " + aclset, found);
-    zkClientConf.set(KEY_REGISTRY_USER_ACCOUNTS,
-        "sasl:somebody@EXAMPLE.COM, sasl:other");
-    RegistryOperations operations =
-        RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
-            id,
-            pass);
-    addToTeardown(operations);
-    operations.start();
-    RegistryOperationsClient operationsClient =
-        (RegistryOperationsClient) operations;
-    List<ACL> digestClientACLs = operationsClient.getClientAcls();
-    LOG.info("digest client ACLs=\n{}",
-        RegistrySecurity.aclsToString(digestClientACLs));
-    operations.stat(base);
-    operations.mknode(base + "/subdir", false);
-    ZKPathDumper pathDumper = registryAdmin.dumpPath(true);
-    LOG.info(pathDumper.toString());
-  }
-
-  @Test(expected = IllegalArgumentException.class)
-  public void testNoDigestAuthMissingId() throws Throwable {
-    RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
-        "",
-        "pass");
-  }
-
-  @Test(expected = ServiceStateException.class)
-  public void testNoDigestAuthMissingId2() throws Throwable {
-    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_DIGEST);
-    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_ID, "");
-    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_PASSWORD, "pass");
-    RegistryOperationsFactory.createInstance("DigestRegistryOperations",
-        zkClientConf);
-  }
-
-  @Test(expected = IllegalArgumentException.class)
-  public void testNoDigestAuthMissingPass() throws Throwable {
-    RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
-        "id",
-        "");
-  }
-
-  @Test(expected = ServiceStateException.class)
-  public void testNoDigestAuthMissingPass2() throws Throwable {
-    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_DIGEST);
-    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_ID, "id");
-    zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_PASSWORD, "");
-    RegistryOperationsFactory.createInstance("DigestRegistryOperations",
-        zkClientConf);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index 512c011..f13d7d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -208,7 +208,25 @@ If you are building from source code, make sure you use `-Pyarn-ui` in the `mvn`
   </property>
 ```
 
-# Try with Docker
+# Run with security
+YARN service framework supports running in a secure(kerberized) environment. User needs to specify the kerberos principal name and keytab when they launch the service.
+E.g. A typical configuration looks like below:
+```
+{
+  "name": "sample-service",
+  ...
+  ...
+  "kerberos_principal" : {
+    "principal_name" : "hdfs-demo@EXAMPLE.COM",
+    "keytab" : "hdfs:///etc/security/keytabs/hdfs.headless.keytab"
+  }
+}
+```
+* principal_name : the principal name of the user who launches the service
+* keytab : URI of the keytab. It supports two modes:
+    * URI starts with `hdfs://`: The URI where the keytab is stored on hdfs. The keytab will be localized to each node by YARN.
+    * URI starts with `file://`: The URI where the keytab is stored on local host. It is assumed that admin pre-installs the keytabs on the local host before AM launches.
+# Run with Docker
 The above example is only for a non-docker container based service. YARN Service Framework also provides first-class support for managing docker based services.
 Most of the steps for managing docker based services are the same except that in docker the `Artifact` type for a component is `DOCKER` and the Artifact `id` is the name of the docker image.
 For details in how to setup docker on YARN, please check [Docker on YARN](../DockerContainers.md).

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index c0e12c7..e224e5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -291,6 +291,15 @@ The current state of the container of a service.
 |----|----|----|----|----|
 |state|enum of the state of the container|false|enum (INIT, STARTED, READY)||
 
+### KerberosPrincipal
+
+The kerberos principal info of the user who launches the service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|principal_name|The principal name of the user who launches the service.|false|string||
+|keytab|The URI of the kerberos keytab. It supports two modes, URI starts with "hdfs://": A path on hdfs where the keytab is stored. The keytab will be localized by YARN to each host; URI starts with "file://": A path on the local host where the keytab is stored. It is assumed that the keytabs are pre-installed by admins before AM launches.|false|string||
+
 
 ### PlacementPolicy
 
@@ -342,7 +351,7 @@ a service resource has the following attributes.
 |state|State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state.|false|ServiceState||
 |quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
 |queue|The YARN queue that this service should be submitted to.|false|string||
-
+|kerberos_principal | The principal info of the user who launches the service|false||
 
 ### ServiceState
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message