hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject hadoop git commit: YARN-7777. Fix user name format in YARN Registry DNS name. Contributed by Jian He
Date Wed, 24 Jan 2018 22:20:59 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk fa8cf4d1b -> 0c559b278


YARN-7777. Fix user name format in YARN Registry DNS name. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c559b27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c559b27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c559b27

Branch: refs/heads/trunk
Commit: 0c559b27820d12ebe5c315c6e2d1685c6de6bd33
Parents: fa8cf4d
Author: Billie Rinaldi <billie@apache.org>
Authored: Wed Jan 24 13:50:47 2018 -0800
Committer: Billie Rinaldi <billie@apache.org>
Committed: Wed Jan 24 13:50:47 2018 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/service/ServiceScheduler.java   | 11 +++-------
 .../yarn/service/utils/ServiceApiUtil.java      | 15 +++++++++----
 .../client/binding/RegistryPathUtils.java       | 20 ++++++++++++++++++
 .../registry/client/binding/RegistryUtils.java  |  5 ++++-
 .../server/dns/BaseServiceRecordProcessor.java  | 22 ++------------------
 .../client/binding/TestRegistryPathUtils.java   | 10 ++++++++-
 .../markdown/yarn-service/Configurations.md     |  2 +-
 7 files changed, 50 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 6cf4e14..dfe9808 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -392,14 +392,9 @@ public class ServiceScheduler extends CompositeService {
     // ZK
     globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM, getConfig()
         .getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
-    String user = null;
-    try {
-      user = UserGroupInformation.getCurrentUser().getShortUserName();
-    } catch (IOException e) {
-      LOG.error("Failed to get user.", e);
-    }
-    globalTokens
-        .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkServiceHomePath(user, app.getName()));
+    String user = RegistryUtils.currentUser();
+    globalTokens.put(SERVICE_ZK_PATH,
+        ServiceRegistryUtils.mkServiceHomePath(user, app.getName()));
 
     globalTokens.put(ServiceApiConstants.USER, user);
     String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 7f85c95..a5716bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -61,6 +61,9 @@ public class ServiceApiUtil {
   private static final PatternValidator namePattern
       = new PatternValidator("[a-z][a-z0-9-]*");
 
+  private static final PatternValidator userNamePattern
+      = new PatternValidator("[a-z][a-z0-9-.]*");
+
   @VisibleForTesting
   public static void setJsonSerDeser(JsonSerDeser jsd) {
     jsonSerDeser = jsd;
@@ -72,11 +75,15 @@ public class ServiceApiUtil {
       IOException {
     boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
         RegistryConstants.DEFAULT_DNS_ENABLED);
-    if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
-        .MAX_FQDN_LABEL_LENGTH) {
-      throw new IllegalArgumentException(RestApiErrorMessages
-          .ERROR_USER_NAME_INVALID);
+    if (dnsEnabled) {
+      if (RegistryUtils.currentUser().length()
+          > RegistryConstants.MAX_FQDN_LABEL_LENGTH) {
+        throw new IllegalArgumentException(
+            RestApiErrorMessages.ERROR_USER_NAME_INVALID);
+      }
+      userNamePattern.validate(RegistryUtils.currentUser());
     }
+
     if (StringUtils.isEmpty(service.getName())) {
       throw new IllegalArgumentException(
           RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
index 5fa45f9..b8e9ba1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
@@ -24,11 +24,13 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException;
 import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
+import org.apache.hadoop.registry.server.dns.BaseServiceRecordProcessor;
 import org.apache.zookeeper.common.PathUtils;
 
 import java.net.IDN;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
@@ -45,6 +47,9 @@ public class RegistryPathUtils {
   private static final Pattern PATH_ENTRY_VALIDATION_PATTERN =
       Pattern.compile(RegistryInternalConstants.VALID_PATH_ENTRY_PATTERN);
 
+  private static final Pattern USER_NAME =
+      Pattern.compile("/users/([a-z][a-z0-9-.]*)");
+
   /**
    * Validate ZK path with the path itself included in
    * the exception text
@@ -215,4 +220,19 @@ public class RegistryPathUtils {
   public static String encodeYarnID(String yarnId) {
     return yarnId.replace("container", "ctr").replace("_", "-");
   }
+
+  /**
+   * Return the username found in the ZK path.
+   *
+   * @param recPath the ZK recPath.
+   * @return the user name.
+   */
+  public static String getUsername(String recPath) {
+    String user = "anonymous";
+    Matcher matcher = USER_NAME.matcher(recPath);
+    if (matcher.find()) {
+      user = matcher.group(1);
+    }
+    return user;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
index 858b6b1..4ef7b8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
@@ -296,7 +296,10 @@ public class RegistryUtils {
    */
   public static String currentUser() {
     String shortUserName = currentUsernameUnencoded();
-    return encodeForRegistry(shortUserName);
+    String encodedName =  encodeForRegistry(shortUserName);
+    // DNS name doesn't allow "_", replace it with "-"
+    encodedName = RegistryUtils.convertUsername(encodedName);
+    return encodedName.replace("_", "-");
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
index fd5c74f..51ae99a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
@@ -36,8 +36,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 /**
  * Provides common service record processing logic.
@@ -51,7 +49,6 @@ public abstract class BaseServiceRecordProcessor
   private String path;
   private String domain;
 
-  private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
   private static final String YARN_SERVICE_API_PREFIX =
       "classpath:org.apache.hadoop.yarn.service.";
   private static final String HTTP_API_TYPE = "http://";
@@ -76,21 +73,6 @@ public abstract class BaseServiceRecordProcessor
   }
 
   /**
-   * Return the username found in the ZK path.
-   *
-   * @param recPath the ZK recPath.
-   * @return the user name.
-   */
-  protected String getUsername(String recPath) {
-    String user = "anonymous";
-    Matcher matcher = USER_NAME.matcher(recPath);
-    if (matcher.find()) {
-      user = matcher.group(1);
-    }
-    return user;
-  }
-
-  /**
    * Return the IPv6 mapped address for the provided IPv4 address. Utilized
    * to create corresponding AAAA records.
    *
@@ -300,7 +282,7 @@ public abstract class BaseServiceRecordProcessor
       String service = RegistryPathUtils.lastPathEntry(
           RegistryPathUtils.parentOf(RegistryPathUtils.parentOf(getPath())));
       String description = getRecord().description.toLowerCase();
-      String user = getUsername(getPath());
+      String user = RegistryPathUtils.getUsername(getPath());
       return Name.fromString(MessageFormat.format("{0}.{1}.{2}.{3}",
           description,
           service,
@@ -352,7 +334,7 @@ public abstract class BaseServiceRecordProcessor
      * @throws TextParseException
      */
     protected Name getServiceName() throws TextParseException {
-      String user = getUsername(getPath());
+      String user = RegistryPathUtils.getUsername(getPath());
       String service =
           String.format("%s.%s.%s",
               RegistryPathUtils.lastPathEntry(getPath()),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
index 9a24f1c..4346c9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
@@ -80,7 +80,15 @@ public class TestRegistryPathUtils extends Assert {
     assertCreatedPathEquals("/alice", "/alice", "/");
   }
 
-
+  @Test
+  public void testGetUserFromPath() throws Exception {
+    assertEquals("bob", RegistryPathUtils
+        .getUsername("/registry/users/bob/services/yarn-service/test1/"));
+    assertEquals("bob-dev", RegistryPathUtils
+        .getUsername("/registry/users/bob-dev/services/yarn-service/test1"));
+    assertEquals("bob.dev", RegistryPathUtils
+        .getUsername("/registry/users/bob.dev/services/yarn-service/test1"));
+  }
 
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
index a6fd998..c2e6d26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
@@ -155,7 +155,7 @@ where `regionserver-0` is the actual component instance name assigned
by the sys
 | Name | Description |
 | ------------ | ------------- |
 | SERVICE_NAME | name of the service defined by the user
-| USER | user who submits the service |
+| USER | user who submits the service. Note that user name which has "\_" will be converted
to use "-", to conform with DNS hostname RFC format which doesn't allow "\_", and all characters
will be lowercased E.g. "Bob_dev" will be converted to "bob-dev"  |
 | DOMAIN | the domain name for the cluster |
 | COMPONENT_NAME | the name for a given component |
 | COMPONENT_INSTANCE_NAME | the name for a given component instance (i.e. container) |


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message