hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1390199 - in /hadoop/common/branches/HDFS-3077/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-common/src/main/resources/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager...
Date Tue, 25 Sep 2012 22:43:12 GMT
Author: todd
Date: Tue Sep 25 22:43:04 2012
New Revision: 1390199

URL: http://svn.apache.org/viewvc?rev=1390199&view=rev
Log:
Merge trunk into branch

Added:
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/tools/
      - copied from r1390198, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/tools/
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/tools/GetGroupsForTesting.java
      - copied unchanged from r1390198, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/tools/GetGroupsForTesting.java
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/tools/TestGetGroups.java
      - copied unchanged from r1390198, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/tools/TestGetGroups.java
Modified:
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto
    hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/CHANGES.txt?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/CHANGES.txt Tue Sep 25 22:43:04 2012
@@ -34,6 +34,12 @@ Release 2.0.3-alpha - Unreleased 
     YARN-28. Fixed TestCompositeService to not depend on test-order and thus
     made it pass on JDK7 (Thomas Graves via vinodkv).
 
+    YARN-82. Change the default local and log dirs to be based on
+    hadoop.tmp.dir and yarn.log.dir. (Hemanth Yamijala via sseth)
+
+    YARN-53. Added the missing getGroups API to ResourceManager. (Bo Wang via
+    vinodkv)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -100,6 +106,9 @@ Release 0.23.4 - UNRELEASED
 
   BUG FIXES
 
+    YARN-88. DefaultContainerExecutor can fail to set proper permissions.
+    (Jason Lowe via sseth)
+
 Release 0.23.3 - Unreleased 
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Tue Sep 25 22:43:04 2012
@@ -328,7 +328,7 @@
       be subdirectories of this.
    </description>
     <name>yarn.nodemanager.local-dirs</name>
-    <value>/tmp/nm-local-dir</value>
+    <value>${hadoop.tmp.dir}/nm-local-dir</value>
   </property>
 
   <property>
@@ -370,7 +370,7 @@
       stderr, stdin, and syslog generated by that container.
     </description>
     <name>yarn.nodemanager.log-dirs</name>
-    <value>/tmp/logs</value>
+    <value>${yarn.log.dir}/userlogs</value>
   </property>
 
   <property>

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
Tue Sep 25 22:43:04 2012
@@ -109,6 +109,7 @@ public class DefaultContainerExecutor ex
       String userName, String appId, Path containerWorkDir,
       List<String> localDirs, List<String> logDirs) throws IOException {
 
+    FsPermission dirPerm = new FsPermission(APPDIR_PERM);
     ContainerId containerId = container.getContainerID();
 
     // create container dirs on all disks
@@ -123,7 +124,7 @@ public class DefaultContainerExecutor ex
       Path appCacheDir = new Path(userdir, ContainerLocalizer.APPCACHE);
       Path appDir = new Path(appCacheDir, appIdStr);
       Path containerDir = new Path(appDir, containerIdStr);
-      lfs.mkdir(containerDir, null, false);
+      createDir(containerDir, dirPerm, false);
     }
 
     // Create the container log-dirs on all disks
@@ -131,7 +132,7 @@ public class DefaultContainerExecutor ex
 
     Path tmpDir = new Path(containerWorkDir,
         YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
-    lfs.mkdir(tmpDir, null, false);
+    createDir(tmpDir, dirPerm, false);
 
     // copy launch script to work dir
     Path launchDst =
@@ -286,20 +287,20 @@ public class DefaultContainerExecutor ex
   }
 
   /** Permissions for user dir.
-   * $loaal.dir/usercache/$user */
-  private static final short USER_PERM = (short)0750;
+   * $local.dir/usercache/$user */
+  static final short USER_PERM = (short)0750;
   /** Permissions for user appcache dir.
-   * $loaal.dir/usercache/$user/appcache */
-  private static final short APPCACHE_PERM = (short)0710;
+   * $local.dir/usercache/$user/appcache */
+  static final short APPCACHE_PERM = (short)0710;
   /** Permissions for user filecache dir.
-   * $loaal.dir/usercache/$user/filecache */
-  private static final short FILECACHE_PERM = (short)0710;
+   * $local.dir/usercache/$user/filecache */
+  static final short FILECACHE_PERM = (short)0710;
   /** Permissions for user app dir.
-   * $loaal.dir/usercache/$user/filecache */
-  private static final short APPDIR_PERM = (short)0710;
+   * $local.dir/usercache/$user/appcache/$appId */
+  static final short APPDIR_PERM = (short)0710;
   /** Permissions for user log dir.
    * $logdir/$user/$appId */
-  private static final short LOGDIR_PERM = (short)0710;
+  static final short LOGDIR_PERM = (short)0710;
 
   private Path getFirstApplicationDir(List<String> localDirs, String user,
       String appId) {
@@ -324,20 +325,28 @@ public class DefaultContainerExecutor ex
         ContainerLocalizer.FILECACHE);
   }
 
+  private void createDir(Path dirPath, FsPermission perms,
+      boolean createParent) throws IOException {
+    lfs.mkdir(dirPath, perms, createParent);
+    if (!perms.equals(perms.applyUMask(lfs.getUMask()))) {
+      lfs.setPermission(dirPath, perms);
+    }
+  }
+
   /**
    * Initialize the local directories for a particular user.
-   * <ul>
+   * <ul>.mkdir
    * <li>$local.dir/usercache/$user</li>
    * </ul>
    */
-  private void createUserLocalDirs(List<String> localDirs, String user)
+  void createUserLocalDirs(List<String> localDirs, String user)
       throws IOException {
     boolean userDirStatus = false;
     FsPermission userperms = new FsPermission(USER_PERM);
     for (String localDir : localDirs) {
       // create $local.dir/usercache/$user and its immediate parent
       try {
-        lfs.mkdir(getUserCacheDir(new Path(localDir), user), userperms, true);
+        createDir(getUserCacheDir(new Path(localDir), user), userperms, true);
       } catch (IOException e) {
         LOG.warn("Unable to create the user directory : " + localDir, e);
         continue;
@@ -359,7 +368,7 @@ public class DefaultContainerExecutor ex
    * <li>$local.dir/usercache/$user/filecache</li>
    * </ul>
    */
-  private void createUserCacheDirs(List<String> localDirs, String user)
+  void createUserCacheDirs(List<String> localDirs, String user)
       throws IOException {
     LOG.info("Initializing user " + user);
 
@@ -373,7 +382,7 @@ public class DefaultContainerExecutor ex
       Path localDirPath = new Path(localDir);
       final Path appDir = getAppcacheDir(localDirPath, user);
       try {
-        lfs.mkdir(appDir, appCachePerms, true);
+        createDir(appDir, appCachePerms, true);
         appcacheDirStatus = true;
       } catch (IOException e) {
         LOG.warn("Unable to create app cache directory : " + appDir, e);
@@ -381,7 +390,7 @@ public class DefaultContainerExecutor ex
       // create $local.dir/usercache/$user/filecache
       final Path distDir = getFileCacheDir(localDirPath, user);
       try {
-        lfs.mkdir(distDir, fileperms, true);
+        createDir(distDir, fileperms, true);
         distributedCacheDirStatus = true;
       } catch (IOException e) {
         LOG.warn("Unable to create file cache directory : " + distDir, e);
@@ -406,7 +415,7 @@ public class DefaultContainerExecutor ex
    * </ul>
    * @param localDirs 
    */
-  private void createAppDirs(List<String> localDirs, String user, String appId)
+  void createAppDirs(List<String> localDirs, String user, String appId)
       throws IOException {
     boolean initAppDirStatus = false;
     FsPermission appperms = new FsPermission(APPDIR_PERM);
@@ -414,7 +423,7 @@ public class DefaultContainerExecutor ex
       Path fullAppDir = getApplicationDir(new Path(localDir), user, appId);
       // create $local.dir/usercache/$user/appcache/$appId
       try {
-        lfs.mkdir(fullAppDir, appperms, true);
+        createDir(fullAppDir, appperms, true);
         initAppDirStatus = true;
       } catch (IOException e) {
         LOG.warn("Unable to create app directory " + fullAppDir.toString(), e);
@@ -430,7 +439,7 @@ public class DefaultContainerExecutor ex
   /**
    * Create application log directories on all disks.
    */
-  private void createAppLogDirs(String appId, List<String> logDirs)
+  void createAppLogDirs(String appId, List<String> logDirs)
       throws IOException {
 
     boolean appLogDirStatus = false;
@@ -439,7 +448,7 @@ public class DefaultContainerExecutor ex
       // create $log.dir/$appid
       Path appLogDir = new Path(rootLogDir, appId);
       try {
-        lfs.mkdir(appLogDir, appLogDirPerms, true);
+        createDir(appLogDir, appLogDirPerms, true);
       } catch (IOException e) {
         LOG.warn("Unable to create the app-log directory : " + appLogDir, e);
         continue;
@@ -455,7 +464,7 @@ public class DefaultContainerExecutor ex
   /**
    * Create application log directories on all disks.
    */
-  private void createContainerLogDirs(String appId, String containerId,
+  void createContainerLogDirs(String appId, String containerId,
       List<String> logDirs) throws IOException {
 
     boolean containerLogDirStatus = false;
@@ -465,7 +474,7 @@ public class DefaultContainerExecutor ex
       Path appLogDir = new Path(rootLogDir, appId);
       Path containerLogDir = new Path(appLogDir, containerId);
       try {
-        lfs.mkdir(containerLogDir, containerLogDirPerms, true);
+        createDir(containerLogDir, containerLogDirPerms, true);
       } catch (IOException e) {
         LOG.warn("Unable to create the container-log directory : "
             + appLogDir, e);

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
Tue Sep 25 22:43:04 2012
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager;
 
+import java.io.FileNotFoundException;
 import java.io.InputStream;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -26,8 +27,11 @@ import java.util.EnumSet;
 import java.util.List;
 import java.util.Random;
 
+import junit.framework.Assert;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileContext;
@@ -38,6 +42,7 @@ import org.apache.hadoop.fs.permission.F
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.FakeFSDataInputStream;
 
 import static org.apache.hadoop.fs.CreateFlag.*;
@@ -91,11 +96,16 @@ public class TestDefaultContainerExecuto
   }
   */
 
+  private static final Path BASE_TMP_PATH = new Path("target",
+      TestDefaultContainerExecutor.class.getSimpleName());
+
   @AfterClass
   public static void deleteTmpFiles() throws IOException {
     FileContext lfs = FileContext.getLocalFSFileContext();
-    lfs.delete(new Path("target",
-          TestDefaultContainerExecutor.class.getSimpleName()), true);
+    try {
+      lfs.delete(BASE_TMP_PATH, true);
+    } catch (FileNotFoundException e) {
+    }
   }
 
   byte[] createTmpFile(Path dst, Random r, int len)
@@ -116,6 +126,71 @@ public class TestDefaultContainerExecuto
     return bytes;
   }
 
+  @Test
+  public void testDirPermissions() throws Exception {
+    deleteTmpFiles();
+
+    final String user = "somebody";
+    final String appId = "app_12345_123";
+    final FsPermission userCachePerm = new FsPermission(
+        DefaultContainerExecutor.USER_PERM);
+    final FsPermission appCachePerm = new FsPermission(
+        DefaultContainerExecutor.APPCACHE_PERM);
+    final FsPermission fileCachePerm = new FsPermission(
+        DefaultContainerExecutor.FILECACHE_PERM);
+    final FsPermission appDirPerm = new FsPermission(
+        DefaultContainerExecutor.APPDIR_PERM);
+    final FsPermission logDirPerm = new FsPermission(
+        DefaultContainerExecutor.LOGDIR_PERM);
+    List<String> localDirs = new ArrayList<String>();
+    localDirs.add(new Path(BASE_TMP_PATH, "localDirA").toString());
+    localDirs.add(new Path(BASE_TMP_PATH, "localDirB").toString());
+    List<String> logDirs = new ArrayList<String>();
+    logDirs.add(new Path(BASE_TMP_PATH, "logDirA").toString());
+    logDirs.add(new Path(BASE_TMP_PATH, "logDirB").toString());
+
+    Configuration conf = new Configuration();
+    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
+    FileContext lfs = FileContext.getLocalFSFileContext(conf);
+    DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs);
+    executor.init();
+
+    try {
+      executor.createUserLocalDirs(localDirs, user);
+      executor.createUserCacheDirs(localDirs, user);
+      executor.createAppDirs(localDirs, user, appId);
+
+      for (String dir : localDirs) {
+        FileStatus stats = lfs.getFileStatus(
+            new Path(new Path(dir, ContainerLocalizer.USERCACHE), user));
+        Assert.assertEquals(userCachePerm, stats.getPermission());
+      }
+
+      for (String dir : localDirs) {
+        Path userCachePath = new Path(
+            new Path(dir, ContainerLocalizer.USERCACHE), user);
+        Path appCachePath = new Path(userCachePath,
+            ContainerLocalizer.APPCACHE);
+        FileStatus stats = lfs.getFileStatus(appCachePath);
+        Assert.assertEquals(appCachePerm, stats.getPermission());
+        stats = lfs.getFileStatus(
+            new Path(userCachePath, ContainerLocalizer.FILECACHE));
+        Assert.assertEquals(fileCachePerm, stats.getPermission());
+        stats = lfs.getFileStatus(new Path(appCachePath, appId));
+        Assert.assertEquals(appDirPerm, stats.getPermission());
+      }
+
+      executor.createAppLogDirs(appId, logDirs);
+
+      for (String dir : logDirs) {
+        FileStatus stats = lfs.getFileStatus(new Path(dir, appId));
+        Assert.assertEquals(logDirPerm, stats.getPermission());
+      }
+    } finally {
+      deleteTmpFiles();
+    }
+  }
+
 //  @Test
 //  public void testInit() throws IOException, InterruptedException {
 //    Configuration conf = new Configuration();

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
Tue Sep 25 22:43:04 2012
@@ -271,5 +271,10 @@ public class AdminService extends Abstra
       PolicyProvider policyProvider) {
     this.server.refreshServiceAcl(configuration, policyProvider);
   }
+
+  @Override
+  public String[] getGroupsForUser(String user) throws IOException {
+    return UserGroupInformation.createRemoteUser(user).getGroupNames();
+  }
   
 }

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java
Tue Sep 25 22:43:04 2012
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.api;
 
+import org.apache.hadoop.tools.GetUserMappingsProtocol;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshAdminAclsRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshAdminAclsResponse;
@@ -32,7 +33,7 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
 
-public interface RMAdminProtocol {
+public interface RMAdminProtocol extends GetUserMappingsProtocol {
   public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) 
   throws YarnRemoteException;
   

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java
Tue Sep 25 22:43:04 2012
@@ -19,14 +19,16 @@
 package org.apache.hadoop.yarn.server.resourcemanager.api.impl.pb.client;
 
 import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
 import java.net.InetSocketAddress;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserResponseProto;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsRequestProto;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesRequestProto;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshQueuesRequestProto;
@@ -154,5 +156,18 @@ public class RMAdminProtocolPBClientImpl
     }
   }
 
+  @Override
+  public String[] getGroupsForUser(String user) throws IOException {
+    GetGroupsForUserRequestProto requestProto = 
+        GetGroupsForUserRequestProto.newBuilder().setUser(user).build();
+    try {
+      GetGroupsForUserResponseProto responseProto =
+          proxy.getGroupsForUser(null, requestProto);
+      return (String[]) responseProto.getGroupsList().toArray(
+          new String[responseProto.getGroupsCount()]);
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
   
 }

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java
Tue Sep 25 22:43:04 2012
@@ -18,9 +18,9 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.api.impl.pb.service;
 
+import java.io.IOException;
+
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshServiceAclsRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshServiceAclsResponseProto;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.*;
 import org.apache.hadoop.yarn.server.resourcemanager.api.RMAdminProtocol;
 import org.apache.hadoop.yarn.server.resourcemanager.api.RMAdminProtocolPB;
@@ -139,4 +139,22 @@ public class RMAdminProtocolPBServiceImp
       }
   }
 
+  @Override
+  public GetGroupsForUserResponseProto getGroupsForUser(
+      RpcController controller, GetGroupsForUserRequestProto request)
+      throws ServiceException {
+    String user = request.getUser();
+    try {
+      String[] groups = real.getGroupsForUser(user);
+      GetGroupsForUserResponseProto.Builder responseBuilder =
+          GetGroupsForUserResponseProto.newBuilder();
+      for (String group : groups) {
+        responseBuilder.addGroups(group);
+      }
+      return responseBuilder.build();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
 }

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java
Tue Sep 25 22:43:04 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
+import java.util.Arrays;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -63,6 +64,7 @@ public class RMAdmin extends Configured 
       " [-refreshUserToGroupsMappings]" +
       " [-refreshAdminAcls]" +
       " [-refreshServiceAcl]" +
+      " [-getGroup [username]]" +
       " [-help [cmd]]\n";
 
     String refreshQueues =
@@ -81,12 +83,16 @@ public class RMAdmin extends Configured 
 
     String refreshAdminAcls =
       "-refreshAdminAcls: Refresh acls for administration of ResourceManager\n";
-    String help = "-help [cmd]: \tDisplays help for the given command or all commands if
none\n" +
-    "\t\tis specified.\n";
 
     String refreshServiceAcl = 
-        "-refreshServiceAcl: Reload the service-level authorization policy file\n" +
-        "\t\tResoureceManager will reload the authorization policy file.\n";
+      "-refreshServiceAcl: Reload the service-level authorization policy file\n" +
+      "\t\tResoureceManager will reload the authorization policy file.\n";
+    
+    String getGroups = 
+      "-getGroups [username]: Get the groups which given user belongs to\n";
+
+    String help = "-help [cmd]: \tDisplays help for the given command or all commands if
none\n" +
+      "\t\tis specified.\n";
 
     if ("refreshQueues".equals(cmd)) {
       System.out.println(refreshQueues);
@@ -100,6 +106,8 @@ public class RMAdmin extends Configured 
       System.out.println(refreshAdminAcls);
     } else if ("refreshServiceAcl".equals(cmd)) {
       System.out.println(refreshServiceAcl);
+    } else if ("getGroups".equals(cmd)) {
+      System.out.println(getGroups);
     } else if ("help".equals(cmd)) {
       System.out.println(help);
     } else {
@@ -110,6 +118,7 @@ public class RMAdmin extends Configured 
       System.out.println(refreshSuperUserGroupsConfiguration);
       System.out.println(refreshAdminAcls);
       System.out.println(refreshServiceAcl);
+      System.out.println(getGroups);
       System.out.println(help);
       System.out.println();
       ToolRunner.printGenericCommandUsage(System.out);
@@ -133,6 +142,8 @@ public class RMAdmin extends Configured 
       System.err.println("Usage: java RMAdmin" + " [-refreshAdminAcls]");
     } else if ("-refreshService".equals(cmd)){
       System.err.println("Usage: java RMAdmin" + " [-refreshServiceAcl]");
+    } else if ("-getGroups".equals(cmd)){
+      System.err.println("Usage: java RMAdmin" + " [-getGroups [username]]");
     } else {
       System.err.println("Usage: java RMAdmin");
       System.err.println("           [-refreshQueues]");
@@ -141,6 +152,7 @@ public class RMAdmin extends Configured 
       System.err.println("           [-refreshSuperUserGroupsConfiguration]");
       System.err.println("           [-refreshAdminAcls]");
       System.err.println("           [-refreshServiceAcl]");
+      System.err.println("           [-getGroups [username]]");
       System.err.println("           [-help [cmd]]");
       System.err.println();
       ToolRunner.printGenericCommandUsage(System.err);
@@ -229,6 +241,27 @@ public class RMAdmin extends Configured 
     return 0;
   }
   
+  private int getGroups(String[] usernames) throws IOException {
+    // Get groups users belongs to
+    RMAdminProtocol adminProtocol = createAdminProtocol();
+
+    if (usernames.length == 0) {
+      usernames = new String[] { UserGroupInformation.getCurrentUser().getUserName() };
+    }
+    
+    for (String username : usernames) {
+      StringBuilder sb = new StringBuilder();
+      sb.append(username + " :");
+      for (String group : adminProtocol.getGroupsForUser(username)) {
+        sb.append(" ");
+        sb.append(group);
+      }
+      System.out.println(sb);
+    }
+    
+    return 0;
+  }
+  
   @Override
   public int run(String[] args) throws Exception {
     if (args.length < 1) {
@@ -251,7 +284,7 @@ public class RMAdmin extends Configured 
         return exitCode;
       }
     }
-
+    
     exitCode = 0;
     try {
       if ("-refreshQueues".equals(cmd)) {
@@ -266,6 +299,9 @@ public class RMAdmin extends Configured 
         exitCode = refreshAdminAcls();
       } else if ("-refreshServiceAcl".equals(cmd)) {
         exitCode = refreshServiceAcls();
+      } else if ("-getGroups".equals(cmd)) {
+        String[] usernames = Arrays.copyOfRange(args, i, args.length);
+        exitCode = getGroups(usernames);
       } else if ("-help".equals(cmd)) {
         if (i < args.length) {
           printUsage(args[i]);

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto
Tue Sep 25 22:43:04 2012
@@ -30,4 +30,5 @@ service RMAdminProtocolService {
   rpc refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequestProto) returns (RefreshUserToGroupsMappingsResponseProto);
   rpc refreshAdminAcls(RefreshAdminAclsRequestProto) returns (RefreshAdminAclsResponseProto);
   rpc refreshServiceAcls(RefreshServiceAclsRequestProto) returns (RefreshServiceAclsResponseProto);
+  rpc getGroupsForUser(GetGroupsForUserRequestProto) returns (GetGroupsForUserResponseProto);
 }

Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto?rev=1390199&r1=1390198&r2=1390199&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto
Tue Sep 25 22:43:04 2012
@@ -52,3 +52,10 @@ message RefreshServiceAclsRequestProto {
 message RefreshServiceAclsResponseProto {
 }
 
+message GetGroupsForUserRequestProto {
+  required string user = 1;
+}
+
+message GetGroupsForUserResponseProto {
+  repeated string groups = 1;
+}



Mime
View raw message