hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r951226 - in /hadoop/mapreduce/trunk: ./ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapred/tools/ src/test/mapred/org/apache/hadoop/security/
Date Fri, 04 Jun 2010 00:32:31 GMT
Author: shv
Date: Fri Jun  4 00:32:31 2010
New Revision: 951226

URL: http://svn.apache.org/viewvc?rev=951226&view=rev
Log:
MAPREDUCE-1836. Refresh for proxy superuser config (mr part for HDFS-1096). Contributed by
Boris Shkolnik.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=951226&r1=951225&r2=951226&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Jun  4 00:32:31 2010
@@ -66,6 +66,9 @@ Trunk (unreleased changes)
     MAPREDUCE-1599. Fixes MRBench so that it reuses tokens across jobs
     correctly. (Jitendra Nath Pandey via ddas)
 
+    MAPREDUCE-1836. Refresh for proxy superuser config (mr part for HDFS-1096).
+    (Boris Shkolnik via shv)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=951226&r1=951225&r2=951226&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Jun  4 00:32:31
2010
@@ -47,9 +47,11 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.Vector;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.security.auth.login.LoginException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -81,7 +83,6 @@ import org.apache.hadoop.mapreduce.TaskT
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
 import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
-import org.apache.hadoop.security.TokenStorage;
 import org.apache.hadoop.mapreduce.security.token.DelegationTokenRenewal;
 import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
 import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
@@ -98,9 +99,11 @@ import org.apache.hadoop.net.NodeBase;
 import org.apache.hadoop.net.ScriptBasedMapping;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.Groups;
-import org.apache.hadoop.security.RefreshUserToGroupMappingsProtocol;
+import org.apache.hadoop.security.RefreshUserMappingsProtocol;
+import org.apache.hadoop.security.TokenStorage;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.authorize.ProxyUsers;
 import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
 import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
 import org.apache.hadoop.security.token.Token;
@@ -117,7 +120,7 @@ import org.apache.hadoop.util.VersionInf
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class JobTracker implements MRConstants, InterTrackerProtocol,
-    ClientProtocol, TaskTrackerManager, RefreshUserToGroupMappingsProtocol,
+    ClientProtocol, TaskTrackerManager, RefreshUserMappingsProtocol,
     RefreshAuthorizationPolicyProtocol, AdminOperationsProtocol, JTConfig {
 
   static{
@@ -297,8 +300,8 @@ public class JobTracker implements MRCon
       return RefreshAuthorizationPolicyProtocol.versionID;
     } else if (protocol.equals(AdminOperationsProtocol.class.getName())){
       return AdminOperationsProtocol.versionID;
-    } else if (protocol.equals(RefreshUserToGroupMappingsProtocol.class.getName())){
-      return RefreshUserToGroupMappingsProtocol.versionID;
+    } else if (protocol.equals(RefreshUserMappingsProtocol.class.getName())){
+      return RefreshUserMappingsProtocol.versionID;
     } else {
       throw new IOException("Unknown protocol to job tracker: " + protocol);
     }
@@ -4411,7 +4414,13 @@ public class JobTracker implements MRCon
             limitMaxMemForReduceTasks).append(")"));
   }
 
-    
+  @Override
+  public void refreshSuperUserGroupsConfiguration(Configuration conf) {
+    LOG.info("Refreshing superuser proxy groups mapping ");
+
+    ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
+  }
+
   @Override
   public void refreshUserToGroupsMappings(Configuration conf) throws IOException {
     LOG.info("Refreshing all user-to-groups mappings. Requested by user: " + 

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java?rev=951226&r1=951225&r2=951226&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java
(original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java
Fri Jun  4 00:32:31 2010
@@ -20,7 +20,7 @@ package org.apache.hadoop.mapred;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
-import org.apache.hadoop.security.RefreshUserToGroupMappingsProtocol;
+import org.apache.hadoop.security.RefreshUserMappingsProtocol;
 import org.apache.hadoop.security.authorize.PolicyProvider;
 import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
 import org.apache.hadoop.security.authorize.Service;
@@ -41,8 +41,8 @@ public class MapReducePolicyProvider ext
                   TaskUmbilicalProtocol.class),
       new Service("security.refresh.policy.protocol.acl", 
                   RefreshAuthorizationPolicyProtocol.class),
-      new Service("security.refresh.usertogroups.mappings.protocol.acl", 
-                  RefreshUserToGroupMappingsProtocol.class),
+      new Service("security.refresh.user.mappings.protocol.acl", 
+                  RefreshUserMappingsProtocol.class),
       new Service("security.admin.operations.protocol.acl", 
                   AdminOperationsProtocol.class),
   };

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java?rev=951226&r1=951225&r2=951226&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java Fri Jun  4
00:32:31 2010
@@ -30,7 +30,7 @@ import org.apache.hadoop.mapred.AdminOpe
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.JobTracker;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.RefreshUserToGroupMappingsProtocol;
+import org.apache.hadoop.security.RefreshUserMappingsProtocol;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
 import org.apache.hadoop.util.Tool;
@@ -59,7 +59,8 @@ public class MRAdmin extends Configured 
     String summary = "hadoop mradmin is the command to execute Map-Reduce administrative
commands.\n" +
     "The full syntax is: \n\n" +
     "hadoop mradmin [-refreshServiceAcl] [-refreshQueues] " +
-    "[-refreshNodes] [-refreshUserToGroupsMappings] [-help [cmd]]\n"; 
+    "[-refreshNodes] [-refreshUserToGroupsMappings] " +
+    "[-refreshSuperUserGroupsConfiguration] [-help [cmd]]\n"; 
 
   String refreshServiceAcl = "-refreshServiceAcl: Reload the service-level authorization
policy file\n" +
     "\t\tJobtracker will reload the authorization policy file.\n";
@@ -72,6 +73,9 @@ public class MRAdmin extends Configured 
   String refreshUserToGroupsMappings = 
     "-refreshUserToGroupsMappings: Refresh user-to-groups mappings\n";
   
+  String refreshSuperUserGroupsConfiguration = 
+    "-refreshSuperUserGroupsConfiguration: Refresh superuser proxy groups mappings\n";
+    
   String refreshNodes =
     "-refreshNodes: Refresh the hosts information at the jobtracker.\n";
   
@@ -86,12 +90,16 @@ public class MRAdmin extends Configured 
     System.out.println(refreshUserToGroupsMappings);
   }  else if ("refreshNodes".equals(cmd)) {
     System.out.println(refreshNodes);
+  } else if ("refreshSuperUserGroupsConfiguration".equals(cmd)) {
+    System.out.println(refreshSuperUserGroupsConfiguration);
   } else if ("help".equals(cmd)) {
     System.out.println(help);
   } else {
     System.out.println(summary);
     System.out.println(refreshServiceAcl);
     System.out.println(refreshQueues);
+    System.out.println(refreshUserToGroupsMappings);
+    System.out.println(refreshSuperUserGroupsConfiguration);
     System.out.println(refreshNodes);
     System.out.println(help);
     System.out.println();
@@ -111,6 +119,9 @@ public class MRAdmin extends Configured 
       System.err.println("Usage: java MRAdmin" + " [-refreshQueues]");
     } else if ("-refreshUserToGroupsMappings".equals(cmd)) {
       System.err.println("Usage: java MRAdmin" + " [-refreshUserToGroupsMappings]");
+    } else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) {
+      System.err.println("Usage: java DFSAdmin" +
+          " [-refreshSuperUserGroupsConfiguration]");
     } else if ("-refreshNodes".equals(cmd)) {
       System.err.println("Usage: java MRAdmin" + " [-refreshNodes]");
     } else {
@@ -118,6 +129,7 @@ public class MRAdmin extends Configured 
       System.err.println("           [-refreshServiceAcl]");
       System.err.println("           [-refreshQueues]");
       System.err.println("           [-refreshUserToGroupsMappings]");
+      System.err.println("           [-refreshSuperUserGroupsConfiguration]");
       System.err.println("           [-refreshNodes]");
       System.err.println("           [-help [cmd]]");
       System.err.println();
@@ -155,6 +167,39 @@ public class MRAdmin extends Configured 
     
     return 0;
   }
+  
+
+  /**
+   * refreshSuperUserGroupsConfiguration {@link JobTracker}.
+   * @return exitcode 0 on success, non-zero on failure
+   * @throws IOException
+   */
+  public int refreshSuperUserGroupsConfiguration() throws IOException {
+    // Get the current configuration
+    Configuration conf = getConf();
+
+    // for security authorization
+    // server principal for this call   
+    // should be JT's one.
+    JobConf jConf = new JobConf(conf);
+    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, 
+        jConf.get(JobTracker.JT_USER_NAME, ""));
+
+    // Create the client
+    RefreshUserMappingsProtocol refreshProtocol = 
+      (RefreshUserMappingsProtocol) 
+      RPC.getProxy(RefreshUserMappingsProtocol.class, 
+          RefreshUserMappingsProtocol.versionID, 
+          JobTracker.getAddress(conf), getUGI(conf), conf,
+          NetUtils.getSocketFactory(conf, 
+              RefreshUserMappingsProtocol.class));
+
+    // Refresh the user-to-groups mappings
+    refreshProtocol.refreshSuperUserGroupsConfiguration(conf);
+
+    return 0;
+  }
+
 
   /**
    * Refresh the user-to-groups mappings on the {@link JobTracker}.
@@ -173,14 +218,14 @@ public class MRAdmin extends Configured 
         jConf.get(JobTracker.JT_USER_NAME, ""));
 
     // Create the client
-    RefreshUserToGroupMappingsProtocol refreshProtocol = 
-      (RefreshUserToGroupMappingsProtocol) 
-      RPC.getProxy(RefreshUserToGroupMappingsProtocol.class, 
-                   RefreshUserToGroupMappingsProtocol.versionID, 
-                   JobTracker.getAddress(conf), getUGI(conf), conf,
-                   NetUtils.getSocketFactory(conf, 
-                                             RefreshUserToGroupMappingsProtocol.class));
-    
+    RefreshUserMappingsProtocol refreshProtocol = 
+      (RefreshUserMappingsProtocol) 
+      RPC.getProxy(RefreshUserMappingsProtocol.class, 
+          RefreshUserMappingsProtocol.versionID, 
+          JobTracker.getAddress(conf), getUGI(conf), conf,
+          NetUtils.getSocketFactory(conf, 
+              RefreshUserMappingsProtocol.class));
+
     // Refresh the user-to-groups mappings
     refreshProtocol.refreshUserToGroupsMappings(conf);
     
@@ -245,7 +290,10 @@ public class MRAdmin extends Configured 
     // verify that we have enough command line parameters
     //
     if ("-refreshServiceAcl".equals(cmd) || "-refreshQueues".equals(cmd) ||
-        "-refreshNodes".equals(cmd) || "-refreshUserToGroupsMappings".equals(cmd)) {
+        "-refreshNodes".equals(cmd) ||
+        "-refreshUserToGroupsMappings".equals(cmd) ||
+        "-refreshSuperUserGroupsConfiguration".equals(cmd)
+    ) {
       if (args.length != 1) {
         printUsage(cmd);
         return exitCode;
@@ -260,6 +308,8 @@ public class MRAdmin extends Configured 
         exitCode = refreshQueues();
       } else if ("-refreshUserToGroupsMappings".equals(cmd)) {
         exitCode = refreshUserToGroupsMappings();
+      } else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) {
+        exitCode = refreshSuperUserGroupsConfiguration();
       } else if ("-refreshNodes".equals(cmd)) {
         exitCode = refreshNodes();
       } else if ("-help".equals(cmd)) {

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java?rev=951226&r1=951225&r2=951226&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
Fri Jun  4 00:32:31 2010
@@ -20,6 +20,10 @@ package org.apache.hadoop.security;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 
 import java.io.IOException;
 import java.net.URI;
@@ -33,10 +37,13 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapred.tools.MRAdmin;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.authorize.ProxyUsers;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -142,4 +149,80 @@ public class TestMapredGroupMappingServi
       assertFalse("Should be different group ", g3.get(i).equals(g4.get(i)));
     }
   }
+  
+  @Test
+  public void testRefreshSuperUserGroupsConfiguration() throws Exception {
+    final String SUPER_USER = "super_user";
+    final String [] GROUP_NAMES1 = new String [] {"gr1" , "gr2"};
+    final String [] GROUP_NAMES2 = new String [] {"gr3" , "gr4"};
+    
+    //keys in conf
+    String userKeyGroups = ProxyUsers.getProxySuperuserGroupConfKey(SUPER_USER);
+    String userKeyHosts = ProxyUsers.getProxySuperuserIpConfKey (SUPER_USER);
+    
+    config.set(userKeyGroups, "gr3,gr4,gr5"); // superuser can proxy for this group
+    config.set(userKeyHosts,"127.0.0.1");
+    
+    UserGroupInformation ugi1 = mock(UserGroupInformation.class);
+    UserGroupInformation ugi2 = mock(UserGroupInformation.class);
+    UserGroupInformation suUgi = mock(UserGroupInformation.class);
+    when(ugi1.getRealUser()).thenReturn(suUgi);
+    when(ugi2.getRealUser()).thenReturn(suUgi);
+
+    when(suUgi.getShortUserName()).thenReturn(SUPER_USER); // super user
+    when(suUgi.getUserName()).thenReturn(SUPER_USER+"L"); // super user
+     
+    when(ugi1.getShortUserName()).thenReturn("user1");
+    when(ugi2.getShortUserName()).thenReturn("user2");
+    
+    when(ugi1.getUserName()).thenReturn("userL1");
+    when(ugi2.getUserName()).thenReturn("userL2");
+   
+    // set groups for users
+    when(ugi1.getGroupNames()).thenReturn(GROUP_NAMES1);
+    when(ugi2.getGroupNames()).thenReturn(GROUP_NAMES2);
+   
+    
+    // check before
+    try {
+      ProxyUsers.authorize(ugi1, "127.0.0.1", config);
+      fail("first auth for " + ugi1.getShortUserName() + " should've failed ");
+    } catch (AuthorizationException e) {
+      // expected
+      System.err.println("auth for " + ugi1.getUserName() + " failed");
+    }
+    try {
+      ProxyUsers.authorize(ugi2, "127.0.0.1", config);
+      System.err.println("auth for " + ugi2.getUserName() + " succeeded");
+      // expected
+    } catch (AuthorizationException e) {
+      fail("first auth for " + ugi2.getShortUserName() + " should've succeeded: " + e.getLocalizedMessage());
+    }
+    
+    MRAdmin admin = new MRAdmin(config);
+    String [] args = new String[]{"-refreshSuperUserGroupsConfiguration"};
+    NameNode nn = cluster.getNameNode();
+    Configuration conf = new Configuration(config);
+    conf.set(userKeyGroups, "gr2"); // superuser can proxy for this group
+    admin.setConf(conf);
+    admin.run(args);
+    
+    //check after...
+    
+    try {
+      ProxyUsers.authorize(ugi2, "127.0.0.1", config);
+      fail("second auth for " + ugi2.getShortUserName() + " should've failed ");
+    } catch (AuthorizationException e) {
+      // expected
+      System.err.println("auth for " + ugi2.getUserName() + " failed");
+    }
+    try {
+      ProxyUsers.authorize(ugi1, "127.0.0.1", config);
+      System.err.println("auth for " + ugi1.getUserName() + " succeeded");
+      // expected
+    } catch (AuthorizationException e) {
+      fail("second auth for " + ugi1.getShortUserName() + " should've succeeded: " + e.getLocalizedMessage());
+    }    
+  }
+
 }



Mime
View raw message