hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r1064894 - in /hadoop/mapreduce/branches/branch-0.22: ./ src/docs/src/documentation/content/xdocs/ src/java/ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapreduce/ src/test/mapred/org/apache/hadoop/mapred/
Date Fri, 28 Jan 2011 21:32:49 GMT
Author: shv
Date: Fri Jan 28 21:32:48 2011
New Revision: 1064894

URL: http://svn.apache.org/viewvc?rev=1064894&view=rev
Log:
MAPREDUCE-1754. Merge -r 1064891:1064892 from trunk to branch 0.22.

Modified:
    hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
    hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
    hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml
    hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java
    hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java
    hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java
    hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java
    hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
    hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java
    hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java

Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Fri Jan 28 21:32:48 2011
@@ -440,6 +440,9 @@ Release 0.22.0 - Unreleased
 
     MAPREDUCE-2283. Add timeout for Raid Tests (Ramkumar Vadali via schen)
 
+    MAPREDUCE-1754. Replace mapred.persmissions.supergroup with an 
+    acl : mapreduce.cluster.administrators (Amareshwari Sriramadasu via shv)
+
 Release 0.21.1 - Unreleased
 
   NEW FEATURES

Modified: hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
Fri Jan 28 21:32:48 2011
@@ -1778,8 +1778,8 @@
           nobody is given access in these properties.</p> 
           
           <p>However, irrespective of the job ACLs configured, a job's owner,
-          the user who started the cluster and members of an admin configured
-          supergroup (<code>mapreduce.cluster.permissions.supergroup</code>)
+          the user who started the cluster and cluster administrators
+          (<code>mapreduce.cluster.administrators</code>)
           and queue administrators of the queue to which the job was submitted
           to (<code>acl-administer-jobs</code>) always have access to view and
           modify a job.</p>
@@ -1809,8 +1809,8 @@
           the queue level ACL, "acl-administer-jobs", configured via
           mapred-queue-acls.xml. The caller will be able to do the operation
           if he/she is part of either queue admins ACL or job modification ACL
-          or the user who started the cluster or a member of an admin configured
-          supergroup (<code>mapreduce.cluster.permissions.supergroup</code>).
+          or the user who started the cluster or a cluster administrator
+          (<code>mapreduce.cluster.administrators</code>).
           </p>
           
           <p>The format of a job level ACL is the same as the format for a

Modified: hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml Fri Jan 28 21:32:48
2011
@@ -967,14 +967,14 @@
     job-level ACL.
 
     Irrespective of this ACL configuration, (a) job-owner, (b) the user who
-    started the cluster, (c) members of an admin configured supergroup
-    configured via mapreduce.cluster.permissions.supergroup and (d) queue
+    started the cluster, (c) cluster administrators
+    configured via mapreduce.cluster.administrators and (d) queue
     administrators of the queue to which this job was submitted to configured
     via acl-administer-jobs for the specific queue in mapred-queues.xml can
     do all the modification operations on a job.
 
     By default, nobody else besides job-owner, the user who started the cluster,
-    members of supergroup and queue administrators can perform modification
+    cluster administrators and queue administrators can perform modification
     operations on a job.
   </description>
 </property>
@@ -1001,14 +1001,14 @@
     user, for e.g., JobStatus, JobProfile, list of jobs in the queue, etc.
 
     Irrespective of this ACL configuration, (a) job-owner, (b) the user who
-    started the cluster, (c) members of an admin configured supergroup
-    configured via mapreduce.cluster.permissions.supergroup and (d) queue
+    started the cluster, (c) cluster administrators
+    configured via mapreduce.cluster.administrators and (d) queue
     administrators of the queue to which this job was submitted to configured
     via acl-administer-jobs for the specific queue in mapred-queues.xml can
     do all the view operations on a job.
 
     By default, nobody else besides job-owner, the user who started the
-    cluster, memebers of supergroup and queue administrators can perform
+    cluster, cluster administrators and queue administrators can perform
     view operations on a job.
   </description>
 </property>

Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java
Fri Jan 28 21:32:48 2011
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapred;
 
 import java.io.IOException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.AuditLogger.Constants;
@@ -36,10 +38,11 @@ import org.apache.hadoop.security.author
 @InterfaceAudience.Private
 class ACLsManager {
 
+  static Log LOG = LogFactory.getLog(ACLsManager.class);
   // MROwner(user who started this mapreduce cluster)'s ugi
   private final UserGroupInformation mrOwner;
-  // members of supergroup are mapreduce cluster administrators
-  private final String superGroup;
+  // mapreduce cluster administrators
+  private final AccessControlList adminAcl;
   
   private final JobACLsManager jobACLsManager;
   private final QueueManager queueManager;
@@ -50,8 +53,15 @@ class ACLsManager {
       QueueManager queueManager) throws IOException {
 
     mrOwner = UserGroupInformation.getCurrentUser();
-    superGroup = conf.get(MRConfig.MR_SUPERGROUP, "supergroup");
+    adminAcl = new AccessControlList(conf.get(MRConfig.MR_ADMINS, " "));
+    adminAcl.addUser(mrOwner.getShortUserName());
     
+    String deprecatedSuperGroup = conf.get(MRConfig.MR_SUPERGROUP);
+    if (deprecatedSuperGroup != null) {
+      LOG.warn(MRConfig.MR_SUPERGROUP + " is deprecated. Use " 
+          + MRConfig.MR_ADMINS + " instead");
+      adminAcl.addGroup(deprecatedSuperGroup);
+    }
     aclsEnabled = conf.getBoolean(MRConfig.MR_ACLS_ENABLED, false);
 
     this.jobACLsManager = jobACLsManager;
@@ -62,8 +72,8 @@ class ACLsManager {
     return mrOwner;
   }
 
-  String getSuperGroup() {
-    return superGroup;
+  AccessControlList getAdminsAcl() {
+    return adminAcl;
   }
 
   JobACLsManager getJobACLsManager() {
@@ -72,20 +82,13 @@ class ACLsManager {
 
   /**
    * Is the calling user an admin for the mapreduce cluster ?
-   * i.e. either cluster owner or member of the supergroup
-   *      mapreduce.cluster.permissions.supergroup.
+   * i.e. either cluster owner or cluster administrator
    * @return true, if user is an admin
    */
   boolean isMRAdmin(UserGroupInformation callerUGI) {
-    if (mrOwner.getShortUserName().equals(callerUGI.getShortUserName())) {
+    if (adminAcl.isUserAllowed(callerUGI)) {
       return true;
     }
-    String[] groups = callerUGI.getGroupNames();
-    for(int i=0; i < groups.length; ++i) {
-      if (groups[i].equals(superGroup)) {
-        return true;
-      }
-    }
     return false;
   }
 

Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java
Fri Jan 28 21:32:48 2011
@@ -62,7 +62,7 @@ class JobACLsManager {
       String aclConfigured = conf.get(aclConfigName);
       if (aclConfigured == null) {
         // If ACLs are not configured at all, we grant no access to anyone. So
-        // jobOwner and superuser/supergroup _only_ can do 'stuff'
+        // jobOwner and cluster administrator _only_ can do 'stuff'
         aclConfigured = " ";
       }
       acls.put(aclName, new AccessControlList(aclConfigured));

Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java
Fri Jan 28 21:32:48 2011
@@ -1444,8 +1444,7 @@ public class JobTracker implements MRCon
     aclsManager = new ACLsManager(conf, new JobACLsManager(conf), queueManager);
 
     LOG.info("Starting jobtracker with owner as " +
-        getMROwner().getShortUserName() + " and supergroup as " +
-        getSuperGroup());
+        getMROwner().getShortUserName());
 
     // Create the scheduler
     Class<? extends TaskScheduler> schedulerClass
@@ -1483,7 +1482,7 @@ public class JobTracker implements MRCon
     int tmpInfoPort = infoSocAddr.getPort();
     this.startTime = clock.getTime();
     infoServer = new HttpServer("job", infoBindAddress, tmpInfoPort, 
-        tmpInfoPort == 0, conf);
+        tmpInfoPort == 0, conf, aclsManager.getAdminsAcl());
     infoServer.setAttribute("job.tracker", this);
     // initialize history parameters.
     jobHistory = new JobHistory();
@@ -4139,9 +4138,9 @@ public class JobTracker implements MRCon
   public synchronized void refreshNodes() throws IOException {
     String user = UserGroupInformation.getCurrentUser().getShortUserName();
     // check access
-    if (!isMRAdmin(UserGroupInformation.getCurrentUser())) {
+    if (!aclsManager.isMRAdmin(UserGroupInformation.getCurrentUser())) {
       AuditLogger.logFailure(user, Constants.REFRESH_NODES,
-          getMROwner() + " " + getSuperGroup(), Constants.JOBTRACKER,
+          aclsManager.getAdminsAcl().toString(), Constants.JOBTRACKER,
           Constants.UNAUTHORIZED_USER);
       throw new AccessControlException(user + 
                                        " is not authorized to refresh nodes.");
@@ -4156,14 +4155,6 @@ public class JobTracker implements MRCon
     return aclsManager.getMROwner();
   }
 
-  String getSuperGroup() {
-    return aclsManager.getSuperGroup();
-  }
-
-  boolean isMRAdmin(UserGroupInformation ugi) {
-    return aclsManager.isMRAdmin(ugi);
-  }
-
   private synchronized void refreshHosts() throws IOException {
     // Reread the config to get HOSTS and HOSTS_EXCLUDE filenames.
     // Update the file names and refresh internal includes and excludes list
@@ -4604,8 +4595,7 @@ public class JobTracker implements MRCon
     aclsManager = new ACLsManager(conf, new JobACLsManager(conf), queueManager);
 
     LOG.info("Starting jobtracker with owner as " +
-        getMROwner().getShortUserName() + " and supergroup as " +
-        getSuperGroup());
+        getMROwner().getShortUserName());
 
     // Create the scheduler
     Class<? extends TaskScheduler> schedulerClass

Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java
Fri Jan 28 21:32:48 2011
@@ -121,7 +121,7 @@ public class TaskLogServlet extends Http
   /**
    * Validates if the given user has job view permissions for this job.
    * conf contains jobOwner and job-view-ACLs.
-   * We allow jobOwner, superUser(i.e. mrOwner) and members of superGroup and
+   * We allow jobOwner, superUser(i.e. mrOwner) and cluster administrators and
    * users and groups specified in configuration using
    * mapreduce.job.acl-view-job to view job.
    */

Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java
Fri Jan 28 21:32:48 2011
@@ -589,6 +589,11 @@ public class TaskTracker 
     }
   }
     
+  
+  int getHttpPort() {
+    return httpPort;
+  }
+
   /**
    * Do the real constructor work here.  It's in a separate method
    * so we can call it again and "recycle" the object after calling
@@ -596,10 +601,8 @@ public class TaskTracker 
    */
   synchronized void initialize() throws IOException, InterruptedException {
 
-    aclsManager = new ACLsManager(fConf, new JobACLsManager(fConf), null);
     LOG.info("Starting tasktracker with owner as " +
-        getMROwner().getShortUserName() + " and supergroup as " +
-        getSuperGroup());
+        aclsManager.getMROwner().getShortUserName());
 
     localFs = FileSystem.getLocal(fConf);
     // use configured nameserver & interface to get local hostname
@@ -739,18 +742,6 @@ public class TaskTracker 
       fConf.getBoolean(TT_OUTOFBAND_HEARBEAT, false);
   }
 
-  UserGroupInformation getMROwner() {
-    return aclsManager.getMROwner();
-  }
-
-  String getSuperGroup() {
-    return aclsManager.getSuperGroup();
-  }
-
-  boolean isMRAdmin(UserGroupInformation ugi) {
-    return aclsManager.isMRAdmin(ugi);
-  }
-
   /**
    * Are ACLs for authorization checks enabled on the MR cluster ?
    */
@@ -1355,13 +1346,14 @@ public class TaskTracker 
     fConf = conf;
     maxMapSlots = conf.getInt(TT_MAP_SLOTS, 2);
     maxReduceSlots = conf.getInt(TT_REDUCE_SLOTS, 2);
+    aclsManager = new ACLsManager(fConf, new JobACLsManager(fConf), null);
     this.jobTrackAddr = JobTracker.getAddress(conf);
     InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(
         conf.get(TT_HTTP_ADDRESS, "0.0.0.0:50060"));
     String httpBindAddress = infoSocAddr.getHostName();
     int httpPort = infoSocAddr.getPort();
     this.server = new HttpServer("task", httpBindAddress, httpPort,
-        httpPort == 0, conf);
+        httpPort == 0, conf, aclsManager.getAdminsAcl());
     workerThreads = conf.getInt(TT_HTTP_THREADS, 40);
     this.shuffleServerMetrics = new ShuffleServerMetrics(conf);
     server.setThreads(1, workerThreads);

Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java
Fri Jan 28 21:32:48 2011
@@ -38,6 +38,9 @@ public interface MRConfig {
   public static final String REDUCEMEMORY_MB = 
     "mapreduce.cluster.reducememory.mb";
   public static final String MR_ACLS_ENABLED = "mapreduce.cluster.acls.enabled";
+  public static final String MR_ADMINS =
+    "mapreduce.cluster.administrators";
+  @Deprecated
   public static final String MR_SUPERGROUP =
     "mapreduce.cluster.permissions.supergroup";
 

Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
Fri Jan 28 21:32:48 2011
@@ -59,7 +59,7 @@ public class TestNodeRefresh extends Tes
   private JobTracker jt = null;
   private String[] hosts = null;
   private String[] trackerHosts = null;
-  private UserGroupInformation owner, user1, user2, user3, user4;
+  private UserGroupInformation owner, user1, user2, user3, user4, user5;
   private static final Log LOG = 
     LogFactory.getLog(TestNodeRefresh.class);
   
@@ -82,6 +82,8 @@ public class TestNodeRefresh extends Tes
                                                        new String[] {"abc"});
       user4= UserGroupInformation.createUserForTesting("user4", 
                                                    new String[] {"supergroup"});
+      user5= UserGroupInformation.createUserForTesting("user5", 
+          new String[] {"user5"});
       conf.setBoolean("dfs.replication.considerLoad", false);
       
       // prepare hosts info
@@ -152,7 +154,7 @@ public class TestNodeRefresh extends Tes
 
   /**
    * Check default value of HOSTS_EXCLUDE. Also check if only 
-   * owner/supergroup user is allowed to this command.
+   * owner is allowed to this command.
    */
   public void testMRRefreshDefault() throws IOException {  
     // start a cluster with 2 hosts and no exclude-hosts file
@@ -182,14 +184,14 @@ public class TestNodeRefresh extends Tes
     assertTrue("Privileged user denied permission for refresh operation",
                success);
 
-    // refresh with super user
+    // refresh with invalid user
     success = false;
     client = getClient(conf, user4);
     try {
       client.refreshNodes();
       success = true;
     } catch (IOException ioe){}
-    assertTrue("Super user denied permission for refresh operation",
+    assertFalse("Invalid user performed privileged refresh operation",
                success);
 
     // check the cluster status and tracker size
@@ -210,13 +212,15 @@ public class TestNodeRefresh extends Tes
   }
 
   /**
-   * Check refresh with a specific user is set in the conf along with supergroup
+   * Check refresh with a specific user/group set in the conf
    */
   public void testMRSuperUsers() throws IOException {  
-    // start a cluster with 1 host and specified superuser and supergroup
+    // start a cluster with 1 host and specified cluster administrators
     Configuration conf = new Configuration();
     // set the supergroup
-    conf.set(MRConfig.MR_SUPERGROUP, "abc");
+    conf.set(MRConfig.MR_SUPERGROUP, "supergroup");
+    // set the admin acl
+    conf.set(MRConfig.MR_ADMINS, "user5 abc");
     startCluster(2, 1, 0, UserGroupInformation.createRemoteUser("user1"), conf);
 
     conf = mr.createJobConf(new JobConf(conf));
@@ -241,22 +245,42 @@ public class TestNodeRefresh extends Tes
     assertTrue("Privileged user denied permission for refresh operation",
                success);
 
-    // refresh with super user
+    // refresh with admin group
     success = false;
     client = getClient(conf, user3);
     try {
       client.refreshNodes();
       success = true;
     } catch (IOException ioe){}
-    assertTrue("Super user denied permission for refresh operation",
+    assertTrue("Admin group member denied permission for refresh operation",
                success);
 
+    // refresh with admin user
+    success = false;
+    client = getClient(conf, user5);
+    try {
+      client.refreshNodes();
+      success = true;
+    } catch (IOException ioe){}
+    assertTrue("Admin user denied permission for refresh operation",
+               success);
+
+    // refresh with deprecated super group member
+    success = false;
+    client = getClient(conf, user4);
+    try {
+      client.refreshNodes();
+      success = true;
+    } catch (IOException ioe){}
+    assertTrue("Deprecated Super group member denied permission for refresh" +
+        " operation", success);
+
     stopCluster();
   }
 
   /**
    * Check node refresh for decommissioning. Check if an allowed host is 
-   * disallowed upon refresh. Also check if only owner/supergroup user is 
+   * disallowed upon refresh. Also check if only owner/cluster administrator is 
    * allowed to fire this command.
    */
   public void testMRRefreshDecommissioning() throws IOException {

Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java
Fri Jan 28 21:32:48 2011
@@ -64,6 +64,10 @@ public class TestQueueManagerWithJobTrac
     deleteQueuesConfigFile();
   }
 
+  String adminUser = "adminUser";
+  String adminGroup = "adminGroup";
+  String deprecatedSuperGroup = "superGroup";
+
   private void startCluster(boolean aclsEnabled)
       throws Exception {
 
@@ -74,6 +78,8 @@ public class TestQueueManagerWithJobTrac
     conf = new Configuration();
     conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
     conf.setBoolean(MRConfig.MR_ACLS_ENABLED, aclsEnabled);
+    conf.set(MRConfig.MR_SUPERGROUP, deprecatedSuperGroup);
+    conf.set(MRConfig.MR_ADMINS, adminUser + " " + adminGroup);
 
     JobConf jobConf = new JobConf(conf);
     String namenode = "file:///";
@@ -122,6 +128,21 @@ public class TestQueueManagerWithJobTrac
     fail("user u1 cannot submit jobs to queue p1:p13");
     } catch (Exception e) {
     }
+
+    // check access to admins
+    job = submitSleepJob(0, 0, 0, 0, true, adminUser+ ",g1",
+        "p1" + NAME_SEPARATOR + "p13", conf);
+    assertTrue("Admin user cannot submit jobs to queue p1:p13",
+        job.isSuccessful());
+    job = submitSleepJob(0, 0, 0, 0, true, "u1,"+ adminGroup,
+        "p1" + NAME_SEPARATOR + "p13", conf);
+    assertTrue("Admin group member cannot submit jobs to queue p1:p13",
+        job.isSuccessful());
+    job = submitSleepJob(0, 0, 0, 0, true, "u1,"+ deprecatedSuperGroup,
+        "p1" + NAME_SEPARATOR + "p13", conf);
+    assertTrue("Deprecated super group member cannot submit jobs to queue" +
+        " p1:p13", job.isSuccessful());
+
     // check for access to submit the job
     try {
       job = submitSleepJob(0, 0, 0, 0, false, "u2,g1", "p1" + NAME_SEPARATOR
@@ -241,6 +262,41 @@ public class TestQueueManagerWithJobTrac
       assertEquals("job submitted for u1 and queue p1:p11 is not killed.",
           cluster.getJob(jobID).getStatus().getState(), (State.KILLED));
     }
+    // check kill access to admins
+    ugi = 
+      UserGroupInformation.createUserForTesting("adminUser", new String[]{"g3"});
+    checkAccessToKill(tracker, jobConf, ugi);
+
+    ugi = 
+      UserGroupInformation.createUserForTesting("u3", new String[]{adminGroup});
+    checkAccessToKill(tracker, jobConf, ugi);
+
+    ugi = 
+      UserGroupInformation.createUserForTesting("u3", 
+          new String[]{deprecatedSuperGroup});
+    checkAccessToKill(tracker, jobConf, ugi);
+
+  }
+
+  private void checkAccessToKill(JobTracker tracker, final JobConf mrConf, 
+      UserGroupInformation killer) throws IOException, InterruptedException,
+      ClassNotFoundException {
+    Job job = submitSleepJob(1, 1, 100, 100, false, "u1,g1",
+        "p1" + NAME_SEPARATOR + "p11", conf);
+    JobID jobID = job.getStatus().getJobID();
+    //Ensure that the jobinprogress is initied before we issue a kill 
+    //signal to the job.
+    JobInProgress jip =  tracker.getJob(
+        org.apache.hadoop.mapred.JobID.downgrade(jobID));
+    tracker.initJob(jip);
+    Cluster cluster = killer.doAs(new PrivilegedExceptionAction<Cluster>() {
+      public Cluster run() throws IOException {
+        return new Cluster(mrConf);
+      }
+    });
+    cluster.getJob(jobID).killJob();
+    assertEquals("job not killed by " + killer,
+        cluster.getJob(jobID).getStatus().getState(), (State.KILLED));
   }
 
   /**

Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
Fri Jan 28 21:32:48 2011
@@ -26,6 +26,7 @@ import java.net.HttpURLConnection;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
@@ -40,6 +41,7 @@ import org.apache.hadoop.mapreduce.jobhi
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
 import org.apache.hadoop.security.Groups;
@@ -69,6 +71,12 @@ public class TestWebUIAuthorization exte
   private static String mrOwner = null;
   // member of supergroup
   private static final String superGroupMember = "user2";
+  // mrAdmin
+  private static final String mrAdminUser = "user4";
+  // Group for mrAdmins
+  private static final String mrAdminGroup = "admingroup";
+  // member of mrAdminGroup
+  private static final String mrAdminGroupMember = "user5";
   // admin of "default" queue
   private static final String qAdmin = "user3";
   // "colleague1" is there in job-view-acls config
@@ -127,7 +135,7 @@ public class TestWebUIAuthorization exte
    * Validates the given jsp/servlet against different user names who
    * can(or cannot) view the job.
    * (1) jobSubmitter can view the job
-   * (2) superGroupMember can view any job
+   * (2) mrAdmin and deprecated superGroupMember can view any job
    * (3) mrOwner can view any job
    * (4) qAdmins of the queue to which job is submitted to can view any job in
    *     that queue.
@@ -145,6 +153,12 @@ public class TestWebUIAuthorization exte
     assertEquals("Incorrect return code for supergroup-member " +
         superGroupMember, HttpURLConnection.HTTP_OK,
         getHttpStatusCode(url, superGroupMember, method));
+    assertEquals("Incorrect return code for admin user " +
+        mrAdminUser, HttpURLConnection.HTTP_OK,
+        getHttpStatusCode(url, mrAdminUser, method));
+    assertEquals("Incorrect return code for admingroup-member " +
+        mrAdminGroupMember, HttpURLConnection.HTTP_OK,
+        getHttpStatusCode(url, mrAdminGroupMember, method));
     assertEquals("Incorrect return code for MR-owner " + mrOwner,
         HttpURLConnection.HTTP_OK, getHttpStatusCode(url, mrOwner, method));
     assertEquals("Incorrect return code for queue admin " + qAdmin,
@@ -166,7 +180,8 @@ public class TestWebUIAuthorization exte
   /**
    * Validates the given jsp/servlet against different user names who
    * can(or cannot) modify the job.
-   * (1) jobSubmitter, mrOwner, qAdmin and superGroupMember can modify the job.
+   * (1) jobSubmitter, mrOwner, qAdmin, mrAdmin and deprecated superGroupMember
+   *     can modify the job.
    *     But we are not validating this in this method. Let the caller
    *     explicitly validate this, if needed.
    * (2) user mentioned in job-view-acl but not in job-modify-acl cannot
@@ -253,11 +268,36 @@ public class TestWebUIAuthorization exte
     }
   }
 
-  public void testAuthorizationForJobHistoryPages() throws Exception {
-    JobConf conf = new JobConf();
+  static void setupGroupsProvider() throws IOException {
+    Configuration conf = new Configuration();
     conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
         MyGroupsProvider.class.getName());
     Groups.getUserToGroupsMappingService(conf);
+    MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
+    MyGroupsProvider.mapping.put(viewColleague, Arrays.asList("group2"));
+    MyGroupsProvider.mapping.put(modifyColleague, Arrays.asList("group1"));
+    MyGroupsProvider.mapping.put(unauthorizedUser,
+        Arrays.asList("evilSociety"));
+    MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
+    MyGroupsProvider.mapping.put(mrAdminGroupMember,
+        Arrays.asList(mrAdminGroup));
+    MyGroupsProvider.mapping.put(viewAndModifyColleague,
+        Arrays.asList("group3"));
+    MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
+
+    mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
+    MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
+        new String[] { "group5", "group6" }));
+    
+    MyGroupsProvider.mapping.put(jobSubmitter1, Arrays.asList("group7"));
+    MyGroupsProvider.mapping.put(jobSubmitter2, Arrays.asList("group7"));
+    MyGroupsProvider.mapping.put(jobSubmitter3, Arrays.asList("group7"));
+
+    MyGroupsProvider.mapping.put(mrAdminUser, Arrays.asList("group8"));
+  }
+
+  public void testAuthorizationForJobHistoryPages() throws Exception {
+    setupGroupsProvider();
     Properties props = new Properties();
     props.setProperty("hadoop.http.filter.initializers",
         DummyFilterInitializer.class.getName());
@@ -269,18 +309,8 @@ public class TestWebUIAuthorization exte
         "historyDoneFolderOnHDFS");
     props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
     props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
-
-    MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
-    MyGroupsProvider.mapping.put(viewColleague, Arrays.asList("group2"));
-    MyGroupsProvider.mapping.put(modifyColleague, Arrays.asList("group1"));
-    MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety"));
-    MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
-    MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3"));
-    MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
-
-    mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
-    MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
-        new String[] { "group5", "group6" }));
+    props.setProperty(MRConfig.MR_ADMINS, mrAdminUser + " " + mrAdminGroup);
+    props.setProperty(JTConfig.JT_RETIREJOBS, "true");
 
     String[] queueNames = {"default"};
     String[] submitAclStrings = new String[] { jobSubmitter };
@@ -290,7 +320,7 @@ public class TestWebUIAuthorization exte
     MiniMRCluster cluster = getMRCluster();
     int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
 
-    conf = new JobConf(cluster.createJobConf());
+    JobConf conf = new JobConf(cluster.createJobConf());
     conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague + " group3");
 
     // Let us add group1 and group3 to modify-job-acl. So modifyColleague and
@@ -312,11 +342,16 @@ public class TestWebUIAuthorization exte
 
     org.apache.hadoop.mapreduce.JobID jobid = job.getJobID();
 
+    // Wait till job retires.
+    for (int i = 0; i < 10 && !job.isRetired(); i++) {
+      UtilsForTests.waitFor(1000);
+      LOG.info("waiting for the job " + jobid + " to retire");
+    }
+    assertTrue("Job did not retire", job.isRetired());
+
     String historyFileName = job.getStatus().getHistoryFile();
     String jtURL = "http://localhost:" + infoPort;
 
-    // Job will automatically be retired. Now test jsps..
-
     // validate access of jobdetails_history.jsp
     String jobDetailsJSP =
         jtURL + "/jobdetailshistory.jsp?logFile=" + historyFileName;
@@ -448,12 +483,13 @@ public class TestWebUIAuthorization exte
   /**
    * Starts a sleep job and tries to kill the job using jobdetails.jsp as
    * (1) viewColleague (2) unauthorizedUser (3) modifyColleague
-   * (4) viewAndModifyColleague (5) mrOwner (6) superGroupMember and
-   * (7) jobSubmitter
+   * (4) viewAndModifyColleague (5) mrOwner (6) deprecated superGroupMember
+   * (7) mrAdmin and (8) jobSubmitter
    *
    * Validates the given jsp/servlet against different user names who
    * can(or cannot) do both view and modify on the job.
-   * (1) jobSubmitter, mrOwner and superGroupMember can do both view and modify
+   * (1) jobSubmitter, mrOwner, mrAdmin and deprecated superGroupMember can do
+   *     both view and modify
    *     on the job. But we are not validating this in this method. Let the
    *     caller explicitly validate this, if needed.
    * (2) user mentioned in job-view-acls and job-modify-acls can do this
@@ -513,6 +549,10 @@ public class TestWebUIAuthorization exte
     confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
                                       superGroupMember);
     confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
+        mrAdminGroupMember);
+    confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
+        mrAdminUser);
+    confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
                                       qAdmin);
   }
 
@@ -565,7 +605,7 @@ public class TestWebUIAuthorization exte
     conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, " ");
     
     // Let us start 4 jobs as 4 different users(none of these 4 users is
-    // mrOwner and none of these users is a member of superGroup). So only
+    // mrOwner and none of these users is a member of mrAdmin/superGroup). Only
     // based on the config MRJobConfig.JOB_ACL_MODIFY_JOB being set here
     // and the jobSubmitter, killJob on each of the jobs will be succeeded.
 
@@ -626,10 +666,7 @@ public class TestWebUIAuthorization exte
    */
   @Test
   public void testWebUIAuthorization() throws Exception {
-    JobConf conf = new JobConf();
-    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
-        MyGroupsProvider.class.getName());
-    Groups.getUserToGroupsMappingService(conf);
+    setupGroupsProvider();
     Properties props = new Properties();
     props.setProperty("hadoop.http.filter.initializers",
         DummyFilterInitializer.class.getName());
@@ -640,22 +677,7 @@ public class TestWebUIAuthorization exte
     props.setProperty(JSPUtil.PRIVATE_ACTIONS_KEY, "true");
     props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
     props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
-
-    MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
-    MyGroupsProvider.mapping.put(viewColleague, Arrays.asList("group2"));
-    MyGroupsProvider.mapping.put(modifyColleague, Arrays.asList("group1"));
-    MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety"));
-    MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
-    MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3"));
-    MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
-
-    mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
-    MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
-        new String[] { "group5", "group6" }));
-    
-    MyGroupsProvider.mapping.put(jobSubmitter1, Arrays.asList("group7"));
-    MyGroupsProvider.mapping.put(jobSubmitter2, Arrays.asList("group7"));
-    MyGroupsProvider.mapping.put(jobSubmitter3, Arrays.asList("group7"));
+    props.setProperty(MRConfig.MR_ADMINS, mrAdminUser + " " + mrAdminGroup);
 
     String[] queueNames = {"default"};
     String[] submitAclStrings = {jobSubmitter + "," + jobSubmitter1 + ","
@@ -667,7 +689,7 @@ public class TestWebUIAuthorization exte
     int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
 
     JobConf clusterConf = cluster.createJobConf();
-    conf = new JobConf(clusterConf);
+    JobConf conf = new JobConf(clusterConf);
     conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague + " group3");
 
     // Let us add group1 and group3 to modify-job-acl. So modifyColleague and
@@ -711,12 +733,14 @@ public class TestWebUIAuthorization exte
     validateJobDetailsJSPKillJob(cluster, clusterConf, jtURL);
 
     // validate killJob of jobtracker.jsp as users viewAndModifyColleague,
-    // jobSubmitter, mrOwner and superGroupMember
+    // jobSubmitter, mrOwner, mrAdmin and superGroupMember
     confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL,
         viewAndModifyColleague);
     confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, jobSubmitter);
     confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrOwner);
     confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, superGroupMember);
+    confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrAdminUser);
+    confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrAdminGroupMember);
     confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, qAdmin);
 
     // validate killing of multiple jobs using jobtracker jsp and check
@@ -725,6 +749,53 @@ public class TestWebUIAuthorization exte
     validateKillMultipleJobs(cluster, conf, jtURL);
   }
 
+  public void testWebUIAuthorizationForCommonServlets() throws Exception {
+    setupGroupsProvider();
+    Properties props = new Properties();
+    props.setProperty("hadoop.http.filter.initializers",
+        DummyFilterInitializer.class.getName());
+    props.setProperty(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
+        "true");
+    props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
+    props.setProperty(MRConfig.MR_ADMINS, mrAdminUser + " " + mrAdminGroup);
+
+    startCluster(true, props);
+    validateCommonServlets(getMRCluster());
+    stopCluster();
+  }
+
+  private void validateCommonServlets(MiniMRCluster cluster)
+      throws IOException {
+    int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
+    String jtURL = "http://localhost:" + infoPort;
+    for (String servlet : new String[] { "logs", "stacks", "logLevel" }) {
+      String url = jtURL + "/" + servlet;
+      checkAccessToCommonServlet(url);
+    }
+    // validate access to common servlets for TaskTracker.
+    String ttURL = "http://localhost:"
+        + cluster.getTaskTrackerRunner(0).getTaskTracker().getHttpPort();
+    for (String servlet : new String[] { "logs", "stacks", "logLevel" }) {
+      String url = ttURL + "/" + servlet;
+      checkAccessToCommonServlet(url);
+    }
+  }
+
+  private void checkAccessToCommonServlet(String url) throws IOException {
+    url = url + "?a=b";
+    assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(url, mrAdminUser,
+        "GET"));
+    assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(url,
+        mrAdminGroupMember, "GET"));
+    assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(url,
+        mrOwner, "GET"));
+    assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(url,
+        superGroupMember, "GET"));
+    // no access for any other user
+    assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(url,
+        jobSubmitter, "GET"));
+  }
+
   // validate killJob of jobtracker.jsp
   private void validateJobTrackerJSPKillJobAction(
       org.apache.hadoop.mapreduce.JobID jobid, String jtURL)
@@ -762,7 +833,8 @@ public class TestWebUIAuthorization exte
         "&changeJobPriority=true&setJobPriority="+"HIGH"+"&jobCheckBox=" +
         jobid.toString();
     validateModifyJob(jobTrackerJSPSetJobPriorityAction, "GET");
-    // jobSubmitter, mrOwner, qAdmin and superGroupMember are not validated for
+    // jobSubmitter, mrOwner, qAdmin, mrAdmin and superGroupMember are not
+    // validated for
     // job-modify permission in validateModifyJob(). So let us do it
     // explicitly here
     assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
@@ -770,6 +842,10 @@ public class TestWebUIAuthorization exte
     assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
         jobTrackerJSPSetJobPriorityAction, superGroupMember, "GET"));
     assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
+        jobTrackerJSPSetJobPriorityAction, mrAdminUser, "GET"));
+    assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
+        jobTrackerJSPSetJobPriorityAction, mrAdminGroupMember, "GET"));
+    assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
         jobTrackerJSPSetJobPriorityAction, qAdmin, "GET"));
     assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
         jobTrackerJSPSetJobPriorityAction, mrOwner, "GET"));
@@ -850,5 +926,9 @@ public class TestWebUIAuthorization exte
         getHttpStatusCode(jobTrackerJSP, qAdmin, "GET"));
     assertEquals(HttpURLConnection.HTTP_OK,
         getHttpStatusCode(jobTrackerJSP, superGroupMember, "GET"));
-  }
+    assertEquals(HttpURLConnection.HTTP_OK,
+        getHttpStatusCode(jobTrackerJSP, mrAdminUser, "GET"));
+    assertEquals(HttpURLConnection.HTTP_OK,
+        getHttpStatusCode(jobTrackerJSP, mrAdminGroupMember, "GET"));
+ }
 }



Mime
View raw message