hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r998003 [3/3] - in /hadoop/mapreduce/trunk: ./ conf/ src/c++/task-controller/ src/c++/task-controller/tests/ src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ src/contrib/mumak/src/test/org/apache/hadoop/mapred/ src/docs/src...
Date Fri, 17 Sep 2010 07:34:40 GMT
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java Fri Sep 17 07:34:39 2010
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.mapred;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -37,6 +36,8 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapreduce.QueueState;
 import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
+
 import org.junit.After;
 import org.junit.Test;
 
@@ -48,19 +49,12 @@ public class TestQueueManagerRefresh {
   private static final Log LOG =
       LogFactory.getLog(TestQueueManagerRefresh.class);
 
-  String queueConfigPath =
-      System.getProperty("test.build.extraconf", "build/test/extraconf");
-  File queueConfigFile =
-      new File(queueConfigPath, QueueManager.QUEUE_CONF_FILE_NAME);
-
   /**
    * Remove the configuration file after the test's done.
    */
   @After
   public void tearDown() {
-    if (queueConfigFile.exists()) {
-      queueConfigFile.delete();
-    }
+    deleteQueuesConfigFile();
   }
 
   /**
@@ -96,8 +90,8 @@ public class TestQueueManagerRefresh {
     JobQueueInfo[] queues = getSimpleQueueHierarchy();
 
     // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+    writeQueueConfigurationFile(
+        QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
 
     QueueManager qManager = new QueueManager();
 
@@ -107,8 +101,8 @@ public class TestQueueManagerRefresh {
     queues[0].addChild(newQueue);
 
     // Rewrite the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+    writeQueueConfigurationFile(
+        QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
 
     testRefreshFailureWithChangeOfHierarchy(qManager);
 
@@ -127,8 +121,8 @@ public class TestQueueManagerRefresh {
     JobQueueInfo[] queues = getSimpleQueueHierarchy();
 
     // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+    writeQueueConfigurationFile(
+        QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
 
     QueueManager qManager = new QueueManager();
 
@@ -137,8 +131,8 @@ public class TestQueueManagerRefresh {
     queues[0].removeChild(q2);
 
     // Rewrite the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+    writeQueueConfigurationFile(
+        QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
 
     testRefreshFailureWithChangeOfHierarchy(qManager);
   }
@@ -187,8 +181,8 @@ public class TestQueueManagerRefresh {
     JobQueueInfo[] queues = getSimpleQueueHierarchy();
 
     // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+    writeQueueConfigurationFile(
+        QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
 
     QueueManager qManager = new QueueManager();
 
@@ -226,8 +220,8 @@ public class TestQueueManagerRefresh {
     }
 
     // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+    writeQueueConfigurationFile(
+        QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
 
     QueueManager qManager = new QueueManager();
 
@@ -261,8 +255,8 @@ public class TestQueueManagerRefresh {
     JobQueueInfo[] queues = getSimpleQueueHierarchy();
 
     // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+    writeQueueConfigurationFile(
+        QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
 
     QueueManager qManager = new QueueManager();
 

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java Fri Sep 17 07:34:39 2010
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.mapreduce.QueueState;
 import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
@@ -48,16 +49,21 @@ import org.apache.hadoop.mapreduce.Sleep
 import org.apache.hadoop.security.UserGroupInformation;
 import static org.apache.hadoop.mapred.DeprecatedQueueConfigurationParser.*;
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
+import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
 
 public class TestQueueManagerWithDeprecatedConf extends TestCase {
 
-  static final Log LOG = LogFactory.getLog(TestQueueManagerWithDeprecatedConf.class);
-  
+  static final Log LOG = LogFactory.getLog(
+      TestQueueManagerWithDeprecatedConf.class);
+
+  String submitAcl = QueueACL.SUBMIT_JOB.getAclName();
+  String adminAcl  = QueueACL.ADMINISTER_JOBS.getAclName();
 
   
   public void testMultipleQueues() {
     JobConf conf = new JobConf();
-    conf.set("mapred.queue.names", "q1,q2,Q3");
+    conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY,
+        "q1,q2,Q3");
     QueueManager qMgr = new QueueManager(conf);
     Set<String> expQueues = new TreeSet<String>();
     expQueues.add("q1");
@@ -68,7 +74,8 @@ public class TestQueueManagerWithDepreca
 
   public void testSchedulerInfo() {
     JobConf conf = new JobConf();
-    conf.set("mapred.queue.names", "qq1,qq2");
+    conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY,
+        "qq1,qq2");
     QueueManager qMgr = new QueueManager(conf);
     qMgr.setSchedulerInfo("qq1", "queueInfoForqq1");
     qMgr.setSchedulerInfo("qq2", "queueInfoForqq2");
@@ -85,44 +92,50 @@ public class TestQueueManagerWithDepreca
     try {
       // queue properties with which the cluster is started.
       Properties hadoopConfProps = new Properties();
-      hadoopConfProps.put("mapred.queue.names", "default,q1,q2");
-      hadoopConfProps.put("mapred.acls.enabled", "true");
-      UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile);
+      hadoopConfProps.put(DeprecatedQueueConfigurationParser.
+          MAPRED_QUEUE_NAMES_KEY, "default,q1,q2");
+      hadoopConfProps.put(MRConfig.MR_ACLS_ENABLED, "true");
 
-      //properties for mapred-queue-acls.xml
       UserGroupInformation ugi =
         UserGroupInformation.createRemoteUser("unknownUser");
-      hadoopConfProps.put("mapred.queue.default.acl-submit-job", ugi.getUserName());
-      hadoopConfProps.put("mapred.queue.q1.acl-submit-job", "u1");
-      hadoopConfProps.put("mapred.queue.q2.acl-submit-job", "*");
-      hadoopConfProps.put("mapred.queue.default.acl-administer-jobs", ugi.getUserName());
-      hadoopConfProps.put("mapred.queue.q1.acl-administer-jobs", "u1");
-      hadoopConfProps.put("mapred.queue.q2.acl-administer-jobs", "*");
+      hadoopConfProps.put(toFullPropertyName(
+          "default", submitAcl), ugi.getUserName());
+      hadoopConfProps.put(toFullPropertyName(
+          "q1", submitAcl), "u1");
+      hadoopConfProps.put(toFullPropertyName(
+          "q2", submitAcl), "*");
+      hadoopConfProps.put(toFullPropertyName(
+          "default", adminAcl), ugi.getUserName());
+      hadoopConfProps.put(toFullPropertyName(
+          "q1", adminAcl), "u2");
+      hadoopConfProps.put(toFullPropertyName(
+          "q2", adminAcl), "*");
 
       UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile);
 
       Configuration conf = new JobConf();
+      conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
       QueueManager queueManager = new QueueManager(conf);
-      //Testing access to queue.
+      //Testing job submission access to queues.
       assertTrue("User Job Submission failed.",
-          queueManager.hasAccess("default", Queue.QueueOperation.
+          queueManager.hasAccess("default", QueueACL.
               SUBMIT_JOB, ugi));
       assertFalse("User Job Submission failed.",
-          queueManager.hasAccess("q1", Queue.QueueOperation.
+          queueManager.hasAccess("q1", QueueACL.
               SUBMIT_JOB, ugi));
       assertTrue("User Job Submission failed.",
-          queueManager.hasAccess("q2", Queue.QueueOperation.
+          queueManager.hasAccess("q2", QueueACL.
               SUBMIT_JOB, ugi));
-      //Testing the admin acls
+      //Testing the administer-jobs acls
       assertTrue("User Job Submission failed.",
-           queueManager.hasAccess("default", Queue.QueueOperation.ADMINISTER_JOBS, ugi));
-       assertFalse("User Job Submission failed.",
-           queueManager.hasAccess("q1", Queue.QueueOperation.
-               ADMINISTER_JOBS, ugi));
-       assertTrue("User Job Submission failed.",
-           queueManager.hasAccess("q2", Queue.QueueOperation.
-               ADMINISTER_JOBS, ugi));
-
+          queueManager.hasAccess("default",
+              QueueACL.ADMINISTER_JOBS, ugi));
+      assertFalse("User Job Submission failed.",
+          queueManager.hasAccess("q1", QueueACL.
+              ADMINISTER_JOBS, ugi));
+      assertTrue("User Job Submission failed.",
+          queueManager.hasAccess("q2", QueueACL.
+              ADMINISTER_JOBS, ugi));
  
     } finally {
       //Cleanup the configuration files in all cases

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java Fri Sep 17 07:34:39 2010
@@ -19,8 +19,8 @@
 package org.apache.hadoop.mapred;
 
 import static org.apache.hadoop.mapred.QueueConfigurationParser.NAME_SEPARATOR;
-import static org.apache.hadoop.mapred.QueueManagerTestUtils.CONFIG;
-import static org.apache.hadoop.mapred.QueueManagerTestUtils.checkForConfigFile;
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.QUEUES_CONFIG_FILE_PATH;
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.deleteQueuesConfigFile;
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.createAcls;
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.createDocument;
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.createProperties;
@@ -29,7 +29,6 @@ import static org.apache.hadoop.mapred.Q
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.createSimpleDocumentWithAcls;
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.createState;
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.miniMRCluster;
-import static org.apache.hadoop.mapred.QueueManagerTestUtils.setUpCluster;
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.submitSleepJob;
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.writeToFile;
 import static org.junit.Assert.assertEquals;
@@ -37,7 +36,6 @@ import static org.junit.Assert.assertFal
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.io.File;
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.Properties;
@@ -47,11 +45,12 @@ import org.apache.hadoop.mapred.tools.MR
 import org.apache.hadoop.mapreduce.Cluster;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.QueueState;
 import org.apache.hadoop.mapreduce.JobStatus.State;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.junit.AfterClass;
-import org.junit.BeforeClass;
 import org.junit.Test;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -60,21 +59,26 @@ public class TestQueueManagerWithJobTrac
 
   private static Configuration conf;
 
-  @BeforeClass
-  public static void setUp() throws Exception {
-    checkForConfigFile();
+  @AfterClass
+  public static void tearDown() throws Exception {
+    deleteQueuesConfigFile();
+  }
+
+  private void startCluster(boolean aclsEnabled)
+      throws Exception {
+
+    deleteQueuesConfigFile();
     Document doc = createDocument();
-    createSimpleDocumentWithAcls(doc, "true");
-    writeToFile(doc, CONFIG);
+    createSimpleDocumentWithAcls(doc);
+    writeToFile(doc, QUEUES_CONFIG_FILE_PATH);
     conf = new Configuration();
-    conf.addResource(CONFIG);
-    conf.set("mapred.committer.job.setup.cleanup.needed", "false");
-    setUpCluster(conf);
-  }
+    conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
+    conf.setBoolean(MRConfig.MR_ACLS_ENABLED, aclsEnabled);
+
+    JobConf jobConf = new JobConf(conf);
+    String namenode = "file:///";
+    miniMRCluster = new MiniMRCluster(0, namenode, 3, null, null, jobConf);
 
-  @AfterClass
-  public static void tearDown() throws Exception {
-    new File(CONFIG).delete();
   }
 
   /**
@@ -83,6 +87,8 @@ public class TestQueueManagerWithJobTrac
    */
   @Test(expected = IOException.class)
   public void testSubmitJobForStoppedQueue() throws Exception {
+    startCluster(true);
+
     submitSleepJob(10, 10, 100, 100, false, null,
         "p1" + NAME_SEPARATOR + "p14", conf);
     fail("queue p1:p14 is in stopped state and should not accept jobs");
@@ -94,8 +100,10 @@ public class TestQueueManagerWithJobTrac
    */
   @Test(expected = IOException.class)
   public void testSubmitJobForContainerQueue() throws Exception {
-      submitSleepJob(10, 10, 100, 100, false, null, "p1", conf);
-      fail("queue p1 is a container queue and cannot have jobs");
+    startCluster(true);
+
+    submitSleepJob(10, 10, 100, 100, false, null, "p1", conf);
+    fail("queue p1 is a container queue and cannot have jobs");
   }
 
   /**
@@ -104,12 +112,16 @@ public class TestQueueManagerWithJobTrac
    */
   @Test
   public void testAclsForSubmitJob() throws Exception {
+    startCluster(true);
+
     Job job;
+    try {
     // submit job to queue p1:p13 with unspecified acls 
     job = submitSleepJob(0, 0, 0, 0, true, "u1,g1", "p1" + NAME_SEPARATOR
         + "p13", conf);
-    assertTrue("Job submission for u1 failed in queue : p1:p13.",
-        job.isSuccessful());
+    fail("user u1 cannot submit jobs to queue p1:p13");
+    } catch (Exception e) {
+    }
     // check for access to submit the job
     try {
       job = submitSleepJob(0, 0, 0, 0, false, "u2,g1", "p1" + NAME_SEPARATOR
@@ -117,11 +129,17 @@ public class TestQueueManagerWithJobTrac
       fail("user u2 cannot submit jobs to queue p1:p11");
     } catch (Exception e) {
     }
-    // submit job to queue p1:p11 with acls-submit-job as u1
+    // submit job to queue p1:p11 with acl-submit-job as u1
     job = submitSleepJob(0, 0, 0, 0, true, "u1,g1", "p1"
         + NAME_SEPARATOR + "p11", conf);
     assertTrue("Job submission for u1 failed in queue : p1:p11.",
         job.isSuccessful());
+    
+    // submit job to queue p1:p12 with acl-submit-job as *
+    job = submitSleepJob(0, 0, 0, 0, true, "u2,g1", "p1"
+        + NAME_SEPARATOR + "p12", conf);
+    assertTrue("Job submission for u2 failed in queue : p1:p12.",
+        job.isSuccessful());
   }
 
   /**
@@ -130,6 +148,8 @@ public class TestQueueManagerWithJobTrac
    */
   @Test
   public void testAccessToKillJob() throws Exception {
+    startCluster(true);
+
     Job job = submitSleepJob(1, 1, 100, 100, false, "u1,g1", "p1"
         + NAME_SEPARATOR + "p11", conf);
     final JobConf jobConf = miniMRCluster.createJobConf();
@@ -229,11 +249,13 @@ public class TestQueueManagerWithJobTrac
    */
   @Test
   public void testSubmitJobsAfterRefresh() throws Exception {
+    startCluster(true);
+
     // test for refresh
-    checkForConfigFile();
+    deleteQueuesConfigFile();
     Document doc = createDocument();
     refreshDocument(doc);
-    writeToFile(doc, CONFIG);
+    writeToFile(doc, QUEUES_CONFIG_FILE_PATH);
     MRAdmin admin = new MRAdmin(miniMRCluster.createJobConf());
     admin.run(new String[] { "-refreshQueues" });
     try {
@@ -242,15 +264,15 @@ public class TestQueueManagerWithJobTrac
       fail("user u1 is not in the submit jobs' list");
     } catch (Exception e) {
     }
-    checkForConfigFile();
+    deleteQueuesConfigFile();
     doc = createDocument();
-    createSimpleDocumentWithAcls(doc, "true");
-    writeToFile(doc, CONFIG);
+    createSimpleDocumentWithAcls(doc);
+    writeToFile(doc, QUEUES_CONFIG_FILE_PATH);
     admin.run(new String[] { "-refreshQueues" });
   }
 
   private void refreshDocument(Document doc) {
-    Element queues = createQueuesNode(doc, "true");
+    Element queues = createQueuesNode(doc);
 
     // Create parent level queue q1.
     Element q1 = createQueue(doc, "q1");
@@ -298,12 +320,7 @@ public class TestQueueManagerWithJobTrac
    */
   @Test
   public void testAclsDisabled() throws Exception {
-    checkForConfigFile();
-    Document doc = createDocument();
-    createSimpleDocumentWithAcls(doc, "false");
-    writeToFile(doc, CONFIG);
-    MRAdmin admin = new MRAdmin(miniMRCluster.createJobConf());
-    admin.run(new String[] { "-refreshQueues" });
+    startCluster(false);
 
     // submit job to queue p1:p11 by any user not in acls-submit-job
     Job job = submitSleepJob(0, 0, 0, 0, true, "u2,g1", "p1" + NAME_SEPARATOR

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java Fri Sep 17 07:34:39 2010
@@ -32,6 +32,10 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapred.JobTracker.RecoveryManager;
+import org.apache.hadoop.mapreduce.MRConfig;
+
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.createQueuesConfigFile;
+import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.security.UserGroupInformation;
 
@@ -156,11 +160,9 @@ public class TestRecoveryManager extends
     // clean up
     FileSystem fs = FileSystem.get(new Configuration());
     fs.delete(TEST_DIR, true);
-    
+
     JobConf conf = new JobConf();
     conf.set(JTConfig.JT_JOBHISTORY_BLOCK_SIZE, "1024");
-    conf.set(
-      DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY, "default");
     
     MiniMRCluster mr = new MiniMRCluster(1, "file:///", 1, null, null, conf);
     JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
@@ -239,10 +241,11 @@ public class TestRecoveryManager extends
                                       true);
     mr.getJobTrackerConf().setInt(JTConfig.JT_TASKS_PER_JOB, 25);
     
-    mr.getJobTrackerConf().setBoolean("mapred.acls.enabled" , true);
+    mr.getJobTrackerConf().setBoolean(MRConfig.MR_ACLS_ENABLED, true);
+
     UserGroupInformation ugi = UserGroupInformation.getLoginUser();
-    mr.getJobTrackerConf().set("mapred.queue.default.acl-submit-job", 
-                               ugi.getUserName());
+    mr.getJobTrackerConf().set(toFullPropertyName(
+        "default", QueueACL.SUBMIT_JOB.getAclName()), ugi.getUserName());
 
     // start the jobtracker
     LOG.info("Starting jobtracker");

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java Fri Sep 17 07:34:39 2010
@@ -39,12 +39,12 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
 import org.apache.hadoop.mapreduce.server.tasktracker.Localizer;
 import org.apache.hadoop.mapreduce.util.MRAsyncDiskService;
+
+import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.mapred.JvmManager.JvmEnv;
 import org.apache.hadoop.mapred.TaskController.JobInitializationContext;
@@ -122,12 +122,14 @@ public class TestTaskTrackerLocalization
     System.setProperty("hadoop.log.dir", HADOOP_LOG_DIR.getAbsolutePath());
 
     trackerFConf = new JobConf();
+
     trackerFConf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
     localDirs = new String[numLocalDirs];
     for (int i = 0; i < numLocalDirs; i++) {
       localDirs[i] = new File(ROOT_MAPRED_LOCAL_DIR, "0_" + i).getPath();
     }
     trackerFConf.setStrings(MRConfig.LOCAL_DIR, localDirs);
+    trackerFConf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
 
     // Create the job configuration file. Same as trackerConf in this test.
     jobConf = new JobConf(trackerFConf);
@@ -139,6 +141,15 @@ public class TestTaskTrackerLocalization
     jobConf.setInt(MRJobConfig.USER_LOG_RETAIN_HOURS, 0);
     jobConf.setUser(getJobOwner().getShortUserName());
 
+    String queue = "default";
+    // set job queue name in job conf
+    jobConf.setQueueName(queue);
+    // Set queue admins acl in job conf similar to what JobClient does so that
+    // it goes into job conf also.
+    jobConf.set(toFullPropertyName(queue,
+        QueueACL.ADMINISTER_JOBS.getAclName()),
+        "qAdmin1,qAdmin2 qAdminsGroup1,qAdminsGroup2");
+
     Job job = Job.getInstance(jobConf);
     String jtIdentifier = "200907202331";
     jobId = new JobID(jtIdentifier, 1);
@@ -527,6 +538,37 @@ public class TestTaskTrackerLocalization
         .exists());
     checkFilePermissions(jobLogDir.toString(), "drwx------", task.getUser(),
         taskTrackerUGI.getGroupNames()[0]);
+
+    // Make sure that the job ACLs file job-acls.xml exists in job userlog dir
+    File jobACLsFile = new File(jobLogDir, TaskTracker.jobACLsFile);
+    assertTrue("JobACLsFile is missing in the job userlog dir " + jobLogDir,
+        jobACLsFile.exists());
+
+    // With default task controller, the job-acls.xml file is owned by TT and
+    // permissions are 700
+    checkFilePermissions(jobACLsFile.getAbsolutePath(), "-rwx------",
+        taskTrackerUGI.getShortUserName(), taskTrackerUGI.getGroupNames()[0]);
+
+    validateJobACLsFileContent();
+  }
+
+  // Validate the contents of jobACLsFile ( i.e. user name, job-view-acl, queue
+  // name and queue-admins-acl ).
+  protected void validateJobACLsFileContent() {
+    JobConf jobACLsConf = TaskLogServlet.getConfFromJobACLsFile(jobId);
+
+    assertTrue(jobACLsConf.get("user.name").equals(
+        localizedJobConf.getUser()));
+    assertTrue(jobACLsConf.get(MRJobConfig.JOB_ACL_VIEW_JOB).
+        equals(localizedJobConf.get(MRJobConfig.JOB_ACL_VIEW_JOB)));
+
+    String queue = localizedJobConf.getQueueName();
+    assertTrue(queue.equalsIgnoreCase(jobACLsConf.getQueueName()));
+
+    String qACLName = toFullPropertyName(queue,
+        QueueACL.ADMINISTER_JOBS.getAclName());
+    assertTrue(jobACLsConf.get(qACLName).equals(
+        localizedJobConf.get(qACLName)));
   }
 
   /**
@@ -645,24 +687,6 @@ public class TestTaskTrackerLocalization
         + expectedStderr.toString() + " Observed : "
         + attemptLogFiles[1].toString(), expectedStderr.toString().equals(
         attemptLogFiles[1].toString()));
-
-    // Make sure that the job ACLs file exists in the task log dir
-    File jobACLsFile = new File(logDir, TaskRunner.jobACLsFile);
-    assertTrue("JobACLsFile is missing in the task log dir " + logDir,
-        jobACLsFile.exists());
-
-    // With default task controller, the job-acls file is owned by TT and
-    // permissions are 700
-    checkFilePermissions(jobACLsFile.getAbsolutePath(), "-rwx------",
-        taskTrackerUGI.getShortUserName(), taskTrackerUGI.getGroupNames()[0]);
-
-    // Validate the contents of jobACLsFile(both user name and job-view-acls)
-    Configuration jobACLsConf = TaskLogServlet.getConfFromJobACLsFile(task
-        .getTaskID(), task.isTaskCleanupTask());
-    assertTrue(jobACLsConf.get(MRJobConfig.USER_NAME).equals(
-        localizedJobConf.getUser()));
-    assertTrue(jobACLsConf.get(MRJobConfig.JOB_ACL_VIEW_JOB).
-        equals(localizedJobConf.get(MRJobConfig.JOB_ACL_VIEW_JOB)));
   }
 
   /**

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java Fri Sep 17 07:34:39 2010
@@ -55,6 +55,7 @@ public class TestUserLogCleanup {
     taskLogCleanupThread = new UserLogCleaner(conf);
     taskLogCleanupThread.setClock(myClock);
     tt = new TaskTracker();
+    tt.setConf(new JobConf(conf));
     tt.setLocalizer(localizer);
     tt.setTaskLogCleanupThread(taskLogCleanupThread);
   }
@@ -67,8 +68,9 @@ public class TestUserLogCleanup {
   private File localizeJob(JobID jobid) throws IOException {
     File jobUserlog = TaskLog.getJobDir(jobid);
 
+    JobConf conf = new JobConf();
     // localize job log directory
-    tt.initializeJobLogDir(jobid);
+    tt.initializeJobLogDir(jobid, conf);
     assertTrue(jobUserlog + " directory is not created.", jobUserlog.exists());
     return jobUserlog;
   }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java Fri Sep 17 07:34:39 2010
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.jobhi
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
 import org.apache.hadoop.security.Groups;
 import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -57,12 +59,18 @@ public class TestWebUIAuthorization exte
   private static final Log LOG = LogFactory.getLog(
       TestWebUIAuthorization.class);
 
-  // user1 submits the jobs
+  // users who submit the jobs
   private static final String jobSubmitter = "user1";
+  private static final String jobSubmitter1 = "user11";
+  private static final String jobSubmitter2 = "user12";
+  private static final String jobSubmitter3 = "user13";
+
   // mrOwner starts the cluster
   private static String mrOwner = null;
   // member of supergroup
   private static final String superGroupMember = "user2";
+  // admin of "default" queue
+  private static final String qAdmin = "user3";
   // "colleague1" is there in job-view-acls config
   private static final String viewColleague = "colleague1";
   // "colleague2" is there in job-modify-acls config
@@ -72,10 +80,17 @@ public class TestWebUIAuthorization exte
   // "evilJohn" is not having view/modify access on the jobs
   private static final String unauthorizedUser = "evilJohn";
 
+  @Override
   protected void setUp() throws Exception {
     // do not do anything
   };
 
+  @Override
+  protected void tearDown() throws Exception {
+    deleteQueuesConfigFile();
+    super.tearDown();
+  }
+
   /** access a url, ignoring some IOException such as the page does not exist */
   static int getHttpStatusCode(String urlstring, String userName,
       String method) throws IOException {
@@ -112,45 +127,51 @@ public class TestWebUIAuthorization exte
    * Validates the given jsp/servlet against different user names who
    * can(or cannot) view the job.
    * (1) jobSubmitter can view the job
-   * (2) superGroupMember can view the job
-   * (3) user mentioned in job-view-acls should be able to view the job
-   * (4) user mentioned in job-modify-acls but not in job-view-acls
+   * (2) superGroupMember can view any job
+   * (3) mrOwner can view any job
+   * (4) qAdmins of the queue to which job is submitted to can view any job in
+   *     that queue.
+   * (5) user mentioned in job-view-acl should be able to view the
+   *     job irrespective of job-modify-acl.
+   * (6) user mentioned in job-modify-acl but not in job-view-acl
    *     cannot view the job
-   * (5) other unauthorized users cannot view the job
+   * (7) other unauthorized users cannot view the job
    */
   private void validateViewJob(String url, String method)
       throws IOException {
-    assertEquals("Incorrect return code for " + jobSubmitter,
+    assertEquals("Incorrect return code for job submitter " + jobSubmitter,
         HttpURLConnection.HTTP_OK, getHttpStatusCode(url, jobSubmitter,
             method));
-    assertEquals("Incorrect return code for " + superGroupMember,
-        HttpURLConnection.HTTP_OK, getHttpStatusCode(url, superGroupMember,
-            method));
-    assertEquals("Incorrect return code for " + mrOwner,
+    assertEquals("Incorrect return code for supergroup-member " +
+        superGroupMember, HttpURLConnection.HTTP_OK,
+        getHttpStatusCode(url, superGroupMember, method));
+    assertEquals("Incorrect return code for MR-owner " + mrOwner,
         HttpURLConnection.HTTP_OK, getHttpStatusCode(url, mrOwner, method));
-    assertEquals("Incorrect return code for " + viewColleague,
-        HttpURLConnection.HTTP_OK, getHttpStatusCode(url, viewColleague,
-            method));
-    assertEquals("Incorrect return code for " + viewAndModifyColleague,
-        HttpURLConnection.HTTP_OK, getHttpStatusCode(url,
-            viewAndModifyColleague, method));
-    assertEquals("Incorrect return code for " + modifyColleague,
-        HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(url,
-            modifyColleague, method));
-    assertEquals("Incorrect return code for " + unauthorizedUser,
-        HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(url,
-            unauthorizedUser, method));
+    assertEquals("Incorrect return code for queue admin " + qAdmin,
+        HttpURLConnection.HTTP_OK, getHttpStatusCode(url, qAdmin, method));
+    assertEquals("Incorrect return code for user in job-view-acl " +
+        viewColleague, HttpURLConnection.HTTP_OK,
+        getHttpStatusCode(url, viewColleague, method));
+    assertEquals("Incorrect return code for user in job-view-acl and " +
+        "job-modify-acl " + viewAndModifyColleague, HttpURLConnection.HTTP_OK,
+        getHttpStatusCode(url, viewAndModifyColleague, method));
+    assertEquals("Incorrect return code for user in job-modify-acl " +
+        modifyColleague, HttpURLConnection.HTTP_UNAUTHORIZED,
+        getHttpStatusCode(url, modifyColleague, method));
+    assertEquals("Incorrect return code for unauthorizedUser " +
+        unauthorizedUser, HttpURLConnection.HTTP_UNAUTHORIZED,
+        getHttpStatusCode(url, unauthorizedUser, method));
   }
 
   /**
    * Validates the given jsp/servlet against different user names who
    * can(or cannot) modify the job.
-   * (1) jobSubmitter and superGroupMember can modify the job. But we are not
-   *     validating this in this method. Let the caller explicitly validate
-   *     this, if needed.
-   * (2) user mentioned in job-view-acls but not in job-modify-acls cannot
+   * (1) jobSubmitter, mrOwner, qAdmin and superGroupMember can modify the job.
+   *     But we are not validating this in this method. Let the caller
+   *     explicitly validate this, if needed.
+   * (2) user mentioned in job-view-acl but not in job-modify-acl cannot
    *     modify the job
-   * (3) user mentioned in job-modify-acls (irrespective of job-view-acls)
+   * (3) user mentioned in job-modify-acl (irrespective of job-view-acl)
    *     can modify the job
    * (4) other unauthorized users cannot modify the job
    */
@@ -240,13 +261,13 @@ public class TestWebUIAuthorization exte
     Properties props = new Properties();
     props.setProperty("hadoop.http.filter.initializers",
         DummyFilterInitializer.class.getName());
-    props.setProperty(MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG,
+    props.setProperty(MRConfig.MR_ACLS_ENABLED,
         String.valueOf(true));
+
     props.setProperty("dfs.permissions.enabled", "false");
     props.setProperty("mapred.job.tracker.history.completed.location",
         "historyDoneFolderOnHDFS");
-    props.setProperty("mapreduce.job.committer.setup.cleanup.needed",
-        "false");
+    props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
     props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
 
     MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
@@ -255,11 +276,17 @@ public class TestWebUIAuthorization exte
     MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety"));
     MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
     MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3"));
+    MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
+
     mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
     MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
-        new String[] { "group4", "group5" }));
+        new String[] { "group5", "group6" }));
+
+    String[] queueNames = {"default"};
+    String[] submitAclStrings = new String[] { jobSubmitter };
+    String[] adminsAclStrings = new String[] { qAdmin };
+    startCluster(props, queueNames, submitAclStrings, adminsAclStrings);
 
-    startCluster(true, props);
     MiniMRCluster cluster = getMRCluster();
     int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
 
@@ -338,30 +365,57 @@ public class TestWebUIAuthorization exte
             Integer.toString(attemptsMap.get(attempt).getHttpPort()),
             attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR;
         validateViewJob(stderrURL, "GET");
+      }
+    }
 
-        // delete job-acls.xml file from the task log dir of attempt and verify
-        // if unauthorized users can view task logs of attempt.
-        File attemptLogDir = TaskLog.getAttemptDir(
-            org.apache.hadoop.mapred.TaskAttemptID.downgrade(attempt), false);
-        Path jobACLsFilePath = new Path(attemptLogDir.toString(),
-            TaskRunner.jobACLsFile);
-        new File(jobACLsFilePath.toUri().getPath()).delete();
+    // For each tip, let us test the effect of deletion of job-acls.xml file and
+    // deletion of task log dir for each of the attempts of the tip.
+
+    // delete job-acls.xml file from the job userlog dir and verify
+    // if unauthorized users can view task logs of each attempt.
+    Path jobACLsFilePath = new Path(TaskLog.getJobDir(jobid).toString(),
+        TaskTracker.jobACLsFile);
+    assertTrue("Could not delete job-acls.xml file.",
+        new File(jobACLsFilePath.toUri().getPath()).delete());
+
+    for (TaskID tip : tipsMap.keySet()) {
+
+      Map<TaskAttemptID, TaskAttemptInfo> attemptsMap =
+        tipsMap.get(tip).getAllTaskAttempts();
+      for (TaskAttemptID attempt : attemptsMap.keySet()) {
+
+        String stdoutURL = TaskLogServlet.getTaskLogUrl("localhost",
+            Integer.toString(attemptsMap.get(attempt).getHttpPort()),
+            attempt.toString()) + "&filter=" + TaskLog.LogName.STDOUT;;
+
+        String stderrURL = TaskLogServlet.getTaskLogUrl("localhost",
+            Integer.toString(attemptsMap.get(attempt).getHttpPort()),
+            attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR;
+
+        // unauthorized users can view task logs of each attempt because
+        // job-acls.xml file is deleted.
         assertEquals("Incorrect return code for " + unauthorizedUser,
             HttpURLConnection.HTTP_OK, getHttpStatusCode(stdoutURL,
-            unauthorizedUser, "GET"));
+                unauthorizedUser, "GET"));
         assertEquals("Incorrect return code for " + unauthorizedUser,
             HttpURLConnection.HTTP_OK, getHttpStatusCode(stderrURL,
-            unauthorizedUser, "GET"));
+                unauthorizedUser, "GET"));
 
         // delete the whole task log dir of attempt and verify that we get
         // correct response code (i.e. HTTP_GONE) when task logs are accessed.
+        File attemptLogDir = TaskLog.getAttemptDir(
+            org.apache.hadoop.mapred.TaskAttemptID.downgrade(attempt), false);
         FileUtil.fullyDelete(attemptLogDir);
+
+        // Try accessing tasklogs - STDOUT and STDERR now(after the whole
+        // attempt log dir is deleted).
         assertEquals("Incorrect return code for " + jobSubmitter,
             HttpURLConnection.HTTP_GONE, getHttpStatusCode(stdoutURL,
-            jobSubmitter, "GET"));
+                jobSubmitter, "GET"));
+
         assertEquals("Incorrect return code for " + jobSubmitter,
             HttpURLConnection.HTTP_GONE, getHttpStatusCode(stderrURL,
-            jobSubmitter, "GET"));
+                jobSubmitter, "GET"));
       }
     }
 
@@ -377,6 +431,21 @@ public class TestWebUIAuthorization exte
   }
 
   /**
+   * Creates queues configuration file with the given queues and acls and starts
+   * cluster with that queues configuration file.
+   * @param props   configuration properties to inject to the mini cluster
+   * @param queueNames   the job queues on the cluster 
+   * @param submitAclStrings acl-submit-job acls for all queues
+   * @param adminsAclStrings acl-administer-jobs acls for all queues
+   * @throws Exception
+   */
+  private void startCluster(Properties props, String[] queueNames,
+      String[] submitAclStrings, String[] adminsAclStrings) throws Exception {
+    createQueuesConfigFile(queueNames, submitAclStrings, adminsAclStrings);
+    startCluster(true, props);
+  }
+
+  /**
    * Starts a sleep job and tries to kill the job using jobdetails.jsp as
    * (1) viewColleague (2) unauthorizedUser (3) modifyColleague
    * (4) viewAndModifyColleague (5) mrOwner (6) superGroupMember and
@@ -387,11 +456,13 @@ public class TestWebUIAuthorization exte
    * (1) jobSubmitter, mrOwner and superGroupMember can do both view and modify
    *     on the job. But we are not validating this in this method. Let the
    *     caller explicitly validate this, if needed.
-   * (2) user mentioned in job-view-acls but not in job-modify-acls cannot
+   * (2) user mentioned in job-view-acls and job-modify-acls can do this
+   * (3) user mentioned in job-view-acls but not in job-modify-acls cannot
    *     do this
-   * (3) user mentioned in job-modify-acls but not in job-view-acls cannot
+   * (4) user mentioned in job-modify-acls but not in job-view-acls cannot
    *     do this
-   * (4) other unauthorized users cannot do this
+   * (5) qAdmin cannot do this because he doesn't have view access to the job
+   * (6) other unauthorized users cannot do this
    *
    * @throws Exception
    */
@@ -419,8 +490,11 @@ public class TestWebUIAuthorization exte
           getHttpStatusCode(url, unauthorizedUser, "POST"));
       assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
           getHttpStatusCode(url, modifyColleague, "POST"));
+
       assertEquals(HttpURLConnection.HTTP_OK,
           getHttpStatusCode(url, viewAndModifyColleague, "POST"));
+      assertTrue("killJob using jobdetails.jsp failed for a job for which "
+          + "user has job-view and job-modify permissions", job.isComplete());
     } finally {
       if (!job.isComplete()) {
         LOG.info("Killing job " + jobid + " from finally block");
@@ -430,14 +504,16 @@ public class TestWebUIAuthorization exte
       }
     }
 
-    // check if jobSubmitter, mrOwner and superGroupMember can do killJob
-    // using jobdetails.jsp url
+    // Check if jobSubmitter, mrOwner, superGroupMember and queueAdmins
+    // can do killJob using jobdetails.jsp url
+    confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
+                                      jobSubmitter);
     confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
-                                       jobSubmitter);
+                                      mrOwner);
     confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
-                                       mrOwner);
+                                      superGroupMember);
     confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
-                                       superGroupMember);
+                                      qAdmin);
   }
 
   /**
@@ -486,47 +562,47 @@ public class TestWebUIAuthorization exte
     // jobTrackerJSP killJob url
     String url = jobTrackerJSP + "&killJobs=true";
     // view-job-acl doesn't matter for killJob from jobtracker jsp page
-    conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "");
+    conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, " ");
     
-    // Let us start jobs as 4 different users(none of these 4 users is
+    // Let us start 4 jobs as 4 different users(none of these 4 users is
     // mrOwner and none of these users is a member of superGroup). So only
-    // based on the config JobContext.JOB_ACL_MODIFY_JOB being set here,
-    // killJob on each of the jobs will be succeeded.
+    // based on the config MRJobConfig.JOB_ACL_MODIFY_JOB being set here
+    // and the jobSubmitter, killJob on each of the jobs will be succeeded.
 
     // start 1st job.
     // Out of these 4 users, only jobSubmitter can do killJob on 1st job
-    conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, "");
+    conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, " ");
     Job job1 = startSleepJobAsUser(jobSubmitter, conf);
     org.apache.hadoop.mapreduce.JobID jobid = job1.getJobID();
     getTIPId(cluster, jobid);// wait till the map task is started
     url = url.concat("&jobCheckBox=" + jobid.toString());
     // start 2nd job.
-    // Out of these 4 users, only viewColleague can do killJob on 2nd job
-    Job job2 = startSleepJobAsUser(viewColleague, conf);
+    // Out of these 4 users, only jobSubmitter1 can do killJob on 2nd job
+    Job job2 = startSleepJobAsUser(jobSubmitter1, conf);
     jobid = job2.getJobID();
     getTIPId(cluster, jobid);// wait till the map task is started
     url = url.concat("&jobCheckBox=" + jobid.toString());
     // start 3rd job.
-    // Out of these 4 users, only modifyColleague can do killJob on 3rd job
-    Job job3 = startSleepJobAsUser(modifyColleague, conf);
+    // Out of these 4 users, only jobSubmitter2 can do killJob on 3rd job
+    Job job3 = startSleepJobAsUser(jobSubmitter2, conf);
     jobid = job3.getJobID();
     getTIPId(cluster, jobid);// wait till the map task is started
     url = url.concat("&jobCheckBox=" + jobid.toString());
     // start 4rd job.
-    // Out of these 4 users, viewColleague and viewAndModifyColleague
+    // Out of these 4 users, jobSubmitter1 and jobSubmitter3
     // can do killJob on 4th job
-    conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, viewColleague);
-    Job job4 = startSleepJobAsUser(viewAndModifyColleague, conf);
+    conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, jobSubmitter1);
+    Job job4 = startSleepJobAsUser(jobSubmitter3, conf);
     jobid = job4.getJobID();
     getTIPId(cluster, jobid);// wait till the map task is started
     url = url.concat("&jobCheckBox=" + jobid.toString());
 
     try {
-      // Try killing all the 4 jobs as user viewColleague who can kill only
+      // Try killing all the 4 jobs as user jobSubmitter1 who can kill only
       // 2nd and 4th jobs. Check if 1st and 3rd jobs are not killed and
       // 2nd and 4th jobs got killed
       assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
-          getHttpStatusCode(url, viewColleague, "POST"));
+          getHttpStatusCode(url, jobSubmitter1, "POST"));
       assertFalse("killJob succeeded for a job for which user doesnot "
           + " have job-modify permission", job1.isComplete());
       assertFalse("killJob succeeded for a job for which user doesnot "
@@ -557,12 +633,12 @@ public class TestWebUIAuthorization exte
     Properties props = new Properties();
     props.setProperty("hadoop.http.filter.initializers",
         DummyFilterInitializer.class.getName());
-    props.setProperty(
-       MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG, String.valueOf(true));
+    props.setProperty(MRConfig.MR_ACLS_ENABLED, String.valueOf(true));
+
     props.setProperty("dfs.permissions.enabled", "false");
     
     props.setProperty(JSPUtil.PRIVATE_ACTIONS_KEY, "true");
-    props.setProperty("mapreduce.job.committer.setup.cleanup.needed", "false");
+    props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
     props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
 
     MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
@@ -571,12 +647,22 @@ public class TestWebUIAuthorization exte
     MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety"));
     MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
     MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3"));
+    MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
 
     mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
     MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
-        new String[] { "group4", "group5" }));
+        new String[] { "group5", "group6" }));
+    
+    MyGroupsProvider.mapping.put(jobSubmitter1, Arrays.asList("group7"));
+    MyGroupsProvider.mapping.put(jobSubmitter2, Arrays.asList("group7"));
+    MyGroupsProvider.mapping.put(jobSubmitter3, Arrays.asList("group7"));
+
+    String[] queueNames = {"default"};
+    String[] submitAclStrings = {jobSubmitter + "," + jobSubmitter1 + ","
+        + jobSubmitter2 + "," + jobSubmitter3};
+    String[] adminsAclStrings = new String[]{qAdmin};
+    startCluster(props, queueNames, submitAclStrings, adminsAclStrings);
 
-    startCluster(true, props);
     MiniMRCluster cluster = getMRCluster();
     int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
 
@@ -630,8 +716,8 @@ public class TestWebUIAuthorization exte
         viewAndModifyColleague);
     confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, jobSubmitter);
     confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrOwner);
-    confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL,
-        superGroupMember);
+    confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, superGroupMember);
+    confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, qAdmin);
 
     // validate killing of multiple jobs using jobtracker jsp and check
     // if all the jobs which can be killed by user are actually the ones that
@@ -676,7 +762,7 @@ public class TestWebUIAuthorization exte
         "&changeJobPriority=true&setJobPriority="+"HIGH"+"&jobCheckBox=" +
         jobid.toString();
     validateModifyJob(jobTrackerJSPSetJobPriorityAction, "GET");
-    // jobSubmitter, mrOwner and superGroupMember are not validated for
+    // jobSubmitter, mrOwner, qAdmin and superGroupMember are not validated for
     // job-modify permission in validateModifyJob(). So let us do it
     // explicitly here
     assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
@@ -684,6 +770,8 @@ public class TestWebUIAuthorization exte
     assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
         jobTrackerJSPSetJobPriorityAction, superGroupMember, "GET"));
     assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
+        jobTrackerJSPSetJobPriorityAction, qAdmin, "GET"));
+    assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
         jobTrackerJSPSetJobPriorityAction, mrOwner, "GET"));
   }
 
@@ -759,6 +847,8 @@ public class TestWebUIAuthorization exte
     assertEquals(HttpURLConnection.HTTP_OK,
         getHttpStatusCode(jobTrackerJSP, mrOwner, "GET"));
     assertEquals(HttpURLConnection.HTTP_OK,
+        getHttpStatusCode(jobTrackerJSP, qAdmin, "GET"));
+    assertEquals(HttpURLConnection.HTTP_OK,
         getHttpStatusCode(jobTrackerJSP, superGroupMember, "GET"));
   }
 }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java Fri Sep 17 07:34:39 2010
@@ -26,11 +26,12 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobACLsManager;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.JobInProgress;
 import org.apache.hadoop.mapred.JobTracker;
 import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.Operation;
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.junit.Before;
@@ -56,23 +57,33 @@ public class TestJobACLs {
           TestJobACLs.class.getCanonicalName() + Path.SEPARATOR
               + "completed-job-store");
 
+  private String jobSubmitter = "jobSubmitter";
+  private String viewColleague = "viewColleague";
+  private String modifyColleague = "modifyColleague";
+  private String qAdmin = "qAdmin";
+
   /**
    * Start the cluster before running the actual test.
    * 
    * @throws IOException
    */
   @Before
-  public void setup() throws IOException {
+  public void setup() throws Exception {
     // Start the cluster
     startCluster(false);
   }
 
-  private void startCluster(boolean reStart) throws IOException {
-    UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser();
+  private void startCluster(boolean reStart) throws Exception {
+
+    // Configure job queues
+    String[] queueNames = {"default"};
+    createQueuesConfigFile(queueNames,
+        new String[] { jobSubmitter }, new String[] { qAdmin });
+
     JobConf conf = new JobConf();
 
-    // Enable job-level authorization
-    conf.setBoolean(MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG, true);
+    // Enable queue and job level authorization
+    conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
 
     // Enable CompletedJobStore
     FileSystem fs = FileSystem.getLocal(conf);
@@ -84,6 +95,7 @@ public class TestJobACLs {
     conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, true);
     conf.set(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1");
 
+    UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser();
     mr = new MiniMRCluster(0, 0, 1, "file:///", 1, null, null, MR_UGI, conf);
   }
 
@@ -92,6 +104,7 @@ public class TestJobACLs {
    */
   @After
   public void tearDown() {
+    deleteQueuesConfigFile();
     if (mr != null) {
       mr.shutdown();
     }
@@ -106,10 +119,10 @@ public class TestJobACLs {
    * @throws ClassNotFoundException
    */
   @Test
-  public void testACLS() throws IOException, InterruptedException,
-      ClassNotFoundException {
+  public void testACLS() throws Exception {
     verifyACLViewJob();
-    verifyACLModifyJob();
+    verifyACLModifyJob(modifyColleague);
+    verifyACLModifyJob(qAdmin);
     verifyACLPersistence();
   }
 
@@ -123,18 +136,21 @@ public class TestJobACLs {
 
     // Set the job up.
     final Configuration myConf = mr.createJobConf();
-    myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user1,user3");
+    myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague);
 
     // Submit the job as user1
-    Job job = submitJobAsUser(myConf, "user1");
+    Job job = submitJobAsUser(myConf, jobSubmitter);
 
     final JobID jobId = job.getJobID();
 
     // Try operations as an unauthorized user.
-    verifyViewJobAsUnauthorizedUser(myConf, jobId, "user2");
+    verifyViewJobAsUnauthorizedUser(myConf, jobId, modifyColleague);
 
-    // Try operations as an authorized user.
-    verifyViewJobAsAuthorizedUser(myConf, jobId, "user3");
+    // Try operations as an authorized user, who is part of view-job-acl.
+    verifyViewJobAsAuthorizedUser(myConf, jobId, viewColleague);
+
+    // Try operations as an authorized user, who is a queue administrator.
+    verifyViewJobAsAuthorizedUser(myConf, jobId, qAdmin);
 
     // Clean up the job
     job.killJob();
@@ -242,7 +258,7 @@ public class TestJobACLs {
           fail("AccessControlException expected..");
         } catch (IOException ioe) {
           assertTrue(ioe.getMessage().contains(
-              JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.VIEW_JOB));
+              " cannot perform operation " + JobACL.VIEW_JOB));
         } catch (InterruptedException e) {
           fail("Exception .. interrupted.." + e);
         }
@@ -253,7 +269,7 @@ public class TestJobACLs {
           fail("AccessControlException expected..");
         } catch (IOException ioe) {
           assertTrue(ioe.getMessage().contains(
-              JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.VIEW_JOB));
+              " cannot perform operation " + JobACL.VIEW_JOB));
         } catch (InterruptedException e) {
           fail("Exception .. interrupted.." + e);
         }
@@ -264,29 +280,29 @@ public class TestJobACLs {
   }
 
   /**
-   * Verify JobContext.Job_ACL_MODIFY_JOB
+   * Verify MRConfig.Job_ACL_MODIFY_JOB
    * 
    * @throws IOException
    * @throws InterruptedException
    * @throws ClassNotFoundException
    */
-  private void verifyACLModifyJob() throws IOException,
+  private void verifyACLModifyJob(String authorizedUser) throws IOException,
       InterruptedException, ClassNotFoundException {
 
     // Set the job up.
     final Configuration myConf = mr.createJobConf();
-    myConf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, "user1,user3");
+    myConf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, modifyColleague);
 
     // Submit the job as user1
-    Job job = submitJobAsUser(myConf, "user1");
+    Job job = submitJobAsUser(myConf, jobSubmitter);
 
     final JobID jobId = job.getJobID();
 
     // Try operations as an unauthorized user.
-    verifyModifyJobAsUnauthorizedUser(myConf, jobId, "user2");
+    verifyModifyJobAsUnauthorizedUser(myConf, jobId, viewColleague);
 
     // Try operations as an authorized user.
-    verifyModifyJobAsAuthorizedUser(myConf, jobId, "user3");
+    verifyModifyJobAsAuthorizedUser(myConf, jobId, authorizedUser);
   }
 
   private void verifyModifyJobAsAuthorizedUser(
@@ -357,7 +373,7 @@ public class TestJobACLs {
           fail("AccessControlException expected..");
         } catch (IOException ioe) {
           assertTrue(ioe.getMessage().contains(
-            JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.MODIFY_JOB));
+              " cannot perform operation " + Operation.KILL_JOB));
         } catch (InterruptedException e) {
           fail("Exception .. interrupted.." + e);
         }
@@ -368,7 +384,7 @@ public class TestJobACLs {
           fail("AccessControlException expected..");
         } catch (IOException ioe) {
           assertTrue(ioe.getMessage().contains(
-            JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.MODIFY_JOB));
+              " cannot perform operation " + Operation.SET_JOB_PRIORITY));
         } catch (InterruptedException e) {
           fail("Exception .. interrupted.." + e);
         }
@@ -378,15 +394,14 @@ public class TestJobACLs {
     });
   }
 
-  private void verifyACLPersistence() throws IOException,
-      InterruptedException {
+  private void verifyACLPersistence() throws Exception {
 
     // Set the job up.
     final Configuration myConf = mr.createJobConf();
-    myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user2 group2");
+    myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague + " group2");
 
     // Submit the job as user1
-    Job job = submitJobAsUser(myConf, "user1");
+    Job job = submitJobAsUser(myConf, jobSubmitter);
 
     final JobID jobId = job.getJobID();
 
@@ -406,11 +421,14 @@ public class TestJobACLs {
 
     final Configuration myNewJobConf = mr.createJobConf();
     // Now verify view-job works off CompletedJobStore
-    verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, "user2");
+    verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, viewColleague);
+    verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, qAdmin);
 
     // Only JobCounters is persisted on the JobStore. So test counters only.
     UserGroupInformation unauthorizedUGI =
-        UserGroupInformation.createUserForTesting("user3", new String[] {});
+        UserGroupInformation.createUserForTesting(
+            modifyColleague, new String[] {});
+
     unauthorizedUGI.doAs(new PrivilegedExceptionAction<Object>() {
       @SuppressWarnings("null")
       @Override
@@ -432,7 +450,7 @@ public class TestJobACLs {
           fail("AccessControlException expected..");
         } catch (IOException ioe) {
           assertTrue(ioe.getMessage().contains(
-              JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.VIEW_JOB));
+              " cannot perform operation " + Operation.VIEW_JOB_COUNTERS));
         } catch (InterruptedException e) {
           fail("Exception .. interrupted.." + e);
         }

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java Fri Sep 17 07:34:39 2010
@@ -21,10 +21,7 @@ import java.io.DataInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaParseException;
 import org.apache.hadoop.mapreduce.jobhistory.EventReader;
-import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
 import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
 

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java Fri Sep 17 07:34:39 2010
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.tools.rumen;
 
-import java.text.ParseException;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -72,7 +71,11 @@ public class Job20LineHistoryEventEmitte
       String submitTime = line.get("SUBMIT_TIME");
       String jobConf = line.get("JOBCONF");
       String user = line.get("USER");
+      if (user == null) {
+        user = "nulluser";
+      }
       String jobName = line.get("JOBNAME");
+      String jobQueueName = line.get("JOB_QUEUE");// could be null
 
       if (submitTime != null) {
         Job20LineHistoryEventEmitter that =
@@ -82,8 +85,8 @@ public class Job20LineHistoryEventEmitte
 
         Map<JobACL, AccessControlList> jobACLs =
           new HashMap<JobACL, AccessControlList>();
-        return new JobSubmittedEvent(jobID, jobName, user == null ? "nulluser"
-            : user, that.originalSubmitTime, jobConf, jobACLs);
+        return new JobSubmittedEvent(jobID, jobName, user,
+            that.originalSubmitTime, jobConf, jobACLs, jobQueueName);
       }
 
       return null;

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java Fri Sep 17 07:34:39 2010
@@ -544,6 +544,8 @@ public class JobBuilder {
     result.setJobName(event.getJobName());
     result.setUser(event.getUserName());
     result.setSubmitTime(event.getSubmitTime());
+    // job queue name is set when conf file is processed.
+    // See JobBuilder.process(Properties) method for details.
   }
 
   private void processJobStatusChangedEvent(JobStatusChangedEvent event) {



Mime
View raw message