Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 41796 invoked from network); 17 Sep 2010 07:35:09 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 17 Sep 2010 07:35:09 -0000 Received: (qmail 23485 invoked by uid 500); 17 Sep 2010 07:35:09 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 23396 invoked by uid 500); 17 Sep 2010 07:35:06 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 23384 invoked by uid 99); 17 Sep 2010 07:35:06 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 17 Sep 2010 07:35:06 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 17 Sep 2010 07:35:02 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id E770C23889E2; Fri, 17 Sep 2010 07:34:41 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r998003 [3/3] - in /hadoop/mapreduce/trunk: ./ conf/ src/c++/task-controller/ src/c++/task-controller/tests/ src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ src/contrib/mumak/src/test/org/apache/hadoop/mapred/ src/docs/src... Date: Fri, 17 Sep 2010 07:34:40 -0000 To: mapreduce-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100917073441.E770C23889E2@eris.apache.org> Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java Fri Sep 17 07:34:39 2010 @@ -18,7 +18,6 @@ package org.apache.hadoop.mapred; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -37,6 +36,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.mapreduce.QueueState; import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; +import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; + import org.junit.After; import org.junit.Test; @@ -48,19 +49,12 @@ public class TestQueueManagerRefresh { private static final Log LOG = LogFactory.getLog(TestQueueManagerRefresh.class); - String queueConfigPath = - System.getProperty("test.build.extraconf", "build/test/extraconf"); - File queueConfigFile = - new File(queueConfigPath, QueueManager.QUEUE_CONF_FILE_NAME); - /** * Remove the configuration file after the test's done. */ @After public void tearDown() { - if (queueConfigFile.exists()) { - queueConfigFile.delete(); - } + deleteQueuesConfigFile(); } /** @@ -96,8 +90,8 @@ public class TestQueueManagerRefresh { JobQueueInfo[] queues = getSimpleQueueHierarchy(); // write the configuration file - QueueManagerTestUtils.writeQueueConfigurationFile( - queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] }); + writeQueueConfigurationFile( + QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); QueueManager qManager = new QueueManager(); @@ -107,8 +101,8 @@ public class TestQueueManagerRefresh { queues[0].addChild(newQueue); // Rewrite the configuration file - QueueManagerTestUtils.writeQueueConfigurationFile( - queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] }); + writeQueueConfigurationFile( + QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); testRefreshFailureWithChangeOfHierarchy(qManager); @@ -127,8 +121,8 @@ public class TestQueueManagerRefresh { JobQueueInfo[] queues = getSimpleQueueHierarchy(); // write the configuration file - QueueManagerTestUtils.writeQueueConfigurationFile( - queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] }); + writeQueueConfigurationFile( + QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); QueueManager qManager = new QueueManager(); @@ -137,8 +131,8 @@ public class TestQueueManagerRefresh { queues[0].removeChild(q2); // Rewrite the configuration file - QueueManagerTestUtils.writeQueueConfigurationFile( - queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] }); + writeQueueConfigurationFile( + QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); testRefreshFailureWithChangeOfHierarchy(qManager); } @@ -187,8 +181,8 @@ public class TestQueueManagerRefresh { JobQueueInfo[] queues = getSimpleQueueHierarchy(); // write the configuration file - QueueManagerTestUtils.writeQueueConfigurationFile( - queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] }); + writeQueueConfigurationFile( + QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); QueueManager qManager = new QueueManager(); @@ -226,8 +220,8 @@ public class TestQueueManagerRefresh { } // write the configuration file - QueueManagerTestUtils.writeQueueConfigurationFile( - queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] }); + writeQueueConfigurationFile( + QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); QueueManager qManager = new QueueManager(); @@ -261,8 +255,8 @@ public class TestQueueManagerRefresh { JobQueueInfo[] queues = getSimpleQueueHierarchy(); // write the configuration file - QueueManagerTestUtils.writeQueueConfigurationFile( - queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] }); + writeQueueConfigurationFile( + QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); QueueManager qManager = new QueueManager(); Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java Fri Sep 17 07:34:39 2010 @@ -38,6 +38,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.mapreduce.QueueState; import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; @@ -48,16 +49,21 @@ import org.apache.hadoop.mapreduce.Sleep import org.apache.hadoop.security.UserGroupInformation; import static org.apache.hadoop.mapred.DeprecatedQueueConfigurationParser.*; import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; +import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName; public class TestQueueManagerWithDeprecatedConf extends TestCase { - static final Log LOG = LogFactory.getLog(TestQueueManagerWithDeprecatedConf.class); - + static final Log LOG = LogFactory.getLog( + TestQueueManagerWithDeprecatedConf.class); + + String submitAcl = QueueACL.SUBMIT_JOB.getAclName(); + String adminAcl = QueueACL.ADMINISTER_JOBS.getAclName(); public void testMultipleQueues() { JobConf conf = new JobConf(); - conf.set("mapred.queue.names", "q1,q2,Q3"); + conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY, + "q1,q2,Q3"); QueueManager qMgr = new QueueManager(conf); Set expQueues = new TreeSet(); expQueues.add("q1"); @@ -68,7 +74,8 @@ public class TestQueueManagerWithDepreca public void testSchedulerInfo() { JobConf conf = new JobConf(); - conf.set("mapred.queue.names", "qq1,qq2"); + conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY, + "qq1,qq2"); QueueManager qMgr = new QueueManager(conf); qMgr.setSchedulerInfo("qq1", "queueInfoForqq1"); qMgr.setSchedulerInfo("qq2", "queueInfoForqq2"); @@ -85,44 +92,50 @@ public class TestQueueManagerWithDepreca try { // queue properties with which the cluster is started. Properties hadoopConfProps = new Properties(); - hadoopConfProps.put("mapred.queue.names", "default,q1,q2"); - hadoopConfProps.put("mapred.acls.enabled", "true"); - UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile); + hadoopConfProps.put(DeprecatedQueueConfigurationParser. + MAPRED_QUEUE_NAMES_KEY, "default,q1,q2"); + hadoopConfProps.put(MRConfig.MR_ACLS_ENABLED, "true"); - //properties for mapred-queue-acls.xml UserGroupInformation ugi = UserGroupInformation.createRemoteUser("unknownUser"); - hadoopConfProps.put("mapred.queue.default.acl-submit-job", ugi.getUserName()); - hadoopConfProps.put("mapred.queue.q1.acl-submit-job", "u1"); - hadoopConfProps.put("mapred.queue.q2.acl-submit-job", "*"); - hadoopConfProps.put("mapred.queue.default.acl-administer-jobs", ugi.getUserName()); - hadoopConfProps.put("mapred.queue.q1.acl-administer-jobs", "u1"); - hadoopConfProps.put("mapred.queue.q2.acl-administer-jobs", "*"); + hadoopConfProps.put(toFullPropertyName( + "default", submitAcl), ugi.getUserName()); + hadoopConfProps.put(toFullPropertyName( + "q1", submitAcl), "u1"); + hadoopConfProps.put(toFullPropertyName( + "q2", submitAcl), "*"); + hadoopConfProps.put(toFullPropertyName( + "default", adminAcl), ugi.getUserName()); + hadoopConfProps.put(toFullPropertyName( + "q1", adminAcl), "u2"); + hadoopConfProps.put(toFullPropertyName( + "q2", adminAcl), "*"); UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile); Configuration conf = new JobConf(); + conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); QueueManager queueManager = new QueueManager(conf); - //Testing access to queue. + //Testing job submission access to queues. assertTrue("User Job Submission failed.", - queueManager.hasAccess("default", Queue.QueueOperation. + queueManager.hasAccess("default", QueueACL. SUBMIT_JOB, ugi)); assertFalse("User Job Submission failed.", - queueManager.hasAccess("q1", Queue.QueueOperation. + queueManager.hasAccess("q1", QueueACL. SUBMIT_JOB, ugi)); assertTrue("User Job Submission failed.", - queueManager.hasAccess("q2", Queue.QueueOperation. + queueManager.hasAccess("q2", QueueACL. SUBMIT_JOB, ugi)); - //Testing the admin acls + //Testing the administer-jobs acls assertTrue("User Job Submission failed.", - queueManager.hasAccess("default", Queue.QueueOperation.ADMINISTER_JOBS, ugi)); - assertFalse("User Job Submission failed.", - queueManager.hasAccess("q1", Queue.QueueOperation. - ADMINISTER_JOBS, ugi)); - assertTrue("User Job Submission failed.", - queueManager.hasAccess("q2", Queue.QueueOperation. - ADMINISTER_JOBS, ugi)); - + queueManager.hasAccess("default", + QueueACL.ADMINISTER_JOBS, ugi)); + assertFalse("User Job Submission failed.", + queueManager.hasAccess("q1", QueueACL. + ADMINISTER_JOBS, ugi)); + assertTrue("User Job Submission failed.", + queueManager.hasAccess("q2", QueueACL. + ADMINISTER_JOBS, ugi)); } finally { //Cleanup the configuration files in all cases Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java Fri Sep 17 07:34:39 2010 @@ -19,8 +19,8 @@ package org.apache.hadoop.mapred; import static org.apache.hadoop.mapred.QueueConfigurationParser.NAME_SEPARATOR; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.CONFIG; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.checkForConfigFile; +import static org.apache.hadoop.mapred.QueueManagerTestUtils.QUEUES_CONFIG_FILE_PATH; +import static org.apache.hadoop.mapred.QueueManagerTestUtils.deleteQueuesConfigFile; import static org.apache.hadoop.mapred.QueueManagerTestUtils.createAcls; import static org.apache.hadoop.mapred.QueueManagerTestUtils.createDocument; import static org.apache.hadoop.mapred.QueueManagerTestUtils.createProperties; @@ -29,7 +29,6 @@ import static org.apache.hadoop.mapred.Q import static org.apache.hadoop.mapred.QueueManagerTestUtils.createSimpleDocumentWithAcls; import static org.apache.hadoop.mapred.QueueManagerTestUtils.createState; import static org.apache.hadoop.mapred.QueueManagerTestUtils.miniMRCluster; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.setUpCluster; import static org.apache.hadoop.mapred.QueueManagerTestUtils.submitSleepJob; import static org.apache.hadoop.mapred.QueueManagerTestUtils.writeToFile; import static org.junit.Assert.assertEquals; @@ -37,7 +36,6 @@ import static org.junit.Assert.assertFal import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.io.File; import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.Properties; @@ -47,11 +45,12 @@ import org.apache.hadoop.mapred.tools.MR import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.QueueState; import org.apache.hadoop.mapreduce.JobStatus.State; import org.apache.hadoop.security.UserGroupInformation; import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -60,21 +59,26 @@ public class TestQueueManagerWithJobTrac private static Configuration conf; - @BeforeClass - public static void setUp() throws Exception { - checkForConfigFile(); + @AfterClass + public static void tearDown() throws Exception { + deleteQueuesConfigFile(); + } + + private void startCluster(boolean aclsEnabled) + throws Exception { + + deleteQueuesConfigFile(); Document doc = createDocument(); - createSimpleDocumentWithAcls(doc, "true"); - writeToFile(doc, CONFIG); + createSimpleDocumentWithAcls(doc); + writeToFile(doc, QUEUES_CONFIG_FILE_PATH); conf = new Configuration(); - conf.addResource(CONFIG); - conf.set("mapred.committer.job.setup.cleanup.needed", "false"); - setUpCluster(conf); - } + conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false"); + conf.setBoolean(MRConfig.MR_ACLS_ENABLED, aclsEnabled); + + JobConf jobConf = new JobConf(conf); + String namenode = "file:///"; + miniMRCluster = new MiniMRCluster(0, namenode, 3, null, null, jobConf); - @AfterClass - public static void tearDown() throws Exception { - new File(CONFIG).delete(); } /** @@ -83,6 +87,8 @@ public class TestQueueManagerWithJobTrac */ @Test(expected = IOException.class) public void testSubmitJobForStoppedQueue() throws Exception { + startCluster(true); + submitSleepJob(10, 10, 100, 100, false, null, "p1" + NAME_SEPARATOR + "p14", conf); fail("queue p1:p14 is in stopped state and should not accept jobs"); @@ -94,8 +100,10 @@ public class TestQueueManagerWithJobTrac */ @Test(expected = IOException.class) public void testSubmitJobForContainerQueue() throws Exception { - submitSleepJob(10, 10, 100, 100, false, null, "p1", conf); - fail("queue p1 is a container queue and cannot have jobs"); + startCluster(true); + + submitSleepJob(10, 10, 100, 100, false, null, "p1", conf); + fail("queue p1 is a container queue and cannot have jobs"); } /** @@ -104,12 +112,16 @@ public class TestQueueManagerWithJobTrac */ @Test public void testAclsForSubmitJob() throws Exception { + startCluster(true); + Job job; + try { // submit job to queue p1:p13 with unspecified acls job = submitSleepJob(0, 0, 0, 0, true, "u1,g1", "p1" + NAME_SEPARATOR + "p13", conf); - assertTrue("Job submission for u1 failed in queue : p1:p13.", - job.isSuccessful()); + fail("user u1 cannot submit jobs to queue p1:p13"); + } catch (Exception e) { + } // check for access to submit the job try { job = submitSleepJob(0, 0, 0, 0, false, "u2,g1", "p1" + NAME_SEPARATOR @@ -117,11 +129,17 @@ public class TestQueueManagerWithJobTrac fail("user u2 cannot submit jobs to queue p1:p11"); } catch (Exception e) { } - // submit job to queue p1:p11 with acls-submit-job as u1 + // submit job to queue p1:p11 with acl-submit-job as u1 job = submitSleepJob(0, 0, 0, 0, true, "u1,g1", "p1" + NAME_SEPARATOR + "p11", conf); assertTrue("Job submission for u1 failed in queue : p1:p11.", job.isSuccessful()); + + // submit job to queue p1:p12 with acl-submit-job as * + job = submitSleepJob(0, 0, 0, 0, true, "u2,g1", "p1" + + NAME_SEPARATOR + "p12", conf); + assertTrue("Job submission for u2 failed in queue : p1:p12.", + job.isSuccessful()); } /** @@ -130,6 +148,8 @@ public class TestQueueManagerWithJobTrac */ @Test public void testAccessToKillJob() throws Exception { + startCluster(true); + Job job = submitSleepJob(1, 1, 100, 100, false, "u1,g1", "p1" + NAME_SEPARATOR + "p11", conf); final JobConf jobConf = miniMRCluster.createJobConf(); @@ -229,11 +249,13 @@ public class TestQueueManagerWithJobTrac */ @Test public void testSubmitJobsAfterRefresh() throws Exception { + startCluster(true); + // test for refresh - checkForConfigFile(); + deleteQueuesConfigFile(); Document doc = createDocument(); refreshDocument(doc); - writeToFile(doc, CONFIG); + writeToFile(doc, QUEUES_CONFIG_FILE_PATH); MRAdmin admin = new MRAdmin(miniMRCluster.createJobConf()); admin.run(new String[] { "-refreshQueues" }); try { @@ -242,15 +264,15 @@ public class TestQueueManagerWithJobTrac fail("user u1 is not in the submit jobs' list"); } catch (Exception e) { } - checkForConfigFile(); + deleteQueuesConfigFile(); doc = createDocument(); - createSimpleDocumentWithAcls(doc, "true"); - writeToFile(doc, CONFIG); + createSimpleDocumentWithAcls(doc); + writeToFile(doc, QUEUES_CONFIG_FILE_PATH); admin.run(new String[] { "-refreshQueues" }); } private void refreshDocument(Document doc) { - Element queues = createQueuesNode(doc, "true"); + Element queues = createQueuesNode(doc); // Create parent level queue q1. Element q1 = createQueue(doc, "q1"); @@ -298,12 +320,7 @@ public class TestQueueManagerWithJobTrac */ @Test public void testAclsDisabled() throws Exception { - checkForConfigFile(); - Document doc = createDocument(); - createSimpleDocumentWithAcls(doc, "false"); - writeToFile(doc, CONFIG); - MRAdmin admin = new MRAdmin(miniMRCluster.createJobConf()); - admin.run(new String[] { "-refreshQueues" }); + startCluster(false); // submit job to queue p1:p11 by any user not in acls-submit-job Job job = submitSleepJob(0, 0, 0, 0, true, "u2,g1", "p1" + NAME_SEPARATOR Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java Fri Sep 17 07:34:39 2010 @@ -32,6 +32,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapred.JobTracker.RecoveryManager; +import org.apache.hadoop.mapreduce.MRConfig; + +import static org.apache.hadoop.mapred.QueueManagerTestUtils.createQueuesConfigFile; +import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.security.UserGroupInformation; @@ -156,11 +160,9 @@ public class TestRecoveryManager extends // clean up FileSystem fs = FileSystem.get(new Configuration()); fs.delete(TEST_DIR, true); - + JobConf conf = new JobConf(); conf.set(JTConfig.JT_JOBHISTORY_BLOCK_SIZE, "1024"); - conf.set( - DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY, "default"); MiniMRCluster mr = new MiniMRCluster(1, "file:///", 1, null, null, conf); JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker(); @@ -239,10 +241,11 @@ public class TestRecoveryManager extends true); mr.getJobTrackerConf().setInt(JTConfig.JT_TASKS_PER_JOB, 25); - mr.getJobTrackerConf().setBoolean("mapred.acls.enabled" , true); + mr.getJobTrackerConf().setBoolean(MRConfig.MR_ACLS_ENABLED, true); + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); - mr.getJobTrackerConf().set("mapred.queue.default.acl-submit-job", - ugi.getUserName()); + mr.getJobTrackerConf().set(toFullPropertyName( + "default", QueueACL.SUBMIT_JOB.getAclName()), ugi.getUserName()); // start the jobtracker LOG.info("Starting jobtracker"); Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java Fri Sep 17 07:34:39 2010 @@ -39,12 +39,12 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.security.TokenCache; -import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier; import org.apache.hadoop.mapreduce.server.tasktracker.Localizer; import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; + +import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Shell; import org.apache.hadoop.mapred.JvmManager.JvmEnv; import org.apache.hadoop.mapred.TaskController.JobInitializationContext; @@ -122,12 +122,14 @@ public class TestTaskTrackerLocalization System.setProperty("hadoop.log.dir", HADOOP_LOG_DIR.getAbsolutePath()); trackerFConf = new JobConf(); + trackerFConf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); localDirs = new String[numLocalDirs]; for (int i = 0; i < numLocalDirs; i++) { localDirs[i] = new File(ROOT_MAPRED_LOCAL_DIR, "0_" + i).getPath(); } trackerFConf.setStrings(MRConfig.LOCAL_DIR, localDirs); + trackerFConf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); // Create the job configuration file. Same as trackerConf in this test. jobConf = new JobConf(trackerFConf); @@ -139,6 +141,15 @@ public class TestTaskTrackerLocalization jobConf.setInt(MRJobConfig.USER_LOG_RETAIN_HOURS, 0); jobConf.setUser(getJobOwner().getShortUserName()); + String queue = "default"; + // set job queue name in job conf + jobConf.setQueueName(queue); + // Set queue admins acl in job conf similar to what JobClient does so that + // it goes into job conf also. + jobConf.set(toFullPropertyName(queue, + QueueACL.ADMINISTER_JOBS.getAclName()), + "qAdmin1,qAdmin2 qAdminsGroup1,qAdminsGroup2"); + Job job = Job.getInstance(jobConf); String jtIdentifier = "200907202331"; jobId = new JobID(jtIdentifier, 1); @@ -527,6 +538,37 @@ public class TestTaskTrackerLocalization .exists()); checkFilePermissions(jobLogDir.toString(), "drwx------", task.getUser(), taskTrackerUGI.getGroupNames()[0]); + + // Make sure that the job ACLs file job-acls.xml exists in job userlog dir + File jobACLsFile = new File(jobLogDir, TaskTracker.jobACLsFile); + assertTrue("JobACLsFile is missing in the job userlog dir " + jobLogDir, + jobACLsFile.exists()); + + // With default task controller, the job-acls.xml file is owned by TT and + // permissions are 700 + checkFilePermissions(jobACLsFile.getAbsolutePath(), "-rwx------", + taskTrackerUGI.getShortUserName(), taskTrackerUGI.getGroupNames()[0]); + + validateJobACLsFileContent(); + } + + // Validate the contents of jobACLsFile ( i.e. user name, job-view-acl, queue + // name and queue-admins-acl ). + protected void validateJobACLsFileContent() { + JobConf jobACLsConf = TaskLogServlet.getConfFromJobACLsFile(jobId); + + assertTrue(jobACLsConf.get("user.name").equals( + localizedJobConf.getUser())); + assertTrue(jobACLsConf.get(MRJobConfig.JOB_ACL_VIEW_JOB). + equals(localizedJobConf.get(MRJobConfig.JOB_ACL_VIEW_JOB))); + + String queue = localizedJobConf.getQueueName(); + assertTrue(queue.equalsIgnoreCase(jobACLsConf.getQueueName())); + + String qACLName = toFullPropertyName(queue, + QueueACL.ADMINISTER_JOBS.getAclName()); + assertTrue(jobACLsConf.get(qACLName).equals( + localizedJobConf.get(qACLName))); } /** @@ -645,24 +687,6 @@ public class TestTaskTrackerLocalization + expectedStderr.toString() + " Observed : " + attemptLogFiles[1].toString(), expectedStderr.toString().equals( attemptLogFiles[1].toString())); - - // Make sure that the job ACLs file exists in the task log dir - File jobACLsFile = new File(logDir, TaskRunner.jobACLsFile); - assertTrue("JobACLsFile is missing in the task log dir " + logDir, - jobACLsFile.exists()); - - // With default task controller, the job-acls file is owned by TT and - // permissions are 700 - checkFilePermissions(jobACLsFile.getAbsolutePath(), "-rwx------", - taskTrackerUGI.getShortUserName(), taskTrackerUGI.getGroupNames()[0]); - - // Validate the contents of jobACLsFile(both user name and job-view-acls) - Configuration jobACLsConf = TaskLogServlet.getConfFromJobACLsFile(task - .getTaskID(), task.isTaskCleanupTask()); - assertTrue(jobACLsConf.get(MRJobConfig.USER_NAME).equals( - localizedJobConf.getUser())); - assertTrue(jobACLsConf.get(MRJobConfig.JOB_ACL_VIEW_JOB). - equals(localizedJobConf.get(MRJobConfig.JOB_ACL_VIEW_JOB))); } /** Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java Fri Sep 17 07:34:39 2010 @@ -55,6 +55,7 @@ public class TestUserLogCleanup { taskLogCleanupThread = new UserLogCleaner(conf); taskLogCleanupThread.setClock(myClock); tt = new TaskTracker(); + tt.setConf(new JobConf(conf)); tt.setLocalizer(localizer); tt.setTaskLogCleanupThread(taskLogCleanupThread); } @@ -67,8 +68,9 @@ public class TestUserLogCleanup { private File localizeJob(JobID jobid) throws IOException { File jobUserlog = TaskLog.getJobDir(jobid); + JobConf conf = new JobConf(); // localize job log directory - tt.initializeJobLogDir(jobid); + tt.initializeJobLogDir(jobid, conf); assertTrue(jobUserlog + " directory is not created.", jobUserlog.exists()); return jobUserlog; } Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java Fri Sep 17 07:34:39 2010 @@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.jobhi import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; + +import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.UserGroupInformation; @@ -57,12 +59,18 @@ public class TestWebUIAuthorization exte private static final Log LOG = LogFactory.getLog( TestWebUIAuthorization.class); - // user1 submits the jobs + // users who submit the jobs private static final String jobSubmitter = "user1"; + private static final String jobSubmitter1 = "user11"; + private static final String jobSubmitter2 = "user12"; + private static final String jobSubmitter3 = "user13"; + // mrOwner starts the cluster private static String mrOwner = null; // member of supergroup private static final String superGroupMember = "user2"; + // admin of "default" queue + private static final String qAdmin = "user3"; // "colleague1" is there in job-view-acls config private static final String viewColleague = "colleague1"; // "colleague2" is there in job-modify-acls config @@ -72,10 +80,17 @@ public class TestWebUIAuthorization exte // "evilJohn" is not having view/modify access on the jobs private static final String unauthorizedUser = "evilJohn"; + @Override protected void setUp() throws Exception { // do not do anything }; + @Override + protected void tearDown() throws Exception { + deleteQueuesConfigFile(); + super.tearDown(); + } + /** access a url, ignoring some IOException such as the page does not exist */ static int getHttpStatusCode(String urlstring, String userName, String method) throws IOException { @@ -112,45 +127,51 @@ public class TestWebUIAuthorization exte * Validates the given jsp/servlet against different user names who * can(or cannot) view the job. * (1) jobSubmitter can view the job - * (2) superGroupMember can view the job - * (3) user mentioned in job-view-acls should be able to view the job - * (4) user mentioned in job-modify-acls but not in job-view-acls + * (2) superGroupMember can view any job + * (3) mrOwner can view any job + * (4) qAdmins of the queue to which job is submitted to can view any job in + * that queue. + * (5) user mentioned in job-view-acl should be able to view the + * job irrespective of job-modify-acl. + * (6) user mentioned in job-modify-acl but not in job-view-acl * cannot view the job - * (5) other unauthorized users cannot view the job + * (7) other unauthorized users cannot view the job */ private void validateViewJob(String url, String method) throws IOException { - assertEquals("Incorrect return code for " + jobSubmitter, + assertEquals("Incorrect return code for job submitter " + jobSubmitter, HttpURLConnection.HTTP_OK, getHttpStatusCode(url, jobSubmitter, method)); - assertEquals("Incorrect return code for " + superGroupMember, - HttpURLConnection.HTTP_OK, getHttpStatusCode(url, superGroupMember, - method)); - assertEquals("Incorrect return code for " + mrOwner, + assertEquals("Incorrect return code for supergroup-member " + + superGroupMember, HttpURLConnection.HTTP_OK, + getHttpStatusCode(url, superGroupMember, method)); + assertEquals("Incorrect return code for MR-owner " + mrOwner, HttpURLConnection.HTTP_OK, getHttpStatusCode(url, mrOwner, method)); - assertEquals("Incorrect return code for " + viewColleague, - HttpURLConnection.HTTP_OK, getHttpStatusCode(url, viewColleague, - method)); - assertEquals("Incorrect return code for " + viewAndModifyColleague, - HttpURLConnection.HTTP_OK, getHttpStatusCode(url, - viewAndModifyColleague, method)); - assertEquals("Incorrect return code for " + modifyColleague, - HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(url, - modifyColleague, method)); - assertEquals("Incorrect return code for " + unauthorizedUser, - HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(url, - unauthorizedUser, method)); + assertEquals("Incorrect return code for queue admin " + qAdmin, + HttpURLConnection.HTTP_OK, getHttpStatusCode(url, qAdmin, method)); + assertEquals("Incorrect return code for user in job-view-acl " + + viewColleague, HttpURLConnection.HTTP_OK, + getHttpStatusCode(url, viewColleague, method)); + assertEquals("Incorrect return code for user in job-view-acl and " + + "job-modify-acl " + viewAndModifyColleague, HttpURLConnection.HTTP_OK, + getHttpStatusCode(url, viewAndModifyColleague, method)); + assertEquals("Incorrect return code for user in job-modify-acl " + + modifyColleague, HttpURLConnection.HTTP_UNAUTHORIZED, + getHttpStatusCode(url, modifyColleague, method)); + assertEquals("Incorrect return code for unauthorizedUser " + + unauthorizedUser, HttpURLConnection.HTTP_UNAUTHORIZED, + getHttpStatusCode(url, unauthorizedUser, method)); } /** * Validates the given jsp/servlet against different user names who * can(or cannot) modify the job. - * (1) jobSubmitter and superGroupMember can modify the job. But we are not - * validating this in this method. Let the caller explicitly validate - * this, if needed. - * (2) user mentioned in job-view-acls but not in job-modify-acls cannot + * (1) jobSubmitter, mrOwner, qAdmin and superGroupMember can modify the job. + * But we are not validating this in this method. Let the caller + * explicitly validate this, if needed. + * (2) user mentioned in job-view-acl but not in job-modify-acl cannot * modify the job - * (3) user mentioned in job-modify-acls (irrespective of job-view-acls) + * (3) user mentioned in job-modify-acl (irrespective of job-view-acl) * can modify the job * (4) other unauthorized users cannot modify the job */ @@ -240,13 +261,13 @@ public class TestWebUIAuthorization exte Properties props = new Properties(); props.setProperty("hadoop.http.filter.initializers", DummyFilterInitializer.class.getName()); - props.setProperty(MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG, + props.setProperty(MRConfig.MR_ACLS_ENABLED, String.valueOf(true)); + props.setProperty("dfs.permissions.enabled", "false"); props.setProperty("mapred.job.tracker.history.completed.location", "historyDoneFolderOnHDFS"); - props.setProperty("mapreduce.job.committer.setup.cleanup.needed", - "false"); + props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false"); props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup"); MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1")); @@ -255,11 +276,17 @@ public class TestWebUIAuthorization exte MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety")); MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup")); MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3")); + MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4")); + mrOwner = UserGroupInformation.getCurrentUser().getShortUserName(); MyGroupsProvider.mapping.put(mrOwner, Arrays.asList( - new String[] { "group4", "group5" })); + new String[] { "group5", "group6" })); + + String[] queueNames = {"default"}; + String[] submitAclStrings = new String[] { jobSubmitter }; + String[] adminsAclStrings = new String[] { qAdmin }; + startCluster(props, queueNames, submitAclStrings, adminsAclStrings); - startCluster(true, props); MiniMRCluster cluster = getMRCluster(); int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort(); @@ -338,30 +365,57 @@ public class TestWebUIAuthorization exte Integer.toString(attemptsMap.get(attempt).getHttpPort()), attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR; validateViewJob(stderrURL, "GET"); + } + } - // delete job-acls.xml file from the task log dir of attempt and verify - // if unauthorized users can view task logs of attempt. - File attemptLogDir = TaskLog.getAttemptDir( - org.apache.hadoop.mapred.TaskAttemptID.downgrade(attempt), false); - Path jobACLsFilePath = new Path(attemptLogDir.toString(), - TaskRunner.jobACLsFile); - new File(jobACLsFilePath.toUri().getPath()).delete(); + // For each tip, let us test the effect of deletion of job-acls.xml file and + // deletion of task log dir for each of the attempts of the tip. + + // delete job-acls.xml file from the job userlog dir and verify + // if unauthorized users can view task logs of each attempt. + Path jobACLsFilePath = new Path(TaskLog.getJobDir(jobid).toString(), + TaskTracker.jobACLsFile); + assertTrue("Could not delete job-acls.xml file.", + new File(jobACLsFilePath.toUri().getPath()).delete()); + + for (TaskID tip : tipsMap.keySet()) { + + Map attemptsMap = + tipsMap.get(tip).getAllTaskAttempts(); + for (TaskAttemptID attempt : attemptsMap.keySet()) { + + String stdoutURL = TaskLogServlet.getTaskLogUrl("localhost", + Integer.toString(attemptsMap.get(attempt).getHttpPort()), + attempt.toString()) + "&filter=" + TaskLog.LogName.STDOUT;; + + String stderrURL = TaskLogServlet.getTaskLogUrl("localhost", + Integer.toString(attemptsMap.get(attempt).getHttpPort()), + attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR; + + // unauthorized users can view task logs of each attempt because + // job-acls.xml file is deleted. assertEquals("Incorrect return code for " + unauthorizedUser, HttpURLConnection.HTTP_OK, getHttpStatusCode(stdoutURL, - unauthorizedUser, "GET")); + unauthorizedUser, "GET")); assertEquals("Incorrect return code for " + unauthorizedUser, HttpURLConnection.HTTP_OK, getHttpStatusCode(stderrURL, - unauthorizedUser, "GET")); + unauthorizedUser, "GET")); // delete the whole task log dir of attempt and verify that we get // correct response code (i.e. HTTP_GONE) when task logs are accessed. + File attemptLogDir = TaskLog.getAttemptDir( + org.apache.hadoop.mapred.TaskAttemptID.downgrade(attempt), false); FileUtil.fullyDelete(attemptLogDir); + + // Try accessing tasklogs - STDOUT and STDERR now(after the whole + // attempt log dir is deleted). assertEquals("Incorrect return code for " + jobSubmitter, HttpURLConnection.HTTP_GONE, getHttpStatusCode(stdoutURL, - jobSubmitter, "GET")); + jobSubmitter, "GET")); + assertEquals("Incorrect return code for " + jobSubmitter, HttpURLConnection.HTTP_GONE, getHttpStatusCode(stderrURL, - jobSubmitter, "GET")); + jobSubmitter, "GET")); } } @@ -377,6 +431,21 @@ public class TestWebUIAuthorization exte } /** + * Creates queues configuration file with the given queues and acls and starts + * cluster with that queues configuration file. + * @param props configuration properties to inject to the mini cluster + * @param queueNames the job queues on the cluster + * @param submitAclStrings acl-submit-job acls for all queues + * @param adminsAclStrings acl-administer-jobs acls for all queues + * @throws Exception + */ + private void startCluster(Properties props, String[] queueNames, + String[] submitAclStrings, String[] adminsAclStrings) throws Exception { + createQueuesConfigFile(queueNames, submitAclStrings, adminsAclStrings); + startCluster(true, props); + } + + /** * Starts a sleep job and tries to kill the job using jobdetails.jsp as * (1) viewColleague (2) unauthorizedUser (3) modifyColleague * (4) viewAndModifyColleague (5) mrOwner (6) superGroupMember and @@ -387,11 +456,13 @@ public class TestWebUIAuthorization exte * (1) jobSubmitter, mrOwner and superGroupMember can do both view and modify * on the job. But we are not validating this in this method. Let the * caller explicitly validate this, if needed. - * (2) user mentioned in job-view-acls but not in job-modify-acls cannot + * (2) user mentioned in job-view-acls and job-modify-acls can do this + * (3) user mentioned in job-view-acls but not in job-modify-acls cannot * do this - * (3) user mentioned in job-modify-acls but not in job-view-acls cannot + * (4) user mentioned in job-modify-acls but not in job-view-acls cannot * do this - * (4) other unauthorized users cannot do this + * (5) qAdmin cannot do this because he doesn't have view access to the job + * (6) other unauthorized users cannot do this * * @throws Exception */ @@ -419,8 +490,11 @@ public class TestWebUIAuthorization exte getHttpStatusCode(url, unauthorizedUser, "POST")); assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(url, modifyColleague, "POST")); + assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(url, viewAndModifyColleague, "POST")); + assertTrue("killJob using jobdetails.jsp failed for a job for which " + + "user has job-view and job-modify permissions", job.isComplete()); } finally { if (!job.isComplete()) { LOG.info("Killing job " + jobid + " from finally block"); @@ -430,14 +504,16 @@ public class TestWebUIAuthorization exte } } - // check if jobSubmitter, mrOwner and superGroupMember can do killJob - // using jobdetails.jsp url + // Check if jobSubmitter, mrOwner, superGroupMember and queueAdmins + // can do killJob using jobdetails.jsp url + confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP, + jobSubmitter); confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP, - jobSubmitter); + mrOwner); confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP, - mrOwner); + superGroupMember); confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP, - superGroupMember); + qAdmin); } /** @@ -486,47 +562,47 @@ public class TestWebUIAuthorization exte // jobTrackerJSP killJob url String url = jobTrackerJSP + "&killJobs=true"; // view-job-acl doesn't matter for killJob from jobtracker jsp page - conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, ""); + conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, " "); - // Let us start jobs as 4 different users(none of these 4 users is + // Let us start 4 jobs as 4 different users(none of these 4 users is // mrOwner and none of these users is a member of superGroup). So only - // based on the config JobContext.JOB_ACL_MODIFY_JOB being set here, - // killJob on each of the jobs will be succeeded. + // based on the config MRJobConfig.JOB_ACL_MODIFY_JOB being set here + // and the jobSubmitter, killJob on each of the jobs will be succeeded. // start 1st job. // Out of these 4 users, only jobSubmitter can do killJob on 1st job - conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, ""); + conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, " "); Job job1 = startSleepJobAsUser(jobSubmitter, conf); org.apache.hadoop.mapreduce.JobID jobid = job1.getJobID(); getTIPId(cluster, jobid);// wait till the map task is started url = url.concat("&jobCheckBox=" + jobid.toString()); // start 2nd job. - // Out of these 4 users, only viewColleague can do killJob on 2nd job - Job job2 = startSleepJobAsUser(viewColleague, conf); + // Out of these 4 users, only jobSubmitter1 can do killJob on 2nd job + Job job2 = startSleepJobAsUser(jobSubmitter1, conf); jobid = job2.getJobID(); getTIPId(cluster, jobid);// wait till the map task is started url = url.concat("&jobCheckBox=" + jobid.toString()); // start 3rd job. - // Out of these 4 users, only modifyColleague can do killJob on 3rd job - Job job3 = startSleepJobAsUser(modifyColleague, conf); + // Out of these 4 users, only jobSubmitter2 can do killJob on 3rd job + Job job3 = startSleepJobAsUser(jobSubmitter2, conf); jobid = job3.getJobID(); getTIPId(cluster, jobid);// wait till the map task is started url = url.concat("&jobCheckBox=" + jobid.toString()); // start 4rd job. - // Out of these 4 users, viewColleague and viewAndModifyColleague + // Out of these 4 users, jobSubmitter1 and jobSubmitter3 // can do killJob on 4th job - conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, viewColleague); - Job job4 = startSleepJobAsUser(viewAndModifyColleague, conf); + conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, jobSubmitter1); + Job job4 = startSleepJobAsUser(jobSubmitter3, conf); jobid = job4.getJobID(); getTIPId(cluster, jobid);// wait till the map task is started url = url.concat("&jobCheckBox=" + jobid.toString()); try { - // Try killing all the 4 jobs as user viewColleague who can kill only + // Try killing all the 4 jobs as user jobSubmitter1 who can kill only // 2nd and 4th jobs. Check if 1st and 3rd jobs are not killed and // 2nd and 4th jobs got killed assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, - getHttpStatusCode(url, viewColleague, "POST")); + getHttpStatusCode(url, jobSubmitter1, "POST")); assertFalse("killJob succeeded for a job for which user doesnot " + " have job-modify permission", job1.isComplete()); assertFalse("killJob succeeded for a job for which user doesnot " @@ -557,12 +633,12 @@ public class TestWebUIAuthorization exte Properties props = new Properties(); props.setProperty("hadoop.http.filter.initializers", DummyFilterInitializer.class.getName()); - props.setProperty( - MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG, String.valueOf(true)); + props.setProperty(MRConfig.MR_ACLS_ENABLED, String.valueOf(true)); + props.setProperty("dfs.permissions.enabled", "false"); props.setProperty(JSPUtil.PRIVATE_ACTIONS_KEY, "true"); - props.setProperty("mapreduce.job.committer.setup.cleanup.needed", "false"); + props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false"); props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup"); MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1")); @@ -571,12 +647,22 @@ public class TestWebUIAuthorization exte MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety")); MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup")); MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3")); + MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4")); mrOwner = UserGroupInformation.getCurrentUser().getShortUserName(); MyGroupsProvider.mapping.put(mrOwner, Arrays.asList( - new String[] { "group4", "group5" })); + new String[] { "group5", "group6" })); + + MyGroupsProvider.mapping.put(jobSubmitter1, Arrays.asList("group7")); + MyGroupsProvider.mapping.put(jobSubmitter2, Arrays.asList("group7")); + MyGroupsProvider.mapping.put(jobSubmitter3, Arrays.asList("group7")); + + String[] queueNames = {"default"}; + String[] submitAclStrings = {jobSubmitter + "," + jobSubmitter1 + "," + + jobSubmitter2 + "," + jobSubmitter3}; + String[] adminsAclStrings = new String[]{qAdmin}; + startCluster(props, queueNames, submitAclStrings, adminsAclStrings); - startCluster(true, props); MiniMRCluster cluster = getMRCluster(); int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort(); @@ -630,8 +716,8 @@ public class TestWebUIAuthorization exte viewAndModifyColleague); confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, jobSubmitter); confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrOwner); - confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, - superGroupMember); + confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, superGroupMember); + confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, qAdmin); // validate killing of multiple jobs using jobtracker jsp and check // if all the jobs which can be killed by user are actually the ones that @@ -676,7 +762,7 @@ public class TestWebUIAuthorization exte "&changeJobPriority=true&setJobPriority="+"HIGH"+"&jobCheckBox=" + jobid.toString(); validateModifyJob(jobTrackerJSPSetJobPriorityAction, "GET"); - // jobSubmitter, mrOwner and superGroupMember are not validated for + // jobSubmitter, mrOwner, qAdmin and superGroupMember are not validated for // job-modify permission in validateModifyJob(). So let us do it // explicitly here assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode( @@ -684,6 +770,8 @@ public class TestWebUIAuthorization exte assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode( jobTrackerJSPSetJobPriorityAction, superGroupMember, "GET")); assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode( + jobTrackerJSPSetJobPriorityAction, qAdmin, "GET")); + assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode( jobTrackerJSPSetJobPriorityAction, mrOwner, "GET")); } @@ -759,6 +847,8 @@ public class TestWebUIAuthorization exte assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(jobTrackerJSP, mrOwner, "GET")); assertEquals(HttpURLConnection.HTTP_OK, + getHttpStatusCode(jobTrackerJSP, qAdmin, "GET")); + assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(jobTrackerJSP, superGroupMember, "GET")); } } Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java Fri Sep 17 07:34:39 2010 @@ -26,11 +26,12 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobACLsManager; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobInProgress; import org.apache.hadoop.mapred.JobTracker; import org.apache.hadoop.mapred.MiniMRCluster; +import org.apache.hadoop.mapred.Operation; +import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.security.UserGroupInformation; import org.junit.Before; @@ -56,23 +57,33 @@ public class TestJobACLs { TestJobACLs.class.getCanonicalName() + Path.SEPARATOR + "completed-job-store"); + private String jobSubmitter = "jobSubmitter"; + private String viewColleague = "viewColleague"; + private String modifyColleague = "modifyColleague"; + private String qAdmin = "qAdmin"; + /** * Start the cluster before running the actual test. * * @throws IOException */ @Before - public void setup() throws IOException { + public void setup() throws Exception { // Start the cluster startCluster(false); } - private void startCluster(boolean reStart) throws IOException { - UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); + private void startCluster(boolean reStart) throws Exception { + + // Configure job queues + String[] queueNames = {"default"}; + createQueuesConfigFile(queueNames, + new String[] { jobSubmitter }, new String[] { qAdmin }); + JobConf conf = new JobConf(); - // Enable job-level authorization - conf.setBoolean(MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG, true); + // Enable queue and job level authorization + conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); // Enable CompletedJobStore FileSystem fs = FileSystem.getLocal(conf); @@ -84,6 +95,7 @@ public class TestJobACLs { conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, true); conf.set(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1"); + UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); mr = new MiniMRCluster(0, 0, 1, "file:///", 1, null, null, MR_UGI, conf); } @@ -92,6 +104,7 @@ public class TestJobACLs { */ @After public void tearDown() { + deleteQueuesConfigFile(); if (mr != null) { mr.shutdown(); } @@ -106,10 +119,10 @@ public class TestJobACLs { * @throws ClassNotFoundException */ @Test - public void testACLS() throws IOException, InterruptedException, - ClassNotFoundException { + public void testACLS() throws Exception { verifyACLViewJob(); - verifyACLModifyJob(); + verifyACLModifyJob(modifyColleague); + verifyACLModifyJob(qAdmin); verifyACLPersistence(); } @@ -123,18 +136,21 @@ public class TestJobACLs { // Set the job up. final Configuration myConf = mr.createJobConf(); - myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user1,user3"); + myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague); // Submit the job as user1 - Job job = submitJobAsUser(myConf, "user1"); + Job job = submitJobAsUser(myConf, jobSubmitter); final JobID jobId = job.getJobID(); // Try operations as an unauthorized user. - verifyViewJobAsUnauthorizedUser(myConf, jobId, "user2"); + verifyViewJobAsUnauthorizedUser(myConf, jobId, modifyColleague); - // Try operations as an authorized user. - verifyViewJobAsAuthorizedUser(myConf, jobId, "user3"); + // Try operations as an authorized user, who is part of view-job-acl. + verifyViewJobAsAuthorizedUser(myConf, jobId, viewColleague); + + // Try operations as an authorized user, who is a queue administrator. + verifyViewJobAsAuthorizedUser(myConf, jobId, qAdmin); // Clean up the job job.killJob(); @@ -242,7 +258,7 @@ public class TestJobACLs { fail("AccessControlException expected.."); } catch (IOException ioe) { assertTrue(ioe.getMessage().contains( - JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.VIEW_JOB)); + " cannot perform operation " + JobACL.VIEW_JOB)); } catch (InterruptedException e) { fail("Exception .. interrupted.." + e); } @@ -253,7 +269,7 @@ public class TestJobACLs { fail("AccessControlException expected.."); } catch (IOException ioe) { assertTrue(ioe.getMessage().contains( - JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.VIEW_JOB)); + " cannot perform operation " + JobACL.VIEW_JOB)); } catch (InterruptedException e) { fail("Exception .. interrupted.." + e); } @@ -264,29 +280,29 @@ public class TestJobACLs { } /** - * Verify JobContext.Job_ACL_MODIFY_JOB + * Verify MRConfig.Job_ACL_MODIFY_JOB * * @throws IOException * @throws InterruptedException * @throws ClassNotFoundException */ - private void verifyACLModifyJob() throws IOException, + private void verifyACLModifyJob(String authorizedUser) throws IOException, InterruptedException, ClassNotFoundException { // Set the job up. final Configuration myConf = mr.createJobConf(); - myConf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, "user1,user3"); + myConf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, modifyColleague); // Submit the job as user1 - Job job = submitJobAsUser(myConf, "user1"); + Job job = submitJobAsUser(myConf, jobSubmitter); final JobID jobId = job.getJobID(); // Try operations as an unauthorized user. - verifyModifyJobAsUnauthorizedUser(myConf, jobId, "user2"); + verifyModifyJobAsUnauthorizedUser(myConf, jobId, viewColleague); // Try operations as an authorized user. - verifyModifyJobAsAuthorizedUser(myConf, jobId, "user3"); + verifyModifyJobAsAuthorizedUser(myConf, jobId, authorizedUser); } private void verifyModifyJobAsAuthorizedUser( @@ -357,7 +373,7 @@ public class TestJobACLs { fail("AccessControlException expected.."); } catch (IOException ioe) { assertTrue(ioe.getMessage().contains( - JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.MODIFY_JOB)); + " cannot perform operation " + Operation.KILL_JOB)); } catch (InterruptedException e) { fail("Exception .. interrupted.." + e); } @@ -368,7 +384,7 @@ public class TestJobACLs { fail("AccessControlException expected.."); } catch (IOException ioe) { assertTrue(ioe.getMessage().contains( - JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.MODIFY_JOB)); + " cannot perform operation " + Operation.SET_JOB_PRIORITY)); } catch (InterruptedException e) { fail("Exception .. interrupted.." + e); } @@ -378,15 +394,14 @@ public class TestJobACLs { }); } - private void verifyACLPersistence() throws IOException, - InterruptedException { + private void verifyACLPersistence() throws Exception { // Set the job up. final Configuration myConf = mr.createJobConf(); - myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user2 group2"); + myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague + " group2"); // Submit the job as user1 - Job job = submitJobAsUser(myConf, "user1"); + Job job = submitJobAsUser(myConf, jobSubmitter); final JobID jobId = job.getJobID(); @@ -406,11 +421,14 @@ public class TestJobACLs { final Configuration myNewJobConf = mr.createJobConf(); // Now verify view-job works off CompletedJobStore - verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, "user2"); + verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, viewColleague); + verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, qAdmin); // Only JobCounters is persisted on the JobStore. So test counters only. UserGroupInformation unauthorizedUGI = - UserGroupInformation.createUserForTesting("user3", new String[] {}); + UserGroupInformation.createUserForTesting( + modifyColleague, new String[] {}); + unauthorizedUGI.doAs(new PrivilegedExceptionAction() { @SuppressWarnings("null") @Override @@ -432,7 +450,7 @@ public class TestJobACLs { fail("AccessControlException expected.."); } catch (IOException ioe) { assertTrue(ioe.getMessage().contains( - JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.VIEW_JOB)); + " cannot perform operation " + Operation.VIEW_JOB_COUNTERS)); } catch (InterruptedException e) { fail("Exception .. interrupted.." + e); } Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java (original) +++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java Fri Sep 17 07:34:39 2010 @@ -21,10 +21,7 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; -import org.apache.avro.Schema; -import org.apache.avro.SchemaParseException; import org.apache.hadoop.mapreduce.jobhistory.EventReader; -import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent; import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent; import org.apache.hadoop.mapreduce.jobhistory.JobHistory; Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java (original) +++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java Fri Sep 17 07:34:39 2010 @@ -17,7 +17,6 @@ */ package org.apache.hadoop.tools.rumen; -import java.text.ParseException; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -72,7 +71,11 @@ public class Job20LineHistoryEventEmitte String submitTime = line.get("SUBMIT_TIME"); String jobConf = line.get("JOBCONF"); String user = line.get("USER"); + if (user == null) { + user = "nulluser"; + } String jobName = line.get("JOBNAME"); + String jobQueueName = line.get("JOB_QUEUE");// could be null if (submitTime != null) { Job20LineHistoryEventEmitter that = @@ -82,8 +85,8 @@ public class Job20LineHistoryEventEmitte Map jobACLs = new HashMap(); - return new JobSubmittedEvent(jobID, jobName, user == null ? "nulluser" - : user, that.originalSubmitTime, jobConf, jobACLs); + return new JobSubmittedEvent(jobID, jobName, user, + that.originalSubmitTime, jobConf, jobACLs, jobQueueName); } return null; Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=998003&r1=998002&r2=998003&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java (original) +++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java Fri Sep 17 07:34:39 2010 @@ -544,6 +544,8 @@ public class JobBuilder { result.setJobName(event.getJobName()); result.setUser(event.getUserName()); result.setSubmitTime(event.getSubmitTime()); + // job queue name is set when conf file is processed. + // See JobBuilder.process(Properties) method for details. } private void processJobStatusChangedEvent(JobStatusChangedEvent event) {