Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 86280 invoked from network); 16 Sep 2009 04:49:48 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 16 Sep 2009 04:49:48 -0000 Received: (qmail 63193 invoked by uid 500); 16 Sep 2009 04:49:48 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 63142 invoked by uid 500); 16 Sep 2009 04:49:48 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 63129 invoked by uid 99); 16 Sep 2009 04:49:48 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Sep 2009 04:49:48 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Sep 2009 04:49:43 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id EA0B02388907; Wed, 16 Sep 2009 04:49:20 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r815605 [3/3] - in /hadoop/mapreduce/trunk: ./ conf/ src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/ src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ src/contrib/dynamic-scheduler/src/java/org/apache/hadoo... Date: Wed, 16 Sep 2009 04:49:20 -0000 To: mapreduce-commits@hadoop.apache.org From: yhemanth@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090916044920.EA0B02388907@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java?rev=815605&r1=815604&r2=815605&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java Wed Sep 16 04:49:18 2009 @@ -40,6 +40,7 @@ import org.apache.hadoop.mapreduce.SleepJob; import org.apache.hadoop.security.UnixUserGroupInformation; import org.apache.hadoop.security.UserGroupInformation; +import static org.apache.hadoop.mapred.DeprecatedQueueConfigurationParser.*; public class TestQueueManager extends TestCase { @@ -47,16 +48,6 @@ private MiniDFSCluster miniDFSCluster; private MiniMRCluster miniMRCluster; - - public void testDefaultQueueConfiguration() { - JobConf conf = new JobConf(); - QueueManager qMgr = new QueueManager(conf); - Set expQueues = new TreeSet(); - expQueues.add("default"); - verifyQueues(expQueues, qMgr.getQueues()); - // pass true so it will fail if the key is not found. - assertFalse(conf.getBoolean("mapred.acls.enabled", true)); - } public void testMultipleQueues() { JobConf conf = new JobConf(); @@ -66,9 +57,9 @@ expQueues.add("q1"); expQueues.add("q2"); expQueues.add("Q3"); - verifyQueues(expQueues, qMgr.getQueues()); + verifyQueues(expQueues, qMgr.getLeafQueueNames()); } - + public void testSchedulerInfo() { JobConf conf = new JobConf(); conf.set("mapred.queue.names", "qq1,qq2"); @@ -78,27 +69,27 @@ assertEquals(qMgr.getSchedulerInfo("qq2"), "queueInfoForqq2"); assertEquals(qMgr.getSchedulerInfo("qq1"), "queueInfoForqq1"); } - - public void testAllEnabledACLForJobSubmission() + + public void testAllEnabledACLForJobSubmission() throws IOException, InterruptedException, ClassNotFoundException { JobConf conf = setupConf("mapred.queue.default.acl-submit-job", "*"); verifyJobSubmission(conf, true); } - - public void testAllDisabledACLForJobSubmission() + + public void testAllDisabledACLForJobSubmission() throws IOException, InterruptedException, ClassNotFoundException { JobConf conf = setupConf("mapred.queue.default.acl-submit-job", ""); verifyJobSubmission(conf, false); } - - public void testUserDisabledACLForJobSubmission() + + public void testUserDisabledACLForJobSubmission() throws IOException, InterruptedException, ClassNotFoundException { - JobConf conf = setupConf("mapred.queue.default.acl-submit-job", + JobConf conf = setupConf("mapred.queue.default.acl-submit-job", "3698-non-existent-user"); verifyJobSubmission(conf, false); } - - public void testDisabledACLForNonDefaultQueue() + + public void testDisabledACLForNonDefaultQueue() throws IOException, InterruptedException, ClassNotFoundException { // allow everyone in default queue JobConf conf = setupConf("mapred.queue.default.acl-submit-job", "*"); @@ -109,8 +100,8 @@ // verify job submission to other queue fails. verifyJobSubmission(conf, false, "q1"); } - - public void testSubmissionToInvalidQueue() + + public void testSubmissionToInvalidQueue() throws IOException, InterruptedException, ClassNotFoundException { JobConf conf = new JobConf(); conf.set("mapred.queue.names","default"); @@ -118,15 +109,17 @@ String queueName = "q1"; try { Job rjob = submitSleepJob(1, 1, 100, 100, true, null, queueName); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains("Queue \"" + queueName + "\" does not exist")); + } catch (IOException ioe) { + assertTrue(ioe.getMessage().contains("Queue \"" + queueName + "\" does " + + "not exist")); return; } finally { tearDownCluster(); } - fail("Job submission to invalid queue job shouldnot complete , it should fail with proper exception "); + fail("Job submission to invalid queue job shouldnot complete " + + ", it should fail with proper exception "); } - + public void testEnabledACLForNonDefaultQueue() throws IOException, LoginException, InterruptedException, ClassNotFoundException { // login as self... @@ -141,61 +134,61 @@ // verify job submission to other queue fails. verifyJobSubmission(conf, true, "q2"); } - - public void testUserEnabledACLForJobSubmission() - throws IOException, LoginException, + + public void testUserEnabledACLForJobSubmission() + throws IOException, LoginException, InterruptedException, ClassNotFoundException { // login as self... UserGroupInformation ugi = UnixUserGroupInformation.login(); String userName = ugi.getUserName(); JobConf conf = setupConf("mapred.queue.default.acl-submit-job", - "3698-junk-user," + userName + "3698-junk-user," + userName + " 3698-junk-group1,3698-junk-group2"); verifyJobSubmission(conf, true); } - - public void testGroupsEnabledACLForJobSubmission() - throws IOException, LoginException, + + public void testGroupsEnabledACLForJobSubmission() + throws IOException, LoginException, InterruptedException, ClassNotFoundException { // login as self, get one group, and add in allowed list. UserGroupInformation ugi = UnixUserGroupInformation.login(); String[] groups = ugi.getGroupNames(); assertTrue(groups.length > 0); JobConf conf = setupConf("mapred.queue.default.acl-submit-job", - "3698-junk-user1,3698-junk-user2 " - + groups[groups.length-1] + "3698-junk-user1,3698-junk-user2 " + + groups[groups.length-1] + ",3698-junk-group"); verifyJobSubmission(conf, true); } - - public void testAllEnabledACLForJobKill() + + public void testAllEnabledACLForJobKill() throws IOException, InterruptedException, ClassNotFoundException { JobConf conf = setupConf("mapred.queue.default.acl-administer-jobs", "*"); verifyJobKill(conf, true); } - public void testAllDisabledACLForJobKill() + public void testAllDisabledACLForJobKill() throws IOException, InterruptedException, ClassNotFoundException { JobConf conf = setupConf("mapred.queue.default.acl-administer-jobs", ""); verifyJobKillAsOtherUser(conf, false, "dummy-user,dummy-user-group"); } - - public void testOwnerAllowedForJobKill() + + public void testOwnerAllowedForJobKill() throws IOException, InterruptedException, ClassNotFoundException { - JobConf conf = setupConf("mapred.queue.default.acl-administer-jobs", + JobConf conf = setupConf("mapred.queue.default.acl-administer-jobs", "junk-user"); verifyJobKill(conf, true); } - - public void testUserDisabledACLForJobKill() + + public void testUserDisabledACLForJobKill() throws IOException, InterruptedException, ClassNotFoundException { //setup a cluster allowing a user to submit - JobConf conf = setupConf("mapred.queue.default.acl-administer-jobs", + JobConf conf = setupConf("mapred.queue.default.acl-administer-jobs", "dummy-user"); verifyJobKillAsOtherUser(conf, false, "dummy-user,dummy-user-group"); } - - public void testUserEnabledACLForJobKill() throws IOException, + + public void testUserEnabledACLForJobKill() throws IOException, LoginException, InterruptedException, ClassNotFoundException { // login as self... UserGroupInformation ugi = UnixUserGroupInformation.login(); @@ -204,25 +197,24 @@ "dummy-user,"+userName); verifyJobKillAsOtherUser(conf, true, "dummy-user,dummy-user-group"); } - - public void testUserDisabledForJobPriorityChange() + + public void testUserDisabledForJobPriorityChange() throws IOException, InterruptedException, ClassNotFoundException { JobConf conf = setupConf("mapred.queue.default.acl-administer-jobs", "junk-user"); - verifyJobPriorityChangeAsOtherUser(conf, false, + verifyJobPriorityChangeAsOtherUser(conf, false, "junk-user,junk-user-group"); } - + /** * Test to verify refreshing of queue properties by using MRAdmin tool. - * + * * @throws Exception */ + public void testACLRefresh() throws Exception { String queueConfigPath = System.getProperty("test.build.extraconf", "build/test/extraconf"); - File queueConfigFile = - new File(queueConfigPath, QueueManager.QUEUE_CONF_FILE_NAME); File hadoopConfigFile = new File(queueConfigPath, "mapred-site.xml"); try { //Setting up default mapred-site.xml @@ -234,18 +226,16 @@ hadoopConfProps.put("mapred.queue.default.acl-submit-job", "u1"); hadoopConfProps.put("mapred.queue.q1.acl-submit-job", "u2"); hadoopConfProps.put("mapred.queue.q2.acl-submit-job", "u1"); - UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile); - //Actual property which would be used. - Properties queueConfProps = new Properties(); - queueConfProps.put("mapred.queue.default.acl-submit-job", " "); + hadoopConfProps.put("mapred.queue.default.acl-submit-job", " "); //Writing out the queue configuration file. - UtilsForTests.setUpConfigFile(queueConfProps, queueConfigFile); - + UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile); + //Create a new configuration to be used with QueueManager JobConf conf = new JobConf(); QueueManager queueManager = new QueueManager(conf); - UserGroupInformation ugi = UnixUserGroupInformation.getCurrentUGI(); + UserGroupInformation ugi = + new UnixUserGroupInformation("unknownUser",new String[]{" "}); //Job Submission should fail because ugi to be used is set to blank. assertFalse("User Job Submission Succeeded before refresh.", queueManager.hasAccess("default", Queue.QueueOperation. @@ -256,22 +246,23 @@ assertFalse("User Job Submission Succeeded before refresh.", queueManager.hasAccess("q2", Queue.QueueOperation. SUBMIT_JOB, ugi)); - + //Test job submission as alternate user. Configuration alternateUserConfig = new Configuration(); alternateUserConfig.set("hadoop.job.ugi","u1,users"); - UserGroupInformation alternateUgi = + UserGroupInformation alternateUgi = UserGroupInformation.readFrom(alternateUserConfig); assertTrue("Alternate User Job Submission failed before refresh.", queueManager.hasAccess("q2", Queue.QueueOperation. SUBMIT_JOB, alternateUgi)); - + //Set acl for the current user. - queueConfProps.put("mapred.queue.default.acl-submit-job", ugi.getUserName()); - queueConfProps.put("mapred.queue.q1.acl-submit-job", ugi.getUserName()); - queueConfProps.put("mapred.queue.q2.acl-submit-job", ugi.getUserName()); - //write out queue-acls.xml. - UtilsForTests.setUpConfigFile(queueConfProps, queueConfigFile); + hadoopConfProps.put(MAPRED_QUEUE_NAMES_KEY, "default,q1,q2"); + hadoopConfProps.put("mapred.acls.enabled", "true"); + hadoopConfProps.put("mapred.queue.default.acl-submit-job", ugi.getUserName()); + hadoopConfProps.put("mapred.queue.q1.acl-submit-job", ugi.getUserName()); + hadoopConfProps.put("mapred.queue.q2.acl-submit-job", ugi.getUserName()); + UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile); //refresh configuration queueManager.refreshQueues(conf); //Submission should succeed @@ -287,10 +278,8 @@ assertFalse("Alternate User Job Submission succeeded after refresh.", queueManager.hasAccess("q2", Queue.QueueOperation. SUBMIT_JOB, alternateUgi)); - //delete the ACL file. - queueConfigFile.delete(); - //rewrite the mapred-site.xml + hadoopConfProps.put("mapred.queue.names", "default,q1,q2"); hadoopConfProps.put("mapred.acls.enabled", "true"); hadoopConfProps.put("mapred.queue.default.acl-submit-job", ugi.getUserName()); UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile); @@ -299,17 +288,12 @@ queueManager.hasAccess("default", Queue.QueueOperation. SUBMIT_JOB, ugi)); } finally{ - if(queueConfigFile.exists()) { - queueConfigFile.delete(); - } if(hadoopConfigFile.exists()) { hadoopConfigFile.delete(); } } } - - /** * Test to verify refreshing of queue properties by using MRAdmin tool. * @@ -318,8 +302,8 @@ public void testStateRefresh() throws Exception { String queueConfigPath = System.getProperty("test.build.extraconf", "build/test/extraconf"); - File queueConfigFile = - new File(queueConfigPath, QueueManager.QUEUE_CONF_FILE_NAME); + File hadoopConfigFile = new File(queueConfigPath, "mapred-site.xml"); + try { //Setting up default mapred-site.xml Properties queueConfProps = new Properties(); @@ -329,12 +313,12 @@ //These property should always be overridden queueConfProps.put("mapred.queue.default.state", "running"); queueConfProps.put("mapred.queue.qu1.state", "stopped"); - UtilsForTests.setUpConfigFile(queueConfProps, queueConfigFile); + UtilsForTests.setUpConfigFile(queueConfProps, hadoopConfigFile); //Create a new configuration to be used with QueueManager JobConf conf = new JobConf(); setUpCluster(conf); - QueueManager queueManager = + QueueManager queueManager = this.miniMRCluster.getJobTrackerRunner().getJobTracker().getQueueManager(); try{ @@ -353,15 +337,16 @@ // verify state of queues before refresh JobQueueInfo queueInfo = queueManager.getJobQueueInfo("default"); - assertEquals(Queue.QueueState.RUNNING.getStateName(), + assertEquals(Queue.QueueState.RUNNING.getStateName(), queueInfo.getQueueState()); queueInfo = queueManager.getJobQueueInfo("qu1"); assertEquals(Queue.QueueState.STOPPED.getStateName(), queueInfo.getQueueState()); - + queueConfProps.put("mapred.queue.names", "default,qu1"); + queueConfProps.put("mapred.acls.enabled", "true"); queueConfProps.put("mapred.queue.default.state", "stopped"); queueConfProps.put("mapred.queue.qu1.state", "running"); - UtilsForTests.setUpConfigFile(queueConfProps, queueConfigFile); + UtilsForTests.setUpConfigFile(queueConfProps, hadoopConfigFile); //refresh configuration queueManager.refreshQueues(conf); @@ -379,17 +364,17 @@ }catch(Exception e){ assert(e.getMessage().contains("Queue \"" + "default" + "\" is not running")); } - + // verify state of queues after refresh queueInfo = queueManager.getJobQueueInfo("default"); - assertEquals(Queue.QueueState.STOPPED.getStateName(), + assertEquals(Queue.QueueState.STOPPED.getStateName(), queueInfo.getQueueState()); queueInfo = queueManager.getJobQueueInfo("qu1"); assertEquals(Queue.QueueState.RUNNING.getStateName(), queueInfo.getQueueState()); } finally{ - if(queueConfigFile.exists()) { - queueConfigFile.delete(); + if(hadoopConfigFile.exists()) { + hadoopConfigFile.delete(); } this.tearDownCluster(); } @@ -398,24 +383,23 @@ public void testQueueAclRefreshWithInvalidConfFile() throws IOException { String queueConfigPath = System.getProperty("test.build.extraconf", "build/test/extraconf"); - File queueConfigFile = - new File(queueConfigPath, QueueManager.QUEUE_CONF_FILE_NAME); - File hadoopConfigFile = new File(queueConfigPath, "hadoop-site.xml"); + + File hadoopConfigFile = new File(queueConfigPath, "mapred-site.xml"); try { // queue properties with which the cluster is started. Properties hadoopConfProps = new Properties(); hadoopConfProps.put("mapred.queue.names", "default,q1,q2"); hadoopConfProps.put("mapred.acls.enabled", "true"); UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile); - + //properties for mapred-queue-acls.xml - Properties queueConfProps = new Properties(); - UserGroupInformation ugi = UnixUserGroupInformation.getCurrentUGI(); - queueConfProps.put("mapred.queue.default.acl-submit-job", ugi.getUserName()); - queueConfProps.put("mapred.queue.q1.acl-submit-job", ugi.getUserName()); - queueConfProps.put("mapred.queue.q2.acl-submit-job", ugi.getUserName()); - UtilsForTests.setUpConfigFile(queueConfProps, queueConfigFile); - + UserGroupInformation ugi = + new UnixUserGroupInformation("unknownUser",new String[]{" "}); + hadoopConfProps.put("mapred.queue.default.acl-submit-job", ugi.getUserName()); + hadoopConfProps.put("mapred.queue.q1.acl-submit-job", ugi.getUserName()); + hadoopConfProps.put("mapred.queue.q2.acl-submit-job", ugi.getUserName()); + UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile); + Configuration conf = new JobConf(); QueueManager queueManager = new QueueManager(conf); //Testing access to queue. @@ -428,9 +412,9 @@ assertTrue("User Job Submission failed.", queueManager.hasAccess("q2", Queue.QueueOperation. SUBMIT_JOB, ugi)); - + //Write out a new incomplete invalid configuration file. - PrintWriter writer = new PrintWriter(new FileOutputStream(queueConfigFile)); + PrintWriter writer = new PrintWriter(new FileOutputStream(hadoopConfigFile)); writer.println(""); writer.println(""); writer.flush(); @@ -456,15 +440,15 @@ if(hadoopConfigFile.exists()) { hadoopConfigFile.delete(); } - if(queueConfigFile.exists()) { - queueConfigFile.delete(); - } } } private JobConf setupConf(String aclName, String aclValue) { JobConf conf = new JobConf(); + if(conf.get("mapred.queue.names") == null) { + conf.set("mapred.queue.names","default"); + } conf.setBoolean("mapred.acls.enabled", true); conf.set(aclName, aclValue); return conf; @@ -486,6 +470,9 @@ private void verifyJobSubmission(JobConf conf, boolean shouldSucceed, String queue) throws IOException, InterruptedException, ClassNotFoundException { + if(conf.get("mapred.queue.names") == null) { + conf.set("mapred.queue.names","default"); + } setUpCluster(conf); try { runAndVerifySubmission(conf, shouldSucceed, queue, null); @@ -649,6 +636,8 @@ } private void setUpCluster(JobConf conf) throws IOException { + if(conf.get("mapred.queue.names") == null) + conf.set("mapred.queue.names","default"); miniDFSCluster = new MiniDFSCluster(conf, 1, true, null); FileSystem fileSys = miniDFSCluster.getFileSystem(); String namenode = fileSys.getUri().toString(); Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java?rev=815605&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java (added) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java Wed Sep 16 04:49:18 2009 @@ -0,0 +1,379 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import junit.framework.TestCase; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; +import static org.apache.hadoop.mapred.QueueConfigurationParser.*; +import org.apache.hadoop.security.UnixUserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Set; + + +public class TestQueueManagerForHierarchialQueues extends TestCase { + + private static final Log LOG = LogFactory.getLog( + TestQueueManagerForHierarchialQueues.class); + + + protected void tearDown() throws Exception { + super.tearDown(); + new File(CONFIG).delete(); + } + + public void testDefault() throws Exception { + QueueManager qm = new QueueManager(); + Queue root = qm.getRoot(); + assertEquals(root.getChildren().size(), 1); + assertEquals(root.getChildren().iterator().next().getName(), "default"); + assertFalse(qm.isAclsEnabled()); + assertNull(root.getChildren().iterator().next().getChildren()); + } + + public void testXMLParsing() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + createSimpleDocument(doc); + writeToFile(doc, CONFIG); + QueueManager qm = new QueueManager(CONFIG); + Set rootQueues = qm.getRoot().getChildren(); + List names = new ArrayList(); + for (Queue q : rootQueues) { + names.add(q.getName()); + } + + //Size of root. + assertEquals(rootQueues.size(), 2); + + //check root level queues + assertTrue(names.contains("q1")); + assertTrue(names.contains("p1")); + + + //check for leaf names + Set leafNames = qm.getLeafQueueNames(); + Queue p = qm.getQueue("p1"); + Set children = p.getChildren(); + assertTrue(children.size() == 2); + + //check leaf level queues + assertTrue( + leafNames.contains( + "p1" + NAME_SEPARATOR + "p11")); + assertTrue( + leafNames.contains( + "p1" + NAME_SEPARATOR + "p12")); + + + Queue q = qm.getQueue( + "p1" + NAME_SEPARATOR + "p12"); + + assertTrue( + q.getAcls().get( + QueueManager.toFullPropertyName( + q.getName(), ACL_SUBMIT_JOB_TAG)).getUsers().contains( + "u1")); + + assertTrue( + q.getAcls().get( + QueueManager.toFullPropertyName( + q.getName(), + ACL_ADMINISTER_JOB_TAG)) + .getUsers().contains("u2")); + assertTrue(q.getState().equals(Queue.QueueState.STOPPED)); + } + + public void testhasAccess() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + createSimpleDocument(doc); + writeToFile(doc, CONFIG); + QueueManager qm = new QueueManager(CONFIG); + + UserGroupInformation ugi = + new UnixUserGroupInformation("u1", new String[]{" "}); + assertTrue( + qm.hasAccess( + "p1" + NAME_SEPARATOR + "p12", + Queue.QueueOperation.SUBMIT_JOB, ugi)); + } + + public void testhasAccessForParent() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + createSimpleDocument(doc); + writeToFile(doc, CONFIG); + QueueManager qm = new QueueManager(CONFIG); + + UserGroupInformation ugi = + new UnixUserGroupInformation("u1", new String[]{" "}); + assertFalse( + qm.hasAccess( + "p1", + Queue.QueueOperation.SUBMIT_JOB, ugi)); + } + + public void testValidation() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + Element queues = createQueuesNode(doc, "false"); + Element q1 = createQueue(doc, "q1"); + + q1.appendChild(createAcls(doc, "acl-submit-job", "u1")); + q1.appendChild(createAcls(doc, "acl-administer-jobs", "u2")); + q1.appendChild(createQueue(doc, "p15")); + q1.appendChild(createQueue(doc, "p16")); + + queues.appendChild(q1); + writeToFile(doc, CONFIG); + try { + new QueueManager(CONFIG); + fail("Should throw an exception as configuration is wrong "); + } catch (RuntimeException re) { + LOG.info(re.getMessage()); + } + } + + public void testInvalidName() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + Element queues = createQueuesNode(doc, "false"); + Element q1 = createQueue(doc, ""); + queues.appendChild(q1); + writeToFile(doc, CONFIG); + try { + new QueueManager(CONFIG); + fail("Should throw an exception as configuration is wrong "); + } catch (Exception re) { + re.printStackTrace(); + LOG.info(re.getMessage()); + } + checkForConfigFile(); + doc = createDocument(); + queues = createQueuesNode(doc, "false"); + q1 = doc.createElement("queue"); + queues.appendChild(q1); + writeToFile(doc, CONFIG); + try { + new QueueManager(CONFIG); + fail("Should throw an exception as configuration is wrong "); + } catch (RuntimeException re) { + re.printStackTrace(); + LOG.info(re.getMessage()); + } + } + + + public void testEmptyProperties() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + Element queues = createQueuesNode(doc, "false"); + Element q1 = createQueue(doc, "q1"); + Element p = createProperties(doc, null); + q1.appendChild(p); + queues.appendChild(q1); + } + + public void testEmptyFile() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + writeToFile(doc, CONFIG); + try { + new QueueManager(CONFIG); + fail("Should throw an exception as configuration is wrong "); + } catch (Exception re) { + re.printStackTrace(); + LOG.info(re.getMessage()); + } + } + + public void testJobQueueInfoGeneration() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + createSimpleDocument(doc); + writeToFile(doc, CONFIG); + QueueManager qm = new QueueManager(CONFIG); + + List rootQueues = + qm.getRoot().getJobQueueInfo().getChildren(); + assertEquals(rootQueues.size(), 2); + List names = new ArrayList(); + for (JobQueueInfo q : rootQueues) { + names.add(q.getQueueName()); + if (q.getQueueName().equals("q1")) { + Properties p = q.getProperties(); + assertEquals(p.getProperty("capacity"), "10"); + assertEquals(p.getProperty("maxCapacity"), "35"); + + assertTrue(q.getChildren().isEmpty()); + } else if (q.getQueueName().equals("p1")) { + List children = q.getChildren(); + assertEquals(children.size(), 2); + for (JobQueueInfo child : children) { + if (child.getQueueName().equals( + "p1" + NAME_SEPARATOR + "p12")) { + assertEquals( + child.getQueueState(), Queue.QueueState.STOPPED.getStateName()); + } else if (child.getQueueName().equals( + "p1" + NAME_SEPARATOR + "p11")) { + assertEquals( + child.getQueueState(), Queue.QueueState.RUNNING.getStateName()); + } else { + fail("Only 2 children"); + } + } + } else { + fail("Only 2 queues with q1 and p1 "); + } + } + } + + public void testRefresh() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + createSimpleDocument(doc); + writeToFile(doc, CONFIG); + QueueManager qm = new QueueManager(CONFIG); + Queue beforeRefreshRoot = qm.getRoot(); + //remove the file and create new one. + Set rootQueues = beforeRefreshRoot.getChildren(); + for (Queue qs : rootQueues) { + if (qs.getName().equals("q1")) { + + assertEquals(qs.getProperties().getProperty("capacity"), "10"); + assertEquals(qs.getProperties().getProperty("maxCapacity"), "35"); + + } else if (qs.getName().equals("p1")) { + + Set children = qs.getChildren(); + for (Queue child : children) { + if (child.getName().equals( + "p1" + NAME_SEPARATOR + "p12")) { + assertTrue( + child.getAcls().get( + QueueManager.toFullPropertyName( + child.getName(), ACL_SUBMIT_JOB_TAG)) + .getUsers().contains("u1")); + + assertTrue( + child.getAcls().get( + QueueManager.toFullPropertyName( + child.getName(), + ACL_ADMINISTER_JOB_TAG)) + .getUsers().contains("u2")); + assertTrue(child.getState().equals(Queue.QueueState.STOPPED)); + } else { + assertTrue(child.getState().equals(Queue.QueueState.RUNNING)); + } + } + } + } + checkForConfigFile(); + doc = createDocument(); + refreshSimpleDocument(doc); + writeToFile(doc, CONFIG); + QueueConfigurationParser cp = new QueueConfigurationParser(CONFIG); + qm.getRoot().isHierarchySameAs(cp.getRoot()); + qm.setQueues( + cp.getRoot().getChildren().toArray( + new Queue[cp.getRoot().getChildren().size()])); + Queue afterRefreshRoot = qm.getRoot(); + //remove the file and create new one. + rootQueues = afterRefreshRoot.getChildren(); + for (Queue qs : rootQueues) { + if (qs.getName().equals("q1")) { + + assertEquals(qs.getProperties().getProperty("capacity"), "70"); + assertEquals(qs.getProperties().getProperty("maxCapacity"), "35"); + + } else if (qs.getName().equals("p1")) { + + Set children = qs.getChildren(); + for (Queue child : children) { + if (child.getName().equals( + "p1" + NAME_SEPARATOR + "p12")) { + assertTrue( + child.getAcls().get( + QueueManager.toFullPropertyName( + child.getName(), + ACL_SUBMIT_JOB_TAG)) + .getUsers().contains("u3")); + + assertTrue( + child.getAcls().get( + QueueManager.toFullPropertyName( + child.getName(), + ACL_ADMINISTER_JOB_TAG)) + .getUsers().contains("u4")); + assertTrue(child.getState().equals(Queue.QueueState.RUNNING)); + } else { + assertTrue(child.getState().equals(Queue.QueueState.STOPPED)); + } + } + } + } + } + + public void testRefreshFailureForHierarchyChange() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + createSimpleDocument(doc); + writeToFile(doc, CONFIG); + QueueManager qm = new QueueManager(CONFIG); + + checkForConfigFile(); + doc = createDocument(); + addMoreChildToSimpleDocumentStructure(doc); + writeToFile(doc, CONFIG); + QueueConfigurationParser cp = new QueueConfigurationParser(CONFIG); + assertFalse(qm.getRoot().isHierarchySameAs(cp.getRoot())); + } + + public void testRefreshWithInvalidFile() throws Exception { + checkForConfigFile(); + Document doc = createDocument(); + createSimpleDocument(doc); + writeToFile(doc, CONFIG); + QueueManager qm = new QueueManager(CONFIG); + + checkForConfigFile(); + doc = createDocument(); + Element queues = createQueuesNode(doc, "false"); + Element q1 = createQueue(doc, ""); + queues.appendChild(q1); + writeToFile(doc, CONFIG); + try { + QueueConfigurationParser cp = new QueueConfigurationParser(CONFIG); + + fail("Should throw an exception as configuration is wrong "); + } catch (Throwable re) { + re.printStackTrace(); + LOG.info(re.getMessage()); + } + } +} Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java?rev=815605&r1=815604&r2=815605&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java Wed Sep 16 04:49:18 2009 @@ -167,6 +167,8 @@ fs.delete(TEST_DIR, true); JobConf conf = new JobConf(); + conf.set( + DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY, "default"); conf.set("mapred.jobtracker.job.history.block.size", "1024"); conf.set("mapred.jobtracker.job.history.buffer.size", "1024");