Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 105D52600 for ; Thu, 28 Apr 2011 01:51:19 +0000 (UTC) Received: (qmail 38870 invoked by uid 500); 28 Apr 2011 01:51:19 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 38783 invoked by uid 500); 28 Apr 2011 01:51:18 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 38775 invoked by uid 99); 28 Apr 2011 01:51:18 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Apr 2011 01:51:18 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Apr 2011 01:51:14 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 4D99923889B1; Thu, 28 Apr 2011 01:50:53 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1097306 - in /hadoop/mapreduce/branches/branch-0.22: ./ src/test/mapred/org/apache/hadoop/mapred/ Date: Thu, 28 Apr 2011 01:50:53 -0000 To: mapreduce-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110428015053.4D99923889B1@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: todd Date: Thu Apr 28 01:50:52 2011 New Revision: 1097306 URL: http://svn.apache.org/viewvc?rev=1097306&view=rev Log: MAPREDUCE-2445. Fix TestMiniMRWithDFSWithDistinctUsers to be a valid test. Contributed by Todd Lipcon. Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1097306&r1=1097305&r2=1097306&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original) +++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Thu Apr 28 01:50:52 2011 @@ -525,6 +525,9 @@ Release 0.22.0 - Unreleased MAPREDUCE-2428. start-mapred.sh script fails if HADOOP_HOME is not set. (tomwhite via eli) + MAPREDUCE-2445. Fix TestMiniMRWithDFSWithDistinctUsers to be a valid test. + (todd) + Release 0.21.1 - Unreleased NEW FEATURES Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java?rev=1097306&r1=1097305&r2=1097306&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java (original) +++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java Thu Apr 28 01:50:52 2011 @@ -24,16 +24,11 @@ import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.mapred.lib.IdentityMapper; -import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.mapreduce.JobSubmissionFiles; -import org.apache.hadoop.mapreduce.protocol.ClientProtocol; -import org.apache.hadoop.mapreduce.split.JobSplitWriter; -import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.security.*; /** @@ -41,8 +36,14 @@ import org.apache.hadoop.security.*; */ public class TestMiniMRWithDFSWithDistinctUsers extends TestCase { static final UserGroupInformation DFS_UGI = createUGI("dfs", true); - static final UserGroupInformation PI_UGI = createUGI("pi", false); - static final UserGroupInformation WC_UGI = createUGI("wc", false); + static final UserGroupInformation ALICE_UGI = createUGI("alice", false); + static final UserGroupInformation BOB_UGI = createUGI("bob", false); + + MiniMRCluster mr = null; + MiniDFSCluster dfs = null; + FileSystem fs = null; + Configuration conf = new Configuration(); + String jobTrackerName; static UserGroupInformation createUGI(String name, boolean issuper) { String group = issuper? "supergroup": name; @@ -50,106 +51,73 @@ public class TestMiniMRWithDFSWithDistin return UserGroupInformation.createUserForTesting(name, new String[]{group}); } - static void mkdir(FileSystem fs, String dir) throws IOException { + static void mkdir(FileSystem fs, String dir, + String user, String group, short mode) throws IOException { Path p = new Path(dir); fs.mkdirs(p); - fs.setPermission(p, new FsPermission((short)0777)); + fs.setPermission(p, new FsPermission(mode)); + fs.setOwner(p, user, group); } // runs a sample job as a user (ugi) - RunningJob runJobAsUser(final JobConf job, UserGroupInformation ugi) + void runJobAsUser(final JobConf job, UserGroupInformation ugi) throws Exception { - ClientProtocol jobSubmitClient = - TestSubmitJob.getJobSubmitClient(job, ugi); - org.apache.hadoop.mapreduce.JobID id = jobSubmitClient.getNewJobID(); - - InputSplit[] splits = computeJobSplit(JobID.downgrade(id), job); - final Path jobSubmitDir = new Path(id.toString()); - FileSystem fs = ugi.doAs(new PrivilegedExceptionAction() { - public FileSystem run() throws IOException { - return jobSubmitDir.getFileSystem(job); - } - }); - Path qJobSubmitDir = jobSubmitDir.makeQualified(fs); - uploadJobFiles(JobID.downgrade(id), splits, qJobSubmitDir, ugi, job); - - jobSubmitClient.submitJob(id, qJobSubmitDir.toString(), null); - - JobClient jc = new JobClient(job); - return jc.getJob(JobID.downgrade(id)); - } - - // a helper api for split computation - private InputSplit[] computeJobSplit(JobID id, JobConf conf) - throws IOException { - InputSplit[] splits = - conf.getInputFormat().getSplits(conf, conf.getNumMapTasks()); - conf.setNumMapTasks(splits.length); - return splits; + RunningJob rj = ugi.doAs(new PrivilegedExceptionAction() { + public RunningJob run() throws IOException { + return JobClient.runJob(job); + } + }); + + rj.waitForCompletion(); + assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState())); } + public void setUp() throws Exception { + dfs = new MiniDFSCluster(conf, 4, true, null); - // a helper api for split submission - private void uploadJobFiles(JobID id, InputSplit[] splits, - Path jobSubmitDir, UserGroupInformation ugi, - final JobConf conf) - throws Exception { - final Path confLocation = JobSubmissionFiles.getJobConfPath(jobSubmitDir); - FileSystem fs = ugi.doAs(new PrivilegedExceptionAction() { - public FileSystem run() throws IOException { - return confLocation.getFileSystem(conf); - } - }); - JobSplitWriter.createSplitFiles(jobSubmitDir, conf, fs, splits); - FsPermission perm = new FsPermission((short)0700); - - // localize conf - DataOutputStream confOut = FileSystem.create(fs, confLocation, perm); - conf.writeXml(confOut); - confOut.close(); - } - - public void testDistinctUsers() throws Exception { - MiniMRCluster mr = null; - Configuration conf = new Configuration(); - final MiniDFSCluster dfs = new MiniDFSCluster(conf, 4, true, null); - try { - - FileSystem fs = DFS_UGI.doAs(new PrivilegedExceptionAction() { + fs = DFS_UGI.doAs(new PrivilegedExceptionAction() { public FileSystem run() throws IOException { return dfs.getFileSystem(); } }); - mkdir(fs, "/user"); - mkdir(fs, "/mapred"); - - UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); - mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(), - 1, null, null, MR_UGI); - String jobTrackerName = "localhost:" + mr.getJobTrackerPort(); - - JobConf job1 = mr.createJobConf(); - String input = "The quick brown fox\nhas many silly\n" - + "red fox sox\n"; - Path inDir = new Path("/testing/distinct/input"); - Path outDir = new Path("/testing/distinct/output"); - TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, - input, 2, 1, inDir, outDir); - runJobAsUser(job1, PI_UGI); - - JobConf job2 = mr.createJobConf(); - Path inDir2 = new Path("/testing/distinct/input2"); - Path outDir2 = new Path("/testing/distinct/output2"); - TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2, - input, 2, 1, inDir2, outDir2); - runJobAsUser(job2, WC_UGI); - } finally { - if (dfs != null) { dfs.shutdown(); } - if (mr != null) { mr.shutdown();} - } + // Home directories for users + mkdir(fs, "/user", "nobody", "nogroup", (short)01777); + mkdir(fs, "/user/alice", "alice", "nogroup", (short)0755); + mkdir(fs, "/user/bob", "bob", "nogroup", (short)0755); + + // staging directory root with sticky bit + UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); + mkdir(fs, "/staging", MR_UGI.getShortUserName(), "nogroup", (short)01777); + + JobConf mrConf = new JobConf(); + mrConf.set(JTConfig.JT_STAGING_AREA_ROOT, "/staging"); + + mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(), + 1, null, null, MR_UGI, mrConf); + jobTrackerName = "localhost:" + mr.getJobTrackerPort(); + } + + public void tearDown() throws Exception { + if (mr != null) { mr.shutdown();} + if (dfs != null) { dfs.shutdown(); } } - public void testRestartWithDistinctUsers() { - + public void testDistinctUsers() throws Exception { + JobConf job1 = mr.createJobConf(); + String input = "The quick brown fox\nhas many silly\n" + + "red fox sox\n"; + Path inDir = new Path("/testing/distinct/input"); + Path outDir = new Path("/user/alice/output"); + TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, + input, 2, 1, inDir, outDir); + runJobAsUser(job1, ALICE_UGI); + + JobConf job2 = mr.createJobConf(); + Path inDir2 = new Path("/testing/distinct/input2"); + Path outDir2 = new Path("/user/bob/output2"); + TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2, + input, 2, 1, inDir2, outDir2); + runJobAsUser(job2, BOB_UGI); } + } Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java?rev=1097306&r1=1097305&r2=1097306&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java (original) +++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java Thu Apr 28 01:50:52 2011 @@ -93,10 +93,10 @@ public class TestSeveral extends TestCas } }); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user"); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred"); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, - conf.get(JTConfig.JT_STAGING_AREA_ROOT)); + TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user", "mapred", "mapred", (short)01777); + TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred", "mapred", "mapred", (short)01777); + TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, conf.get(JTConfig.JT_STAGING_AREA_ROOT), + "mapred", "mapred", (short)01777); UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java?rev=1097306&r1=1097305&r2=1097306&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java (original) +++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java Thu Apr 28 01:50:52 2011 @@ -206,10 +206,11 @@ public class TestSubmitJob extends TestC return dfs.getFileSystem(); } }); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user"); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred"); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, - conf.get(JTConfig.JT_STAGING_AREA_ROOT)); + TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user", "mapred", "mapred", (short)01777); + TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred", "mapred", "mapred", (short)01777); + TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, conf.get(JTConfig.JT_STAGING_AREA_ROOT), + "mapred", "mapred", (short)01777); + UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); mr = new MiniMRCluster(0, 0, 1, dfs.getFileSystem().getUri().toString(), 1, null, null, MR_UGI);