hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1097305 - in /hadoop/mapreduce/trunk: ./ src/test/mapred/org/apache/hadoop/mapred/
Date Thu, 28 Apr 2011 01:50:46 GMT
Author: todd
Date: Thu Apr 28 01:50:45 2011
New Revision: 1097305

URL: http://svn.apache.org/viewvc?rev=1097305&view=rev
Log:
MAPREDUCE-2445. Fix TestMiniMRWithDFSWithDistinctUsers to be a valid test. Contributed by
Todd Lipcon.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1097305&r1=1097304&r2=1097305&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Apr 28 01:50:45 2011
@@ -663,6 +663,9 @@ Release 0.22.0 - Unreleased
     MAPREDUCE-2428. start-mapred.sh script fails if HADOOP_HOME is not set.
     (tomwhite via eli)
 
+    MAPREDUCE-2445. Fix TestMiniMRWithDFSWithDistinctUsers to be a valid test.
+    (todd)
+
 Release 0.21.1 - Unreleased
 
   NEW FEATURES

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java?rev=1097305&r1=1097304&r2=1097305&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
Thu Apr 28 01:50:45 2011
@@ -24,16 +24,11 @@ import junit.framework.TestCase;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.mapred.lib.IdentityMapper;
-import org.apache.hadoop.mapred.lib.IdentityReducer;
-import org.apache.hadoop.mapreduce.JobSubmissionFiles;
-import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
-import org.apache.hadoop.mapreduce.split.JobSplitWriter;
-import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.security.*;
 
 /**
@@ -41,8 +36,14 @@ import org.apache.hadoop.security.*;
  */
 public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
   static final UserGroupInformation DFS_UGI = createUGI("dfs", true); 
-  static final UserGroupInformation PI_UGI = createUGI("pi", false); 
-  static final UserGroupInformation WC_UGI = createUGI("wc", false); 
+  static final UserGroupInformation ALICE_UGI = createUGI("alice", false); 
+  static final UserGroupInformation BOB_UGI = createUGI("bob", false); 
+
+  MiniMRCluster mr = null;
+  MiniDFSCluster dfs = null;
+  FileSystem fs = null;
+  Configuration conf = new Configuration();
+  String jobTrackerName;
 
   static UserGroupInformation createUGI(String name, boolean issuper) {
     String group = issuper? "supergroup": name;
@@ -50,106 +51,73 @@ public class TestMiniMRWithDFSWithDistin
     return UserGroupInformation.createUserForTesting(name, new String[]{group});
   }
   
-  static void mkdir(FileSystem fs, String dir) throws IOException {
+  static void mkdir(FileSystem fs, String dir,
+                    String user, String group, short mode) throws IOException {
     Path p = new Path(dir);
     fs.mkdirs(p);
-    fs.setPermission(p, new FsPermission((short)0777));
+    fs.setPermission(p, new FsPermission(mode));
+    fs.setOwner(p, user, group);
   }
 
   // runs a sample job as a user (ugi)
-  RunningJob runJobAsUser(final JobConf job, UserGroupInformation ugi) 
+  void runJobAsUser(final JobConf job, UserGroupInformation ugi) 
   throws Exception {
-    ClientProtocol jobSubmitClient = 
-      TestSubmitJob.getJobSubmitClient(job, ugi);
-    org.apache.hadoop.mapreduce.JobID id = jobSubmitClient.getNewJobID();
-    
-    InputSplit[] splits = computeJobSplit(JobID.downgrade(id), job);
-    final Path jobSubmitDir = new Path(id.toString());
-    FileSystem fs = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
-      public FileSystem run() throws IOException {
-        return jobSubmitDir.getFileSystem(job);
-      }
-    });
-    Path qJobSubmitDir = jobSubmitDir.makeQualified(fs);
-    uploadJobFiles(JobID.downgrade(id), splits, qJobSubmitDir, ugi, job);
-    
-    jobSubmitClient.submitJob(id, qJobSubmitDir.toString(), null);
-    
-    JobClient jc = new JobClient(job);
-    return jc.getJob(JobID.downgrade(id));
-  }
-  
-  // a helper api for split computation
-  private InputSplit[] computeJobSplit(JobID id, JobConf conf) 
-  throws IOException {
-    InputSplit[] splits = 
-      conf.getInputFormat().getSplits(conf, conf.getNumMapTasks());
-    conf.setNumMapTasks(splits.length);
-    return splits;
+    RunningJob rj = ugi.doAs(new PrivilegedExceptionAction<RunningJob>() {
+        public RunningJob run() throws IOException {
+          return JobClient.runJob(job);
+        }
+      });
+
+    rj.waitForCompletion();
+    assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
   }
 
+  public void setUp() throws Exception {
+    dfs = new MiniDFSCluster(conf, 4, true, null);
 
-  // a helper api for split submission
-  private void uploadJobFiles(JobID id, InputSplit[] splits,
-                             Path jobSubmitDir, UserGroupInformation ugi, 
-                             final JobConf conf) 
-  throws Exception {
-    final Path confLocation = JobSubmissionFiles.getJobConfPath(jobSubmitDir);
-    FileSystem fs = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
-      public FileSystem run() throws IOException {
-        return confLocation.getFileSystem(conf);
-      }
-    });
-    JobSplitWriter.createSplitFiles(jobSubmitDir, conf, fs, splits);
-    FsPermission perm = new FsPermission((short)0700);
-    
-    // localize conf
-    DataOutputStream confOut = FileSystem.create(fs, confLocation, perm);
-    conf.writeXml(confOut);
-    confOut.close();
-  }
-  
-  public void testDistinctUsers() throws Exception {
-    MiniMRCluster mr = null;
-    Configuration conf = new Configuration();
-    final MiniDFSCluster dfs = new MiniDFSCluster(conf, 4, true, null);
-    try {
-          
-      FileSystem fs = DFS_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
+    fs = DFS_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
         public FileSystem run() throws IOException {
           return dfs.getFileSystem();
         }
       });
-      mkdir(fs, "/user");
-      mkdir(fs, "/mapred");
-
-      UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); 
-      mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
-           1, null, null, MR_UGI);
-      String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
-
-      JobConf job1 = mr.createJobConf();
-      String input = "The quick brown fox\nhas many silly\n" 
-                     + "red fox sox\n";
-      Path inDir = new Path("/testing/distinct/input");
-      Path outDir = new Path("/testing/distinct/output");
-      TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, 
-                                             input, 2, 1, inDir, outDir);
-      runJobAsUser(job1, PI_UGI);
-
-      JobConf job2 = mr.createJobConf();
-      Path inDir2 = new Path("/testing/distinct/input2");
-      Path outDir2 = new Path("/testing/distinct/output2");
-      TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2, 
-                                             input, 2, 1, inDir2, outDir2);
-      runJobAsUser(job2, WC_UGI);
-    } finally {
-      if (dfs != null) { dfs.shutdown(); }
-      if (mr != null) { mr.shutdown();}
-    }
+    // Home directories for users
+    mkdir(fs, "/user", "nobody", "nogroup", (short)01777);
+    mkdir(fs, "/user/alice", "alice", "nogroup", (short)0755);
+    mkdir(fs, "/user/bob", "bob", "nogroup", (short)0755);
+
+    // staging directory root with sticky bit
+    UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); 
+    mkdir(fs, "/staging", MR_UGI.getShortUserName(), "nogroup", (short)01777);
+
+    JobConf mrConf = new JobConf();
+    mrConf.set(JTConfig.JT_STAGING_AREA_ROOT, "/staging");
+
+    mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
+                           1, null, null, MR_UGI, mrConf);
+    jobTrackerName = "localhost:" + mr.getJobTrackerPort();
+  }
+
+  public void tearDown() throws Exception {
+    if (mr != null) { mr.shutdown();}
+    if (dfs != null) { dfs.shutdown(); }
   }
   
-  public void testRestartWithDistinctUsers() {
-    
+  public void testDistinctUsers() throws Exception {
+    JobConf job1 = mr.createJobConf();
+    String input = "The quick brown fox\nhas many silly\n" 
+      + "red fox sox\n";
+    Path inDir = new Path("/testing/distinct/input");
+    Path outDir = new Path("/user/alice/output");
+    TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, 
+                                           input, 2, 1, inDir, outDir);
+    runJobAsUser(job1, ALICE_UGI);
+
+    JobConf job2 = mr.createJobConf();
+    Path inDir2 = new Path("/testing/distinct/input2");
+    Path outDir2 = new Path("/user/bob/output2");
+    TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2, 
+                                           input, 2, 1, inDir2, outDir2);
+    runJobAsUser(job2, BOB_UGI);
   }
+
 }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java?rev=1097305&r1=1097304&r2=1097305&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java Thu Apr
28 01:50:45 2011
@@ -93,10 +93,10 @@ public class TestSeveral extends TestCas
           }
         });
 
-        TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user");
-        TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred");
-        TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, 
-            conf.get(JTConfig.JT_STAGING_AREA_ROOT));
+        TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user", "mapred", "mapred", (short)01777);
+        TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred", "mapred", "mapred", (short)01777);
+        TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, conf.get(JTConfig.JT_STAGING_AREA_ROOT),
+                                                 "mapred", "mapred", (short)01777);
 
         UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); 
 

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java?rev=1097305&r1=1097304&r2=1097305&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java Thu
Apr 28 01:50:45 2011
@@ -206,10 +206,11 @@ public class TestSubmitJob extends TestC
             return dfs.getFileSystem();
           }
         });
-      TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user");
-      TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred");
-      TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, 
-          conf.get(JTConfig.JT_STAGING_AREA_ROOT));
+      TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user", "mapred", "mapred", (short)01777);
+      TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred", "mapred", "mapred", (short)01777);
+      TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, conf.get(JTConfig.JT_STAGING_AREA_ROOT),
+                                               "mapred", "mapred", (short)01777);
+
       UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser();
       mr = new MiniMRCluster(0, 0, 1, dfs.getFileSystem().getUri().toString(),
                              1, null, null, MR_UGI);



Mime
View raw message