hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r984680 - in /hadoop/mapreduce/trunk/src: java/org/apache/hadoop/mapred/ java/org/apache/hadoop/mapreduce/ java/org/apache/hadoop/mapreduce/jobhistory/ test/mapred/org/apache/hadoop/mapred/ test/mapred/org/apache/hadoop/mapreduce/
Date Thu, 12 Aug 2010 08:22:48 GMT
Author: vinodkv
Date: Thu Aug 12 08:22:48 2010
New Revision: 984680

URL: http://svn.apache.org/viewvc?rev=984680&view=rev
Log:
MAPREDUCE-1780. Fixes serialization of ACL in JobStatus.java to use AccessControlList.write()
instead of AccessControlList.toString(). Contributed by Ravi Gummadi.

Modified:
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java Thu Aug 12
08:22:48 2010
@@ -690,18 +690,17 @@ class QueueManager {
           queue.getAcls().get(QueueManager.toFullPropertyName(queue.getName(),
               Queue.QueueOperation.ADMINISTER_JOBS.getAclName()));
       }
-      StringBuilder aclsSubmitJobValue = new StringBuilder();
+      String aclsSubmitJobValue = " ";
       if (submitJobList != null ) {
-        aclsSubmitJobValue = getAclsInfo(submitJobList);
+        aclsSubmitJobValue = submitJobList.getAclString();
       }
-      dumpGenerator.writeStringField("acl_submit_job",
-          aclsSubmitJobValue.toString());
-      StringBuilder aclsAdministerValue = new StringBuilder();
+      dumpGenerator.writeStringField("acl_submit_job", aclsSubmitJobValue);
+      String aclsAdministerValue = " ";
       if (administerJobsList != null) {
-        aclsAdministerValue = getAclsInfo(administerJobsList);
+        aclsAdministerValue = administerJobsList.getAclString();
       }
       dumpGenerator.writeStringField("acl_administer_jobs",
-          aclsAdministerValue.toString());
+          aclsAdministerValue);
       dumpGenerator.writeFieldName("properties");
       dumpGenerator.writeStartArray();
       if (queue.getProperties() != null) {
@@ -725,7 +724,4 @@ class QueueManager {
     }
   }
 
-  private static StringBuilder getAclsInfo(AccessControlList accessControlList){
-    return new StringBuilder(accessControlList.toString());
-  }
 }

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java Thu Aug 12
08:22:48 2010
@@ -350,7 +350,7 @@ public class JobStatus implements Writab
     out.writeInt(jobACLs.size());
     for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
       WritableUtils.writeEnum(out, entry.getKey());
-      Text.writeString(out, entry.getValue().toString());
+      entry.getValue().write(out);
     }
   }
 
@@ -377,8 +377,9 @@ public class JobStatus implements Writab
     int numACLs = in.readInt();
     for (int i = 0; i < numACLs; i++) {
       JobACL aclType = WritableUtils.readEnum(in, JobACL.class);
-      String acl = Text.readString(in);
-      this.jobACLs.put(aclType, new AccessControlList(acl));
+      AccessControlList acl = new AccessControlList(" ");
+      acl.readFields(in);
+      this.jobACLs.put(aclType, acl);
     }
   }
 

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
(original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
Thu Aug 12 08:22:48 2010
@@ -71,7 +71,7 @@ public class JobSubmittedEvent implement
     Map<Utf8, Utf8> jobAcls = new HashMap<Utf8, Utf8>();
     for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
       jobAcls.put(new Utf8(entry.getKey().getAclName()), new Utf8(
-          entry.getValue().toString()));
+          entry.getValue().getAclString()));
     }
     datum.acls = jobAcls;
   }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java Thu
Aug 12 08:22:48 2010
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.Clust
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.JobACL;
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskID;
 import org.apache.hadoop.mapreduce.TaskType;
@@ -585,6 +586,21 @@ public class TestJobHistory extends Test
     validateJobLevelKeyValues(mr, job, jobInfo, conf);
     validateTaskLevelKeyValues(mr, job, jobInfo);
     validateTaskAttemptLevelKeyValues(mr, job, jobInfo);
+    
+    // Also JobACLs should be correct
+    if (mr.getJobTrackerRunner().getJobTracker()
+        .isJobLevelAuthorizationEnabled()) {
+      AccessControlList acl = new AccessControlList(
+          conf.get(JobACL.VIEW_JOB.getAclName(), " "));
+      assertTrue("VIEW_JOB ACL is not properly logged to history file.",
+          acl.toString().equals(
+          jobInfo.getJobACLs().get(JobACL.VIEW_JOB).toString()));
+      acl = new AccessControlList(
+          conf.get(JobACL.MODIFY_JOB.getAclName(), " "));
+      assertTrue("MODIFY_JOB ACL is not properly logged to history file.",
+          acl.toString().equals(
+          jobInfo.getJobACLs().get(JobACL.MODIFY_JOB).toString()));
+    }
   }
 
   public void testDoneFolderOnHDFS() throws IOException, InterruptedException {
@@ -721,7 +737,10 @@ public class TestJobHistory extends Test
       //set the done folder location
       String doneFolder = TEST_ROOT_DIR + "history_done";
       conf.set(JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION, doneFolder);
-      
+
+      // Enable ACLs so that they are logged to history
+      conf.setBoolean(MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG, true);
+
       mr = new MiniMRCluster(2, "file:///", 3, null, null, conf);
 
       // run the TCs
@@ -737,6 +756,10 @@ public class TestJobHistory extends Test
       //Disable speculative execution
       conf.setSpeculativeExecution(false);
 
+      // set the job acls
+      conf.set(JobACL.VIEW_JOB.getAclName(), "user1,user2 group1,group2");
+      conf.set(JobACL.MODIFY_JOB.getAclName(), "user3,user4 group3,group4");
+
       // Make sure that the job is not removed from memory until we do finish
       // the validation of history file content
       conf.setInt("mapred.jobtracker.completeuserjobs.maximum", 10);

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java
Thu Aug 12 08:22:48 2010
@@ -26,24 +26,18 @@ import static org.junit.Assert.*;
 
 import org.apache.hadoop.mapreduce.QueueState;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AccessControlList;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.junit.After;
 import org.junit.Test;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-
 import java.io.File;
 import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
-import java.util.Map.Entry;
 
 
 public class TestQueueManager {
@@ -596,7 +590,7 @@ public class TestQueueManager {
     assertEquals("35", q1_properties.get("maxCapacity").getValue());
     
     // check for acls
-    assertEquals("u1", childQueues.get("p1:p12").getAcl_submit_job());
-    assertEquals("u2", childQueues.get("p1:p12").getAcl_administer_jobs());
+    assertEquals("u1 ", childQueues.get("p1:p12").getAcl_submit_job());
+    assertEquals("u2 ", childQueues.get("p1:p12").getAcl_administer_jobs());
   }
 }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java Thu
Aug 12 08:22:48 2010
@@ -383,7 +383,7 @@ public class TestJobACLs {
 
     // Set the job up.
     final Configuration myConf = mr.createJobConf();
-    myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user1,user2");
+    myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user2 group2");
 
     // Submit the job as user1
     Job job = submitJobAsUser(myConf, "user1");



Mime
View raw message