hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yhema...@apache.org
Subject svn commit: r816471 - in /hadoop/mapreduce/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobTracker.java src/java/org/apache/hadoop/mapred/QueueManager.java src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java
Date Fri, 18 Sep 2009 05:55:57 GMT
Author: yhemanth
Date: Fri Sep 18 05:55:56 2009
New Revision: 816471

URL: http://svn.apache.org/viewvc?rev=816471&view=rev
Log:
MAPREDUCE-953. Fix QueueManager to dump queue configuration in JSON format. Contributed by
V.V. Chaitanya Krishna.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=816471&r1=816470&r2=816471&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Sep 18 05:55:56 2009
@@ -632,3 +632,7 @@
 
     MAPREDUCE-995. Fix a bug in JobHistory where tasks completing after the job
     is closed cause a NPE. (Jothi Padmanabhan via cdouglas)
+
+    MAPREDUCE-953. Fix QueueManager to dump queue configuration in JSON format.
+    (V.V. Chaitanya Krishna via yhemanth)
+

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=816471&r1=816470&r2=816471&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Sep 18 05:55:56
2009
@@ -3574,6 +3574,9 @@
       else {
         if ("-dumpConfiguration".equals(argv[0]) && argv.length == 1) {
           dumpConfiguration(new PrintWriter(System.out));
+          System.out.println();
+          QueueManager.dumpConfiguration(new PrintWriter(System.out),
+              new JobConf());
         }
         else {
           System.out.println("usage: JobTracker [-dumpConfiguration]");

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java?rev=816471&r1=816470&r2=816471&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java Fri Sep 18
05:55:56 2009
@@ -21,13 +21,19 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.Queue.QueueState;
 import org.apache.hadoop.security.SecurityUtil.AccessControlList;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonGenerator;
 
 import java.io.IOException;
+import java.io.Writer;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.List;
@@ -535,4 +541,133 @@
   Queue getRoot() {
     return root;
   }
+  
+  /**
+   * Dumps the configuration of hierarchy of queues
+   * @param out the writer object to which dump is written
+   * @throws IOException
+   */
+  static void dumpConfiguration(Writer out,Configuration conf) throws IOException {
+    dumpConfiguration(out, null,conf);
+  }
+  
+  /***
+   * Dumps the configuration of hierarchy of queues with 
+   * the xml file path given. It is to be used directly ONLY FOR TESTING.
+   * @param out the writer object to which dump is written to.
+   * @param configFile the filename of xml file
+   * @throws IOException
+   */
+  static void dumpConfiguration(Writer out, String configFile,
+      Configuration conf) throws IOException {
+    if (conf != null && conf.get(DeprecatedQueueConfigurationParser.
+        MAPRED_QUEUE_NAMES_KEY) != null) {
+      return;
+    }
+    JsonFactory dumpFactory = new JsonFactory();
+    JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
+    QueueConfigurationParser parser;
+    if (configFile != null && !"".equals(configFile)) {
+      parser = new QueueConfigurationParser(configFile);
+    }
+    else {
+      parser = QueueManager.getQueueConfigurationParser(null, false);
+    }
+    dumpGenerator.writeStartObject();
+    dumpGenerator.writeBooleanField("acls_enabled", parser.isAclsEnabled());
+    dumpGenerator.writeFieldName("queues");
+    dumpGenerator.writeStartArray();
+    dumpConfiguration(dumpGenerator,parser.getRoot().getChildren());
+    dumpGenerator.writeEndArray();
+    dumpGenerator.writeEndObject();
+    dumpGenerator.flush();
+  }
+
+  /**
+   * method to perform depth-first search and write the parameters of every 
+   * queue in JSON format.
+   * @param dumpGenerator JsonGenerator object which takes the dump and flushes
+   *  to a writer object
+   * @param rootQueues the top-level queues
+   * @throws JsonGenerationException
+   * @throws IOException
+   */
+  private static void dumpConfiguration(JsonGenerator dumpGenerator,
+      Set<Queue> rootQueues) throws JsonGenerationException, IOException {
+    for (Queue queue : rootQueues) {
+      dumpGenerator.writeStartObject();
+      dumpGenerator.writeStringField("name", queue.getName());
+      dumpGenerator.writeStringField("state", queue.getState().toString());
+      AccessControlList submitJobList = null;
+      AccessControlList administerJobsList = null;
+      if (queue.getAcls() != null) {
+        submitJobList =
+          queue.getAcls().get(QueueManager.toFullPropertyName(queue.getName(),
+              Queue.QueueOperation.SUBMIT_JOB.getAclName()));
+        administerJobsList =
+          queue.getAcls().get(QueueManager.toFullPropertyName(queue.getName(),
+              Queue.QueueOperation.ADMINISTER_JOBS.getAclName()));
+      }
+      StringBuilder aclsSubmitJobValue = new StringBuilder();
+      if (submitJobList != null ) {
+        aclsSubmitJobValue = getAclsInfo(submitJobList);
+      }
+      dumpGenerator.writeStringField("acl_submit_job",
+          aclsSubmitJobValue.toString());
+      StringBuilder aclsAdministerValue = new StringBuilder();
+      if (administerJobsList != null) {
+        aclsAdministerValue = getAclsInfo(administerJobsList);
+      }
+      dumpGenerator.writeStringField("acl_administer_jobs",
+          aclsAdministerValue.toString());
+      dumpGenerator.writeFieldName("properties");
+      dumpGenerator.writeStartArray();
+      if (queue.getProperties() != null) {
+        for (Map.Entry<Object, Object>property :
+          queue.getProperties().entrySet()) {
+          dumpGenerator.writeStartObject();
+          dumpGenerator.writeStringField("key", (String)property.getKey());
+          dumpGenerator.writeStringField("value", (String)property.getValue());
+          dumpGenerator.writeEndObject();
+        }
+      }
+      dumpGenerator.writeEndArray();
+      Set<Queue> childQueues = queue.getChildren();
+      dumpGenerator.writeFieldName("children");
+      dumpGenerator.writeStartArray();
+      if (childQueues != null && childQueues.size() > 0) {
+        dumpConfiguration(dumpGenerator, childQueues);
+      }
+      dumpGenerator.writeEndArray();
+      dumpGenerator.writeEndObject();
+    }
+  }
+
+  private static StringBuilder getAclsInfo(AccessControlList accessControlList) {
+    StringBuilder sb = new StringBuilder();
+    if (accessControlList.getUsers() != null &&
+        accessControlList.getUsers().size() > 0) {
+      Set<String> users = accessControlList.getUsers();
+      Iterator<String> iterator = users.iterator();
+      while (iterator.hasNext()) {
+        sb.append(iterator.next());
+        if (iterator.hasNext()) {
+          sb.append(",");
+        }
+      }
+    }
+    if (accessControlList.getGroups() != null &&
+        accessControlList.getGroups().size() > 0) {
+      sb.append(" ");
+      Set<String> groups = accessControlList.getGroups();
+      Iterator<String> iterator = groups.iterator();
+      while (iterator.hasNext()) {
+        sb.append(iterator.next());
+        if (iterator.hasNext()) {
+          sb.append(",");
+        }
+      }
+    }
+    return sb;
+  }
 }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java?rev=816471&r1=816470&r2=816471&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerForHierarchialQueues.java
Fri Sep 18 05:55:56 2009
@@ -23,16 +23,25 @@
 import org.apache.commons.logging.LogFactory;
 import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
 import static org.apache.hadoop.mapred.QueueConfigurationParser.*;
+
 import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.SecurityUtil.AccessControlList;
+import org.codehaus.jackson.map.ObjectMapper;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
+import org.w3c.dom.Node;
 
 import java.io.File;
+import java.io.StringWriter;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.Map.Entry;
 
 
 public class TestQueueManagerForHierarchialQueues extends TestCase {
@@ -376,4 +385,150 @@
       LOG.info(re.getMessage());
     }
   }
+
+  /**
+   * Class to store the array of queues retrieved by parsing the string 
+   * that is dumped in Json format
+   */
+  static class JsonQueueTree {
+    boolean acls_enabled;
+    
+    JsonQueue[] queues;
+
+    public JsonQueue[] getQueues() {
+      return queues;
+    }
+
+    public void setQueues(JsonQueue[] queues) {
+      this.queues = queues;
+    }
+
+    public boolean isAcls_enabled() {
+      return acls_enabled;
+    }
+
+    public void setAcls_enabled(boolean aclsEnabled) {
+      acls_enabled = aclsEnabled;
+    }
+  }
+  
+  /**
+   * Class to store the contents of each queue that is dumped in JSON format.
+   */
+  static class JsonQueue {
+    String name;
+    String state;
+    String acl_submit_job;
+    String acl_administer_jobs;
+    JsonProperty[] properties;
+    JsonQueue[] children;
+    public String getName() {
+      return name;
+    }
+    public String getState() {
+      return state;
+    }
+    public JsonProperty[] getProperties() {
+      return properties;
+    }
+    public JsonQueue[] getChildren() {
+      return children;
+    }
+    public void setName(String name) {
+      this.name = name;
+    }
+    public void setState(String state) {
+      this.state = state;
+    }
+    public void setProperties(JsonProperty[] properties) {
+      this.properties = properties;
+    }
+    public void setChildren(JsonQueue[] children) {
+      this.children = children;
+    }
+    public String getAcl_submit_job() {
+      return acl_submit_job;
+    }
+    public void setAcl_submit_job(String aclSubmitJob) {
+      acl_submit_job = aclSubmitJob;
+    }
+    public String getAcl_administer_jobs() {
+      return acl_administer_jobs;
+    }
+    public void setAcl_administer_jobs(String aclAdministerJobs) {
+      acl_administer_jobs = aclAdministerJobs;
+    }
+  }
+  
+  /**
+   * Class to store the contents of attribute "properties" in Json dump
+   */
+  static class JsonProperty {
+    String key;
+    String value;
+    public String getKey() {
+      return key;
+    }
+    public void setKey(String key) {
+      this.key = key;
+    }
+    public String getValue() {
+      return value;
+    }
+    public void setValue(String value) {
+      this.value = value;
+    }
+  }
+
+  /**
+   * checks the format of the dump in JSON format when 
+   * QueueManager.dumpConfiguration(Writer) is called.
+   * @throws Exception
+   */
+  public void testDumpConfiguration() throws Exception {
+    checkForConfigFile();
+    Document doc = createDocument();
+    createSimpleDocument(doc);    
+    writeToFile(doc, CONFIG);
+    StringWriter out = new StringWriter();
+    QueueManager.dumpConfiguration(out,CONFIG,null);
+    ObjectMapper mapper = new ObjectMapper();
+    // parse the Json dump
+    JsonQueueTree queueTree =
+      mapper.readValue(out.toString(), JsonQueueTree.class);
+    
+    // check if acls_enabled is correct
+    assertEquals(true, queueTree.isAcls_enabled());
+    // check for the number of top-level queues
+    assertEquals(2, queueTree.getQueues().length);
+    
+    HashMap<String, JsonQueue> topQueues = new HashMap<String, JsonQueue>();
+    for (JsonQueue topQueue : queueTree.getQueues()) {
+      topQueues.put(topQueue.getName(), topQueue);
+    }
+    
+    // check for consistency in number of children
+    assertEquals(2, topQueues.get("p1").getChildren().length);
+    
+    HashMap<String, JsonQueue> childQueues = new HashMap<String, JsonQueue>();
+    for (JsonQueue child : topQueues.get("p1").getChildren()) {
+      childQueues.put(child.getName(), child);
+    }
+    
+    // check for consistency in state
+    assertEquals("stopped", childQueues.get("p1:p12").getState());
+     
+    // check for consistency in properties
+    HashMap<String, JsonProperty> q1_properties =
+      new HashMap<String, JsonProperty>();
+    for (JsonProperty prop : topQueues.get("q1").getProperties()) {
+      q1_properties.put(prop.getKey(), prop);
+    }
+    assertEquals("10", q1_properties.get("capacity").getValue());
+    assertEquals("35", q1_properties.get("maxCapacity").getValue());
+    
+    // check for acls
+    assertEquals("u1", childQueues.get("p1:p12").getAcl_submit_job());
+    assertEquals("u2", childQueues.get("p1:p12").getAcl_administer_jobs());
+  }
 }



Mime
View raw message