hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r1213505 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test...
Date Mon, 12 Dec 2011 23:58:23 GMT
Author: mahadev
Date: Mon Dec 12 23:58:22 2011
New Revision: 1213505

URL: http://svn.apache.org/viewvc?rev=1213505&view=rev
Log:
MAPREDUCE-3328. mapred queue -list output inconsistent and missing child queues. (Ravi Prakash
via mahadev) - Merging r1213504 from trunk

Added:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java
      - copied unchanged from r1213504, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
      - copied unchanged from r1213504, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
Modified:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon Dec 12 23:58:22
2011
@@ -218,6 +218,9 @@ Release 0.23.1 - Unreleased
     MAPREDUCE-3527. Fix minor API incompatibilities between 1.0 and 0.23.
     (tomwhite)
 
+    MAPREDUCE-3328. mapred queue -list output inconsistent and missing child 
+    queues. (Ravi Prakash via mahadev)
+
 Release 0.23.0 - 2011-11-01 
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
Mon Dec 12 23:58:22 2011
@@ -450,9 +450,19 @@ public class TypeConverter {
 
   public static QueueInfo fromYarn(org.apache.hadoop.yarn.api.records.QueueInfo
       queueInfo, Configuration conf) {
-    return new QueueInfo(queueInfo.getQueueName(),queueInfo.toString(),
-        fromYarn(queueInfo.getQueueState()), TypeConverter.fromYarnApps(
-        queueInfo.getApplications(), conf));
+    QueueInfo toReturn = new QueueInfo(queueInfo.getQueueName(), "Capacity: " +
+      queueInfo.getCapacity() * 100 + ", MaximumCapacity: " +
+      (queueInfo.getMaximumCapacity() < 0 ? "UNDEFINED" :
+        queueInfo.getMaximumCapacity()) + ", CurrentCapacity: " +
+      queueInfo.getCurrentCapacity() * 100, fromYarn(queueInfo.getQueueState()),
+      TypeConverter.fromYarnApps(queueInfo.getApplications(), conf));
+    List<QueueInfo> childQueues = new ArrayList<QueueInfo>();
+    for(org.apache.hadoop.yarn.api.records.QueueInfo childQueue :
+      queueInfo.getChildQueues()) {
+      childQueues.add(fromYarn(childQueue, conf));
+    }
+    toReturn.setQueueChildren(childQueues);
+    return toReturn;
   }
 
   public static QueueInfo[] fromYarnQueueInfo(

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
Mon Dec 12 23:58:22 2011
@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.mapreduce;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import junit.framework.Assert;
 
 import org.apache.hadoop.conf.Configuration;
@@ -36,6 +39,7 @@ import org.apache.hadoop.yarn.api.record
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 public class TestTypeConverter {
   @Test
@@ -134,4 +138,33 @@ public class TestTypeConverter {
     Assert.assertEquals("queueInfo translation didn't work.",
       returned.getState().toString(), queueInfo.getQueueState().toString().toLowerCase());
   }
+
+  /**
+   * Test that child queues are converted too during conversion of the parent
+   * queue
+   */
+  @Test
+  public void testFromYarnQueue() {
+    //Define child queue
+    org.apache.hadoop.yarn.api.records.QueueInfo child =
+      Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class);
+    Mockito.when(child.getQueueState()).thenReturn(QueueState.RUNNING);
+
+    //Define parent queue
+    org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
+      Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class);
+    List<org.apache.hadoop.yarn.api.records.QueueInfo> children =
+      new ArrayList<org.apache.hadoop.yarn.api.records.QueueInfo>();
+    children.add(child); //Add one child
+    Mockito.when(queueInfo.getChildQueues()).thenReturn(children);
+    Mockito.when(queueInfo.getQueueState()).thenReturn(QueueState.RUNNING);
+
+    //Call the function we're testing
+    org.apache.hadoop.mapreduce.QueueInfo returned =
+      TypeConverter.fromYarn(queueInfo, new Configuration());
+
+    //Verify that the converted queue has the 1 child we had added
+    Assert.assertEquals("QueueInfo children weren't properly converted",
+      returned.getQueueChildren().size(), 1);
+  }
 }

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
Mon Dec 12 23:58:22 2011
@@ -110,40 +110,33 @@ class JobQueueClient extends Configured 
   }
 
   // format and print information about the passed in job queue.
-  void printJobQueueInfo(JobQueueInfo jobQueueInfo, Writer writer)
-      throws IOException {
+  void printJobQueueInfo(JobQueueInfo jobQueueInfo, Writer writer,
+    String prefix) throws IOException {
     if (jobQueueInfo == null) {
       writer.write("No queue found.\n");
       writer.flush();
       return;
     }
-    writer.write(String.format("Queue Name : %s \n",
+    writer.write(String.format(prefix + "======================\n"));
+    writer.write(String.format(prefix + "Queue Name : %s \n",
         jobQueueInfo.getQueueName()));
-    writer.write(String.format("Queue State : %s \n",
+    writer.write(String.format(prefix + "Queue State : %s \n",
         jobQueueInfo.getQueueState()));
-    writer.write(String.format("Scheduling Info : %s \n",
+    writer.write(String.format(prefix + "Scheduling Info : %s \n",
         jobQueueInfo.getSchedulingInfo()));
     List<JobQueueInfo> childQueues = jobQueueInfo.getChildren();
     if (childQueues != null && childQueues.size() > 0) {
-      writer.write(String.format("Child Queues : "));
       for (int i = 0; i < childQueues.size(); i++) {
-        JobQueueInfo childQueue = childQueues.get(i);
-        writer.write(String.format("%s", childQueue.getQueueName()));
-        if (i != childQueues.size() - 1) {
-          writer.write(String.format(", "));
-        }
+	  printJobQueueInfo(childQueues.get(i), writer, "    " + prefix);
       }
-      writer.write("\n");
     }
-    writer.write(String.format("======================\n"));
     writer.flush();
   }
   
   private void displayQueueList() throws IOException {
     JobQueueInfo[] rootQueues = jc.getRootQueues();
-    List<JobQueueInfo> allQueues = expandQueueList(rootQueues);
-    for (JobQueueInfo queue : allQueues) {
-      printJobQueueInfo(queue, new PrintWriter(System.out));
+    for (JobQueueInfo queue : rootQueues) {
+      printJobQueueInfo(queue, new PrintWriter(System.out), "");
     }
   }
   
@@ -181,7 +174,7 @@ class JobQueueClient extends Configured 
       System.out.println("Queue \"" + queue + "\" does not exist.");
       return;
     }
-    printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out));
+    printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out), "");
     if (showJobs && (jobQueueInfo.getChildren() == null ||
         jobQueueInfo.getChildren().size() == 0)) {
       JobStatus[] jobs = jc.getJobsFromQueue(queue);

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
Mon Dec 12 23:58:22 2011
@@ -105,7 +105,7 @@ public class JobQueueInfo extends QueueI
   public List<JobQueueInfo> getChildren() {
     List<JobQueueInfo> list = new ArrayList<JobQueueInfo>();
     for (QueueInfo q : super.getQueueChildren()) {
-      list.add((JobQueueInfo)q);
+      list.add(new JobQueueInfo(q));
     }
     return list;
   }

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
Mon Dec 12 23:58:22 2011
@@ -198,13 +198,16 @@ public class ResourceMgrDelegate {
   }
   
   private void getChildQueues(org.apache.hadoop.yarn.api.records.QueueInfo parent, 
-      List<org.apache.hadoop.yarn.api.records.QueueInfo> queues) {
+      List<org.apache.hadoop.yarn.api.records.QueueInfo> queues,
+      boolean recursive) {
     List<org.apache.hadoop.yarn.api.records.QueueInfo> childQueues = 
       parent.getChildQueues();
 
     for (org.apache.hadoop.yarn.api.records.QueueInfo child : childQueues) {
       queues.add(child);
-      getChildQueues(child, queues);
+      if(recursive) {
+        getChildQueues(child, queues, recursive);
+      }
     }
   }
 
@@ -226,7 +229,7 @@ public class ResourceMgrDelegate {
     org.apache.hadoop.yarn.api.records.QueueInfo rootQueue = 
       applicationsManager.getQueueInfo(
           getQueueInfoRequest(ROOT, false, true, true)).getQueueInfo();
-    getChildQueues(rootQueue, queues);
+    getChildQueues(rootQueue, queues, true);
 
     return TypeConverter.fromYarnQueueInfo(queues, this.conf);
   }
@@ -238,8 +241,8 @@ public class ResourceMgrDelegate {
 
     org.apache.hadoop.yarn.api.records.QueueInfo rootQueue = 
       applicationsManager.getQueueInfo(
-          getQueueInfoRequest(ROOT, false, true, false)).getQueueInfo();
-    getChildQueues(rootQueue, queues);
+          getQueueInfoRequest(ROOT, false, true, true)).getQueueInfo();
+    getChildQueues(rootQueue, queues, false);
 
     return TypeConverter.fromYarnQueueInfo(queues, this.conf);
   }
@@ -252,7 +255,7 @@ public class ResourceMgrDelegate {
         org.apache.hadoop.yarn.api.records.QueueInfo parentQueue = 
           applicationsManager.getQueueInfo(
               getQueueInfoRequest(parent, false, true, false)).getQueueInfo();
-        getChildQueues(parentQueue, queues);
+        getChildQueues(parentQueue, queues, true);
         
         return TypeConverter.fromYarnQueueInfo(queues, this.conf);
   }



Mime
View raw message