hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1094103 [2/2] - in /hadoop/mapreduce/branches/MR-279: ./ mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/ mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ mr-client/hadoop-ma...
Date Sun, 17 Apr 2011 05:19:07 GMT
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
Sun Apr 17 05:19:05 2011
@@ -21,8 +21,10 @@ package org.apache.hadoop.yarn.server.re
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -31,9 +33,13 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Application;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager;
@@ -66,6 +72,13 @@ public class ParentQueue implements Queu
   private volatile int numApplications;
   private volatile int numContainers;
 
+  private QueueInfo queueInfo; 
+  private Map<ApplicationId, org.apache.hadoop.yarn.api.records.Application> 
+  applicationInfos;
+
+  private final RecordFactory recordFactory = 
+    RecordFactoryProvider.getRecordFactory(null);
+
   public ParentQueue(CapacitySchedulerContext cs, 
       String queueName, Comparator<Queue> comparator, Queue parent) {
     minimumAllocation = cs.getMinimumAllocation();
@@ -91,6 +104,16 @@ public class ParentQueue implements Queu
     
     this.childQueues = new TreeSet<Queue>(comparator);
     
+    this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
+    this.queueInfo.setCapacity(capacity);
+    this.queueInfo.setMaximumCapacity(maximumCapacity);
+    this.queueInfo.setQueueName(queueName);
+    this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
+    
+    this.applicationInfos = 
+      new HashMap<ApplicationId, 
+      org.apache.hadoop.yarn.api.records.Application>();
+
     LOG.info("Initialized parent-queue " + queueName + 
         " name=" + queueName + 
         ", fullname=" + getQueuePath() + 
@@ -188,6 +211,33 @@ public class ParentQueue implements Queu
     return numApplications;
   }
 
+  @Override
+  public synchronized QueueInfo getQueueInfo(boolean includeApplications, 
+      boolean includeChildQueues, boolean recursive) {
+    queueInfo.setCurrentCapacity(usedCapacity);
+
+    if (includeApplications) {
+      queueInfo.setApplications( 
+        new ArrayList<org.apache.hadoop.yarn.api.records.Application>(
+            applicationInfos.values()));
+    } else {
+      queueInfo.setApplications(
+          new ArrayList<org.apache.hadoop.yarn.api.records.Application>());
+    }
+
+    List<QueueInfo> childQueuesInfo = new ArrayList<QueueInfo>();
+    if (includeChildQueues) {
+      for (Queue child : childQueues) {
+        // Get queue information recursively?
+        childQueuesInfo.add(
+            child.getQueueInfo(includeApplications, recursive, recursive));
+      }
+    }
+    queueInfo.setChildQueues(childQueuesInfo);
+    
+    return queueInfo;
+}
+
   public String toString() {
     return queueName + ":" + capacity + ":" + absoluteCapacity + ":" + 
       getUsedCapacity() + ":" + getUtilization() + ":" + 
@@ -207,6 +257,9 @@ public class ParentQueue implements Queu
     
     ++numApplications;
    
+    applicationInfos.put(application.getApplicationId(), 
+        application.getApplicationInfo());
+
     LOG.info("Application submission -" +
     		" appId: " + application.getApplicationId() + 
         " user: " + user + 
@@ -229,7 +282,8 @@ public class ParentQueue implements Queu
     }
     
     --numApplications;
-    
+    applicationInfos.remove(application.getApplicationId());
+
     LOG.info("Application completion -" +
         " appId: " + application.getApplicationId() + 
         " user: " + application.getUser() + 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Sun Apr 17 05:19:05 2011
@@ -32,17 +32,16 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.Node;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerToken;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.server.api.records.NodeId;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ASMEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
@@ -76,13 +75,32 @@ public class FifoScheduler implements Re
     new TreeMap<ApplicationId, Application>(
         new org.apache.hadoop.yarn.server.resourcemanager.resource.ApplicationID.Comparator());
 
-  private static final Queue DEFAULT_QUEUE = new Queue() {
+  private final Queue DEFAULT_QUEUE = new Queue() {
+    QueueInfo queueInfo;
+    
     @Override
     public String getQueueName() {
       return "default";
     }
+
+    @Override
+    public QueueInfo getQueueInfo(boolean includeApplications, 
+        boolean includeChildQueues, boolean recursive) {
+      queueInfo.setQueueName(DEFAULT_QUEUE.getQueueName());
+      queueInfo.setCapacity(100.0f);
+      queueInfo.setMaximumCapacity(100.0f);
+      queueInfo.setChildQueues(new ArrayList<QueueInfo>());
+
+      if (includeApplications) {
+        queueInfo.setApplications(getApplications());
+      } else {
+        queueInfo.setApplications(
+            new ArrayList<org.apache.hadoop.yarn.api.records.Application>());
+      }
+      return queueInfo;
+    }
   };
-  
+
   public FifoScheduler() {}
   
   public FifoScheduler(Configuration conf,
@@ -160,6 +178,7 @@ public class FifoScheduler implements Re
     return applications.get(applicationId);
   }
 
+  @Override
   public synchronized void addApplication(ApplicationId applicationId, 
       String user, String unusedQueue, Priority unusedPriority) 
   throws IOException {
@@ -169,6 +188,7 @@ public class FifoScheduler implements Re
         ", currently active: " + applications.size());
   }
 
+  @Override
   public synchronized void removeApplication(ApplicationId applicationId)
   throws IOException {
     Application application = getApplication(applicationId);
@@ -462,6 +482,25 @@ public class FifoScheduler implements Re
     nodes.remove(nodeInfo.getHostName());
   }
   
+
+  @Override
+  public QueueInfo getQueueInfo(String queueName, boolean includeApplications,
+      boolean includeChildQueues, boolean recursive) {
+    QueueInfo queueInfo = 
+      DEFAULT_QUEUE.getQueueInfo(includeApplications, false, false);
+    return queueInfo;
+  }
+  
+  private synchronized List<org.apache.hadoop.yarn.api.records.Application> 
+  getApplications() {
+    List<org.apache.hadoop.yarn.api.records.Application> applications = 
+      new ArrayList<org.apache.hadoop.yarn.api.records.Application>();
+    for (Application application : this.applications.values()) {
+      applications.add(application.getApplicationInfo());
+    }
+    return applications;
+  }
+
   public synchronized boolean isTracked(NodeInfo nodeInfo) {
     NodeManager node = nodes.get(nodeInfo.getHostName());
     return (node == null? false: true);

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
Sun Apr 17 05:19:05 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -70,6 +71,23 @@ public class TestAMLaunchFailure extends
         List<ResourceRequest> ask, List<Container> release) throws IOException
{
       return Arrays.asList(container);
     }
+
+    @Override
+    public void addApplication(ApplicationId applicationId, String user,
+        String queue, Priority priority) throws IOException {
+    }
+
+    @Override
+    public void removeApplication(ApplicationId applicationId)
+        throws IOException {
+    }
+
+    @Override
+    public QueueInfo getQueueInfo(String queueName,
+        boolean includeApplications, boolean includeChildQueues,
+        boolean recursive) throws IOException {
+      return null;
+    }
   }
 
   private class DummyApplicationTracker implements EventHandler<ASMEvent<ApplicationTrackerEventType>>
{

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
Sun Apr 17 05:19:05 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -80,6 +81,23 @@ public class TestAMRMRPCResponseId exten
         List<ResourceRequest> ask, List<Container> release) throws IOException
{
       return null;
     }
+
+    @Override
+    public void addApplication(ApplicationId applicationId, String user,
+        String queue, Priority priority) throws IOException {
+    }
+
+    @Override
+    public void removeApplication(ApplicationId applicationId)
+        throws IOException {
+    }
+
+    @Override
+    public QueueInfo getQueueInfo(String queueName,
+        boolean includeApplications, boolean includeChildQueues,
+        boolean recursive) throws IOException {
+      return null;
+    }
   }
   
   @Before

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
Sun Apr 17 05:19:05 2011
@@ -21,6 +21,8 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -179,6 +181,20 @@ public class TestAMRestart extends TestC
     public void reinitialize(Configuration conf,
         ContainerTokenSecretManager secretManager) {
     }
+    @Override
+    public void addApplication(ApplicationId applicationId, String user,
+        String queue, Priority priority) throws IOException {
+    }
+    @Override
+    public void removeApplication(ApplicationId applicationId)
+        throws IOException {
+    }
+    @Override
+    public QueueInfo getQueueInfo(String queueName,
+        boolean includeApplications, boolean includeChildQueues,
+        boolean recursive) throws IOException {
+      return null;
+    }
   }
 
   @Before

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
Sun Apr 17 05:19:05 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -95,6 +96,20 @@ public class TestSchedulerNegotiator ext
     @Override
     public void handle(ASMEvent<ApplicationTrackerEventType> event) {
     }
+    @Override
+    public void addApplication(ApplicationId applicationId, String user,
+        String queue, Priority priority) throws IOException {
+    }
+    @Override
+    public void removeApplication(ApplicationId applicationId)
+        throws IOException {
+    }
+    @Override
+    public QueueInfo getQueueInfo(String queueName,
+        boolean includeApplications, boolean includeChildQueues,
+        boolean recursive) throws IOException {
+      return null;
+    }
   }
   
   @Before



Mime
View raw message