hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r1130967 - in /hadoop/mapreduce/branches/MR-279: ./ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job...
Date Fri, 03 Jun 2011 09:48:57 GMT
Author: sharad
Date: Fri Jun  3 09:48:56 2011
New Revision: 1130967

URL: http://svn.apache.org/viewvc?rev=1130967&view=rev
Log:
Support for min and max container capacity.

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri Jun  3 09:48:56 2011
@@ -4,6 +4,7 @@ Trunk (unreleased changes)
 
 
     MAPREDUCE-279
+    Support for min and max container capacity. (acmurthy and sharad)
 
     Re-enabled TestCapacityScheduler. (acmurthy)
 

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java Fri Jun  3 09:48:56 2011
@@ -19,42 +19,21 @@
 package org.apache.hadoop.mapreduce.v2.app.job.event;
 
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.Container;
 
 
 
 public class TaskAttemptContainerAssignedEvent extends TaskAttemptEvent {
 
-  private final ContainerId containerID;
-  private final String containerManagerAddress;
-  private final String nodeHttpAddress;
-  private final ContainerToken containerToken;
+  private final Container container;
 
   public TaskAttemptContainerAssignedEvent(TaskAttemptId id,
-      ContainerId containerID, String containerManagerAddress,
-      String nodeHttpAddress,
-      ContainerToken containerToken) {
+      Container container) {
     super(id, TaskAttemptEventType.TA_ASSIGNED);
-    this.containerID = containerID;
-    this.containerManagerAddress = containerManagerAddress;
-    this.nodeHttpAddress = nodeHttpAddress;
-    this.containerToken = containerToken;
+    this.container = container;
   }
 
-  public ContainerId getContainerID() {
-    return this.containerID;
-  }
-
-  public String getContainerManagerAddress() {
-    return this.containerManagerAddress;
-  }
-
-  public ContainerToken getContainerToken() {
-    return this.containerToken;
-  }
-
-  public String getNodeHttpAddress() {
-    return this.nodeHttpAddress;
+  public Container getContainer() {
+    return this.container;
   }
 }

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Fri Jun  3 09:48:56 2011
@@ -375,6 +375,7 @@ public abstract class TaskAttemptImpl im
   private String nodeHttpAddress;
   private WrappedJvmID jvmID;
   private ContainerToken containerToken;
+  private Resource assignedCapability;
   
   //this takes good amount of memory ~ 30KB. Instantiate it lazily
   //and make it null once task is launched.
@@ -591,7 +592,7 @@ public abstract class TaskAttemptImpl im
     // Construct the actual Container
     container.setContainerId(containerID);
     container.setUser(conf.get(MRJobConfig.USER_NAME));
-    container.setResource(resourceCapability);
+    container.setResource(assignedCapability);
     return container;
   }
 
@@ -888,10 +889,11 @@ public abstract class TaskAttemptImpl im
         TaskAttemptEvent event) {
       TaskAttemptContainerAssignedEvent cEvent = 
         (TaskAttemptContainerAssignedEvent) event;
-      taskAttempt.containerID = cEvent.getContainerID();
-      taskAttempt.containerMgrAddress = cEvent.getContainerManagerAddress();
-      taskAttempt.nodeHttpAddress = cEvent.getNodeHttpAddress();
-      taskAttempt.containerToken = cEvent.getContainerToken();
+      taskAttempt.containerID = cEvent.getContainer().getId();
+      taskAttempt.containerMgrAddress = cEvent.getContainer().getContainerManagerAddress();
+      taskAttempt.nodeHttpAddress = cEvent.getContainer().getNodeHttpAddress();
+      taskAttempt.containerToken = cEvent.getContainer().getContainerToken();
+      taskAttempt.assignedCapability = cEvent.getContainer().getResource();
       // this is a _real_ Task (classic Hadoop mapred flavor):
       taskAttempt.remoteTask = taskAttempt.createRemoteTask();
       taskAttempt.jvmID = new WrappedJvmID(

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java Fri Jun  3 09:48:56 2011
@@ -29,6 +29,7 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
 import org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -66,12 +67,15 @@ public class LocalContainerAllocator ext
       cID.setAppId(appID);
       // use negative ids to denote that these are local. Need a better way ??
       cID.setId((-1) * containerCount.getAndIncrement());
+      
+      Container container = recordFactory.newRecordInstance(Container.class);
+      container.setId(cID);
+      container.setContainerManagerAddress("localhost");
+      container.setContainerToken(null);
+      container.setNodeHttpAddress("localhost:9999");
       // send the container-assigned event to task attempt
       eventHandler.handle(new TaskAttemptContainerAssignedEvent(
-          event.getAttemptID(), cID,
-          "localhost",//put the AppMaster hostname (TODO)
-          "localhost:9999",// put the httpAddress
-          null));
+          event.getAttemptID(), container));
     }
   }
 

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java Fri Jun  3 09:48:56 2011
@@ -58,6 +58,7 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
@@ -349,9 +350,15 @@ public class RecoveryService extends Com
       LOG.info("Sending assigned event to " + yarnAttemptID);
       ContainerId cId = RecordFactoryProvider.getRecordFactory(null)
           .newRecordInstance(ContainerId.class);
+      Container container = RecordFactoryProvider.getRecordFactory(null)
+          .newRecordInstance(Container.class);
+      container.setId(cId);
+      container.setContainerManagerAddress("localhost");
+      container.setContainerToken(null);
+      container.setNodeHttpAddress(attemptInfo.getHostname() + ":" + 
+          attemptInfo.getHttpPort());
       actualHandler.handle(new TaskAttemptContainerAssignedEvent(yarnAttemptID,
-          cId, null, attemptInfo.getHostname() + ":" + 
-          attemptInfo.getHttpPort(), null));
+          container));
     }
   }
 

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java Fri Jun  3 09:48:56 2011
@@ -41,12 +41,14 @@ import org.apache.hadoop.yarn.api.protoc
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
 import org.apache.hadoop.yarn.api.records.AMResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationMaster;
 import org.apache.hadoop.yarn.api.records.ApplicationState;
 import org.apache.hadoop.yarn.api.records.ApplicationStatus;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -70,6 +72,8 @@ public class RMCommunicator extends Abst
   protected AMRMProtocol scheduler;
   private final ClientService clientService;
   private int lastResponseID;
+  private Resource minContainerCapability;
+  private Resource maxContainerCapability;
 
   private final RecordFactory recordFactory =
       RecordFactoryProvider.getRecordFactory(null);
@@ -116,7 +120,12 @@ public class RMCommunicator extends Abst
       RegisterApplicationMasterRequest request =
         recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
       request.setApplicationMaster(applicationMaster);
-      scheduler.registerApplicationMaster(request);
+      RegisterApplicationMasterResponse response = 
+        scheduler.registerApplicationMaster(request);
+      minContainerCapability = response.getMinimumResourceCapability();
+      maxContainerCapability = response.getMaximumResourceCapability();
+      LOG.info("minContainerCapability: " + minContainerCapability.getMemory());
+      LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory());
     } catch (Exception are) {
       LOG.info("Exception while registering", are);
       throw new YarnException(are);
@@ -135,6 +144,14 @@ public class RMCommunicator extends Abst
     }
   }
 
+  protected Resource getMinContainerCapability() {
+    return minContainerCapability;
+  }
+  
+  protected Resource getMaxContainerCapability() {
+    return maxContainerCapability;
+  }
+
   @Override
   public void stop() {
     stopped = true;

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Fri Jun  3 09:48:56 2011
@@ -44,6 +44,8 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
 import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
@@ -183,14 +185,33 @@ public class RMContainerAllocator extend
       if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) {
         if (mapResourceReqt == 0) {
           mapResourceReqt = reqEvent.getCapability().getMemory();
+          int minSlotMemSize = getMinContainerCapability().getMemory();
+          mapResourceReqt = (int) Math.ceil((float) mapResourceReqt/minSlotMemSize) * minSlotMemSize;
           LOG.info("mapResourceReqt:"+mapResourceReqt);
+          if (mapResourceReqt > getMaxContainerCapability().getMemory()) {
+            LOG.info("Map capability required is more than the supported " +
+            		"max container capability in the cluster. Killing the Job.");
+            eventHandler.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
+          }
         }
+        //set the rounded off memory
+        reqEvent.getCapability().setMemory(mapResourceReqt);
         scheduledRequests.addMap(reqEvent);//maps are immediately scheduled
       } else {
         if (reduceResourceReqt == 0) {
           reduceResourceReqt = reqEvent.getCapability().getMemory();
+          int minSlotMemSize = getMinContainerCapability().getMemory();
+          //round off on slotsize
+          reduceResourceReqt = (int) Math.ceil((float) reduceResourceReqt/minSlotMemSize) * minSlotMemSize;
           LOG.info("reduceResourceReqt:"+reduceResourceReqt);
+          if (reduceResourceReqt > getMaxContainerCapability().getMemory()) {
+            LOG.info("Reduce capability required is more than the supported " +
+                    "max container capability in the cluster. Killing the Job.");
+            eventHandler.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
+          }
         }
+        //set the rounded off memory
+        reqEvent.getCapability().setMemory(reduceResourceReqt);
         if (reqEvent.getEarlierAttemptFailed()) {
           //add to the front of queue for fail fast
           pendingReduces.addFirst(new ContainerRequest(reqEvent, PRIORITY_REDUCE));
@@ -480,10 +501,7 @@ public class RMContainerAllocator extend
 
           // send the container-assigned event to task attempt
           eventHandler.handle(new TaskAttemptContainerAssignedEvent(
-              assigned.attemptID, allocated.getId(),
-              allocated.getContainerManagerAddress(),
-              allocated.getNodeHttpAddress(),
-              allocated.getContainerToken()));
+              assigned.attemptID, allocated));
 
           assignedRequests.add(allocated, assigned.attemptID);
           

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java Fri Jun  3 09:48:56 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapreduce.v2.MR
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
 import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.service.AbstractService;
@@ -139,11 +140,16 @@ public class StaticContainerAllocator ex
         if (nextIndex < containerMgrList.size()) {
           String containerMgr = containerMgrList.get(nextIndex);
           ContainerId containerID = generateContainerID();
-
-        context.getEventHandler().handle(
+          
+          Container container = RecordFactoryProvider.getRecordFactory(null)
+              .newRecordInstance(Container.class);
+          container.setId(containerID);
+          container.setContainerManagerAddress(containerMgr);
+          container.setContainerToken(null);
+          container.setNodeHttpAddress("localhost:9999");
+          context.getEventHandler().handle(
             new TaskAttemptContainerAssignedEvent(
-                event.getAttemptID(),
-                containerID, containerMgr, "localhost:9999", null));
+                event.getAttemptID(), container));
         }
       }
     }

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java Fri Jun  3 09:48:56 2011
@@ -67,6 +67,7 @@ import org.apache.hadoop.security.Creden
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -320,9 +321,14 @@ public class MRApp extends MRAppMaster {
         ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
         cId.setAppId(getContext().getApplicationID());
         cId.setId(containerCount++);
+        Container container = recordFactory.newRecordInstance(Container.class);
+        container.setId(cId);
+        container.setContainerManagerAddress("dummy");
+        container.setContainerToken(null);
+        container.setNodeHttpAddress("localhost:9999");
         getContext().getEventHandler().handle(
-            new TaskAttemptContainerAssignedEvent(event.getAttemptID(), cId,
-                "dummy", "localhost:9999",null));
+            new TaskAttemptContainerAssignedEvent(event.getAttemptID(),
+                container));
       }
     };
   }

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java Fri Jun  3 09:48:56 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
 import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
 import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.service.AbstractService;
@@ -124,10 +125,17 @@ public class MRAppBenchmark {
                   cId.setAppId(getContext().getApplicationID());
                   cId.setId(containerCount++);
                   //System.out.println("Allocating " + containerCount);
-                  getContext().getEventHandler().handle(
+                  
+                  Container container = RecordFactoryProvider.getRecordFactory(
+                      null).newRecordInstance(Container.class);
+                  container.setId(cId);
+                  container.setContainerManagerAddress("dumm");
+                  container.setContainerToken(null);
+                  container.setNodeHttpAddress("localhost:9999");
+                  getContext().getEventHandler()
+                      .handle(
                       new TaskAttemptContainerAssignedEvent(event
-                          .getAttemptID(), cId, "dummy", "localhost:9999",
-                          null));
+                          .getAttemptID(), container));
                   concurrentRunningTasks++;
                 } else {
                   Thread.sleep(1000);

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java Fri Jun  3 09:48:56 2011
@@ -213,7 +213,7 @@ public class TestRMContainerAllocator {
     //validate that no container is assigned to h1 as it doesn't have 2048
     for (TaskAttemptContainerAssignedEvent assig : assigned) {
       Assert.assertFalse("Assigned count not correct", 
-          "h1".equals(assig.getContainerManagerAddress()));
+          "h1".equals(assig.getContainer().getContainerManagerAddress()));
     }
   }
 
@@ -313,7 +313,7 @@ public class TestRMContainerAllocator {
     //check for uniqueness of containerIDs
     Set<ContainerId> containerIds = new HashSet<ContainerId>();
     for (TaskAttemptContainerAssignedEvent assigned : assignments) {
-      containerIds.add(assigned.getContainerID());
+      containerIds.add(assigned.getContainer().getId());
     }
     Assert.assertEquals("Assigned containers must be different", 
         assignments.size(), containerIds.size());
@@ -339,7 +339,7 @@ public class TestRMContainerAllocator {
         assigned.getTaskAttemptID());
     if (checkHostMatch) {
       Assert.assertTrue("Not assigned to requested host", Arrays.asList
-          (request.getHosts()).contains(assigned.getContainerManagerAddress()));
+          (request.getHosts()).contains(assigned.getContainer().getContainerManagerAddress()));
     }
 
   }
@@ -408,6 +408,20 @@ public class TestRMContainerAllocator {
     @Override
     protected void unregister() {}
 
+    @Override
+    protected Resource getMinContainerCapability() {
+      Resource res = recordFactory.newRecordInstance(Resource.class);
+      res.setMemory(1024);
+      return res;
+    }
+    
+    @Override
+    protected Resource getMaxContainerCapability() {
+      Resource res = recordFactory.newRecordInstance(Resource.class);
+      res.setMemory(10240);
+      return res;
+    }
+    
     public void sendRequest(ContainerRequestEvent req) {
       sendRequests(Arrays.asList(new ContainerRequestEvent[]{req}));
     }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java Fri Jun  3 09:48:56 2011
@@ -1,5 +1,10 @@
 package org.apache.hadoop.yarn.api.protocolrecords;
 
+import org.apache.hadoop.yarn.api.records.Resource;
+
 public interface RegisterApplicationMasterResponse {
-  
+  public Resource getMinimumResourceCapability();
+  public void setMinimumResourceCapability(Resource capability);
+  public Resource getMaximumResourceCapability();
+  public void setMaximumResourceCapability(Resource capability);
 }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java Fri Jun  3 09:48:56 2011
@@ -3,15 +3,26 @@ package org.apache.hadoop.yarn.api.proto
 
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
 import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder;
 
 
     
-public class RegisterApplicationMasterResponsePBImpl extends ProtoBase<RegisterApplicationMasterResponseProto> implements RegisterApplicationMasterResponse {
-  RegisterApplicationMasterResponseProto proto = RegisterApplicationMasterResponseProto.getDefaultInstance();
+public class RegisterApplicationMasterResponsePBImpl 
+extends ProtoBase<RegisterApplicationMasterResponseProto> 
+implements RegisterApplicationMasterResponse {
+  RegisterApplicationMasterResponseProto proto = 
+    RegisterApplicationMasterResponseProto.getDefaultInstance();
   RegisterApplicationMasterResponseProto.Builder builder = null;
   boolean viaProto = false;
   
+  private Resource minimumResourceCapability;
+  private Resource maximumResourceCapability;
+  
   public RegisterApplicationMasterResponsePBImpl() {
     builder = RegisterApplicationMasterResponseProto.newBuilder();
   }
@@ -22,10 +33,31 @@ public class RegisterApplicationMasterRe
   }
   
   public RegisterApplicationMasterResponseProto getProto() {
+    mergeLocalToProto();
     proto = viaProto ? proto : builder.build();
     viaProto = true;
     return proto;
   }
+  
+  private void mergeLocalToProto() {
+    if (viaProto) 
+      maybeInitBuilder();
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void mergeLocalToBuilder() {
+    if (this.minimumResourceCapability != null) {
+      builder.setMinimumCapability(
+          convertToProtoFormat(this.minimumResourceCapability));
+    }
+    if (this.maximumResourceCapability != null) {
+      builder.setMaximumCapability(
+          convertToProtoFormat(this.maximumResourceCapability));
+    }
+  }
+
 
   private void maybeInitBuilder() {
     if (viaProto || builder == null) {
@@ -33,9 +65,61 @@ public class RegisterApplicationMasterRe
     }
     viaProto = false;
   }
+
+  @Override
+  public Resource getMaximumResourceCapability() {
+    if (this.maximumResourceCapability != null) {
+      return this.maximumResourceCapability;
+    }
+
+    RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasMaximumCapability()) {
+      return null;
+    }
     
-  
+    this.maximumResourceCapability = convertFromProtoFormat(p.getMaximumCapability());
+    return this.maximumResourceCapability;
+  }
 
+  @Override
+  public Resource getMinimumResourceCapability() {
+    if (this.minimumResourceCapability != null) {
+      return this.minimumResourceCapability;
+    }
+
+    RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasMinimumCapability()) {
+      return null;
+    }
+    
+    this.minimumResourceCapability = convertFromProtoFormat(p.getMinimumCapability());
+    return this.minimumResourceCapability;
+  }
+
+  @Override
+  public void setMaximumResourceCapability(Resource capability) {
+    maybeInitBuilder();
+    if(maximumResourceCapability == null) {
+      builder.clearMaximumCapability();
+    }
+    this.maximumResourceCapability = capability;
+  }
+
+  @Override
+  public void setMinimumResourceCapability(Resource capability) {
+    maybeInitBuilder();
+    if(minimumResourceCapability == null) {
+      builder.clearMinimumCapability();
+    }
+    this.minimumResourceCapability = capability;
+  }
 
+  private Resource convertFromProtoFormat(ResourceProto resource) {
+    return new ResourcePBImpl(resource);
+  }
+
+  private ResourceProto convertToProtoFormat(Resource resource) {
+    return ((ResourcePBImpl)resource).getProto();
+  }
 
 }  

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto Fri Jun  3 09:48:56 2011
@@ -13,6 +13,8 @@ message RegisterApplicationMasterRequest
 }
 
 message RegisterApplicationMasterResponseProto {
+  optional ResourceProto minimumCapability = 1;
+  optional ResourceProto maximumCapability = 2;
 }
 
 message FinishApplicationMasterRequestProto {

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Fri Jun  3 09:48:56 2011
@@ -123,7 +123,14 @@ AMRMProtocol, EventHandler<ASMEvent<Appl
       LOG.info("Exception registering application ", ie);
       throw RPCUtil.getRemoteException(ie);
     }
-    RegisterApplicationMasterResponse response = recordFactory.newRecordInstance(RegisterApplicationMasterResponse.class);
+    
+    // Pick up min/max resource from scheduler...
+    RegisterApplicationMasterResponse response = 
+      recordFactory.newRecordInstance(RegisterApplicationMasterResponse.class);
+    response.setMinimumResourceCapability(
+        rScheduler.getMinimumResourceCapability());
+    response.setMaximumResourceCapability(
+        rScheduler.getMaximumResourceCapability());
     return response;
   }
 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java Fri Jun  3 09:48:56 2011
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
 
@@ -91,4 +92,17 @@ public interface YarnScheduler {
    * @throws IOException
    */
   public List<QueueUserACLInfo> getQueueUserAclInfo();
+  
+  /**
+   * Get minimum allocatable {@link Resource}.
+   * @return minimum allocatable resource
+   */
+  public Resource getMinimumResourceCapability();
+  
+  /**
+   * Get maximum allocatable {@link Resource}.
+   * @return maximum allocatable resource
+   */
+  public Resource getMaximumResourceCapability();
+
 }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Fri Jun  3 09:48:56 2011
@@ -106,6 +106,7 @@ implements ResourceScheduler, CapacitySc
   private int numNodeManagers = 0;
 
   private Resource minimumAllocation;
+  private Resource maximumAllocation;
 
   private Map<ApplicationId, Application> applications =
     Collections.synchronizedMap(
@@ -129,10 +130,15 @@ implements ResourceScheduler, CapacitySc
   }
 
   @Override
-  public Resource getMinimumAllocation() {
+  public Resource getMinimumResourceCapability() {
     return minimumAllocation;
   }
 
+  @Override
+  public Resource getMaximumResourceCapability() {
+    return maximumAllocation;
+  }
+
   public synchronized int getNumClusterNodes() {
     return numNodeManagers;
   }
@@ -144,6 +150,7 @@ implements ResourceScheduler, CapacitySc
     if (!initialized) {
       this.conf = new CapacitySchedulerConfiguration(conf);
       this.minimumAllocation = this.conf.getMinimumAllocation();
+      this.maximumAllocation = this.conf.getMaximumAllocation();
       this.containerTokenSecretManager = containerTokenSecretManager;
       this.clusterTracker = clusterTracker;
       if (clusterTracker != null) clusterTracker.addListener(this);

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Fri Jun  3 09:48:56 2011
@@ -72,7 +72,13 @@ public class CapacitySchedulerConfigurat
   @Private
   public static final String MINIMUM_ALLOCATION = 
     PREFIX + "minimum-allocation-mb";
-  
+
+  private static final int MAXIMUM_MEMORY = 10240;
+
+  @Private
+  public static final String MAXIMUM_ALLOCATION = 
+    PREFIX + "maximum-allocation-mb";
+
   @Private
   public static int DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS = 10000;
   
@@ -199,6 +205,11 @@ public class CapacitySchedulerConfigurat
     return Resources.createResource(minimumMemory);
   }
 
+  public Resource getMaximumAllocation() {
+    int maximumMemory = getInt(MAXIMUM_ALLOCATION, MAXIMUM_MEMORY);
+    return Resources.createResource(maximumMemory);
+  }
+
   public boolean getEnableUserMetrics() {
     return getBoolean(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS);
   }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java Fri Jun  3 09:48:56 2011
@@ -27,8 +27,10 @@ import org.apache.hadoop.yarn.server.sec
 public interface CapacitySchedulerContext {
   CapacitySchedulerConfiguration getConfiguration();
   
-  Resource getMinimumAllocation();
-  
+  Resource getMinimumResourceCapability();
+
+  Resource getMaximumResourceCapability();
+
   ContainerTokenSecretManager getContainerTokenSecretManager();
   
   int getNumClusterNodes();

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Fri Jun  3 09:48:56 2011
@@ -114,7 +114,7 @@ public class LeafQueue implements Queue 
         QueueMetrics.forQueue(getQueuePath(), parent,
         cs.getConfiguration().getEnableUserMetrics());
     
-    this.minimumAllocation = cs.getMinimumAllocation();
+    this.minimumAllocation = cs.getMinimumResourceCapability();
     this.containerTokenSecretManager = cs.getContainerTokenSecretManager();
 
     float capacity = 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Fri Jun  3 09:48:56 2011
@@ -97,7 +97,7 @@ public class ParentQueue implements Queu
 
   public ParentQueue(CapacitySchedulerContext cs, 
       String queueName, Comparator<Queue> comparator, Queue parent, Queue old) {
-    minimumAllocation = cs.getMinimumAllocation();
+    minimumAllocation = cs.getMinimumResourceCapability();
     
     this.parent = parent;
     this.queueName = queueName;

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Jun  3 09:48:56 2011
@@ -31,6 +31,7 @@ import java.util.TreeMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -46,9 +47,11 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.RMConfig;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ASMEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
@@ -79,14 +82,26 @@ public class FifoScheduler implements Re
   Configuration conf;
   private ContainerTokenSecretManager containerTokenSecretManager;
 
-  // TODO: The memory-block size should be site-configurable?
-  public static final int MINIMUM_MEMORY = 1024;
   private final static Container[] EMPTY_CONTAINER_ARRAY = new Container[] {};
   private final static List<Container> EMPTY_CONTAINER_LIST = Arrays.asList(EMPTY_CONTAINER_ARRAY);
   private ClusterTracker clusterTracker;
 
-  public static final Resource MINIMUM_ALLOCATION =
-    Resources.createResource(MINIMUM_MEMORY);
+  private static final int MINIMUM_MEMORY = 1024;
+
+  private static final String FIFO_PREFIX = 
+    YarnConfiguration.RM_PREFIX + "fifo.";
+  @Private
+  public static final String MINIMUM_ALLOCATION = 
+    FIFO_PREFIX + "minimum-allocation-mb";
+
+  private static final int MAXIMUM_MEMORY = 10240;
+
+  @Private
+  public static final String MAXIMUM_ALLOCATION = 
+    FIFO_PREFIX + "maximum-allocation-mb";
+
+  private Resource minimumAllocation;
+  private Resource maximumAllocation;
 
   Map<ApplicationId, Application> applications =
       new TreeMap<ApplicationId, Application>(
@@ -155,16 +170,31 @@ public class FifoScheduler implements Re
       this.clusterTracker.addListener(this);
     }
   }
+  
+  @Override
+  public Resource getMinimumResourceCapability() {
+    return minimumAllocation;
+  }
+
+  @Override
+  public Resource getMaximumResourceCapability() {
+    return maximumAllocation;
+  }
 
   @Override
-  public void reinitialize(Configuration conf,
-      ContainerTokenSecretManager containerTokenSecretManager, ClusterTracker clusterTracker) 
+  public synchronized void reinitialize(Configuration conf,
+      ContainerTokenSecretManager containerTokenSecretManager, 
+      ClusterTracker clusterTracker) 
   throws IOException 
   {
     this.conf = conf;
     this.containerTokenSecretManager = containerTokenSecretManager;
     this.clusterTracker = clusterTracker;
     if (clusterTracker != null) this.clusterTracker.addListener(this);
+    this.minimumAllocation = 
+      Resources.createResource(conf.getInt(MINIMUM_ALLOCATION, MINIMUM_MEMORY));
+    this.maximumAllocation = 
+      Resources.createResource(conf.getInt(MAXIMUM_ALLOCATION, MAXIMUM_MEMORY));
   }
 
   @Override
@@ -312,7 +342,7 @@ public class FifoScheduler implements Re
       application.showRequests();
 
       // Done
-      if (Resources.lessThan(node.getAvailableResource(), MINIMUM_ALLOCATION)) {
+      if (Resources.lessThan(node.getAvailableResource(), minimumAllocation)) {
         return;
       }
     }
@@ -526,7 +556,7 @@ public class FifoScheduler implements Re
     applicationCompletedContainers(getCompletedContainers(containers));
     LOG.info("Node heartbeat " + node.getNodeID() + " resource = " + node.getAvailableResource());
     if (Resources.greaterThanOrEqual(node.getAvailableResource(),
-        MINIMUM_ALLOCATION)) {
+        minimumAllocation)) {
       assignContainers(node);
     }
     metrics.setAvailableResourcesToQueue(

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java Fri Jun  3 09:48:56 2011
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -103,6 +104,18 @@ public class TestAMLaunchFailure extends
       // TODO Auto-generated method stub
       
     }
+
+    @Override
+    public Resource getMaximumResourceCapability() {
+      // TODO Auto-generated method stub
+      return null;
+    }
+
+    @Override
+    public Resource getMinimumResourceCapability() {
+      // TODO Auto-generated method stub
+      return null;
+    }
   }
 
   private class DummyApplicationTracker implements EventHandler<ASMEvent<ApplicationTrackerEventType>> {

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java Fri Jun  3 09:48:56 2011
@@ -112,6 +112,18 @@ public class TestAMRMRPCResponseId exten
         ApplicationStore store)
         throws IOException { 
     }
+
+    @Override
+    public Resource getMaximumResourceCapability() {
+      // TODO Auto-generated method stub
+      return null;
+    }
+
+    @Override
+    public Resource getMinimumResourceCapability() {
+      // TODO Auto-generated method stub
+      return null;
+    }
   }
   
   @Before

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Fri Jun  3 09:48:56 2011
@@ -212,6 +212,18 @@ public class TestAMRestart extends TestC
     public void nodeUpdate(NodeInfo nodeInfo,
         Map<String, List<Container>> containers) {      
     }
+
+    @Override
+    public Resource getMaximumResourceCapability() {
+      // TODO Auto-generated method stub
+      return null;
+    }
+
+    @Override
+    public Resource getMinimumResourceCapability() {
+      // TODO Auto-generated method stub
+      return null;
+    }
   }
 
   @Before

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java?rev=1130967&r1=1130966&r2=1130967&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java Fri Jun  3 09:48:56 2011
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -136,6 +137,20 @@ public class TestSchedulerNegotiator ext
         ContainerTokenSecretManager secretManager, ClusterTracker clusterTracker)
         throws IOException {
     }
+
+
+    @Override
+    public Resource getMaximumResourceCapability() {
+      // TODO Auto-generated method stub
+      return null;
+    }
+
+
+    @Override
+    public Resource getMinimumResourceCapability() {
+      // TODO Auto-generated method stub
+      return null;
+    }
   }
   
   @Before



Mime
View raw message