hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1170378 [9/12] - in /hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/ dev-support/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-mapreduce-clie...
Date Tue, 13 Sep 2011 22:49:38 GMT
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Tue Sep 13 22:49:27 2011
@@ -99,7 +99,7 @@ public class ResourceManager extends Com
   protected NMLivelinessMonitor nmLivelinessMonitor;
   protected NodesListManager nodesListManager;
   private SchedulerEventDispatcher schedulerDispatcher;
-  private RMAppManager rmAppManager;
+  protected RMAppManager rmAppManager;
 
   private final AtomicBoolean shutdown = new AtomicBoolean(false);
   private WebApp webApp;
@@ -176,13 +176,13 @@ public class ResourceManager extends Com
     masterService = createApplicationMasterService();
     addService(masterService) ;
 
-    clientRM = createClientRMService();
-    addService(clientRM);
-
     this.rmAppManager = createRMAppManager();
     // Register event handler for RMAppManagerEvents
     this.rmDispatcher.register(RMAppManagerEventType.class,
         this.rmAppManager);
+
+    clientRM = createClientRMService();
+    addService(clientRM);
     
     adminService = createAdminService();
     addService(adminService);
@@ -202,7 +202,7 @@ public class ResourceManager extends Com
   protected ResourceScheduler createScheduler() {
     return 
     ReflectionUtils.newInstance(
-        conf.getClass(RMConfig.RESOURCE_SCHEDULER, 
+        conf.getClass(YarnConfiguration.RM_SCHEDULER, 
             FifoScheduler.class, ResourceScheduler.class), 
         this.conf);
   }
@@ -384,8 +384,8 @@ public class ResourceManager extends Com
 
   protected void startWepApp() {
     webApp = WebApps.$for("yarn", masterService).at(
-        conf.get(YarnConfiguration.RM_WEBAPP_BIND_ADDRESS,
-        YarnConfiguration.DEFAULT_RM_WEBAPP_BIND_ADDRESS)).
+        conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+        YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)).
       start(new RMWebApp(this));
 
   }
@@ -415,8 +415,8 @@ public class ResourceManager extends Com
   }
   
   protected void doSecureLogin() throws IOException {
-    SecurityUtil.login(conf, RMConfig.RM_KEYTAB,
-        YarnConfiguration.RM_SERVER_PRINCIPAL_KEY);
+    SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB,
+        YarnConfiguration.RM_PRINCIPAL);
   }
 
   @Override
@@ -441,7 +441,7 @@ public class ResourceManager extends Com
   }
 
   protected ClientRMService createClientRMService() {
-    return new ClientRMService(this.rmContext, scheduler);
+    return new ClientRMService(this.rmContext, scheduler, this.rmAppManager);
   }
 
   protected ApplicationMasterService createApplicationMasterService() {

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Tue Sep 13 22:49:27 2011
@@ -20,10 +20,6 @@ package org.apache.hadoop.yarn.server.re
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.crypto.SecretKey;
 
@@ -31,7 +27,6 @@ import org.apache.avro.ipc.Server;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.security.SecurityInfo;
@@ -44,7 +39,6 @@ import org.apache.hadoop.yarn.factory.pr
 import org.apache.hadoop.yarn.ipc.RPCUtil;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.server.RMNMSecurityInfoClass;
-import org.apache.hadoop.yarn.server.YarnServerConfig;
 import org.apache.hadoop.yarn.server.api.ResourceTracker;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
@@ -102,8 +96,8 @@ public class ResourceTrackerService exte
   @Override
   public synchronized void init(Configuration conf) {
     String resourceTrackerBindAddress =
-      conf.get(YarnServerConfig.RESOURCETRACKER_ADDRESS,
-          YarnServerConfig.DEFAULT_RESOURCETRACKER_BIND_ADDRESS);
+      conf.get(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+          YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS);
     resourceTrackerAddress = NetUtils.createSocketAddr(resourceTrackerBindAddress);
 
     RackResolver.init(conf);
@@ -123,8 +117,8 @@ public class ResourceTrackerService exte
     this.server =
       rpc.getServer(ResourceTracker.class, this, resourceTrackerAddress,
           rtServerConf, null,
-          rtServerConf.getInt(RMConfig.RM_RESOURCE_TRACKER_THREADS, 
-              RMConfig.DEFAULT_RM_RESOURCE_TRACKER_THREADS));
+          rtServerConf.getInt(YarnConfiguration.RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT, 
+              YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT));
     this.server.start();
 
   }
@@ -253,7 +247,7 @@ public class ResourceTrackerService exte
       // 4. Send status to RMNode, saving the latest response.
       this.rmContext.getDispatcher().getEventHandler().handle(
           new RMNodeStatusEvent(nodeId, remoteNodeStatus.getNodeHealthStatus(),
-              remoteNodeStatus.getAllContainers(), latestResponse));
+              remoteNodeStatus.getContainersStatuses(), latestResponse));
 
       nodeHeartBeatResponse.setHeartbeatResponse(latestResponse);
       return nodeHeartBeatResponse;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Tue Sep 13 22:49:27 2011
@@ -33,7 +33,6 @@ import org.apache.commons.codec.binary.B
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.io.DataInputByteBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
@@ -80,7 +79,8 @@ public class AMLauncher implements Runna
 
   private final RMAppAttempt application;
   private final Configuration conf;
-  private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+  private final RecordFactory recordFactory = 
+      RecordFactoryProvider.getRecordFactory(null);
   private final ApplicationTokenSecretManager applicationTokenSecretManager;
   private final ClientToAMSecretManager clientToAMSecretManager;
   private final AMLauncherEventType eventType;
@@ -88,9 +88,9 @@ public class AMLauncher implements Runna
   @SuppressWarnings("rawtypes")
   private final EventHandler handler;
   
-  @SuppressWarnings("unchecked")
   public AMLauncher(RMContext rmContext, RMAppAttempt application,
-      AMLauncherEventType eventType,ApplicationTokenSecretManager applicationTokenSecretManager,
+      AMLauncherEventType eventType,
+      ApplicationTokenSecretManager applicationTokenSecretManager,
       ClientToAMSecretManager clientToAMSecretManager, Configuration conf) {
     this.application = application;
     this.conf = new Configuration(conf); // Just not to touch the sec-info class
@@ -107,7 +107,8 @@ public class AMLauncher implements Runna
     ContainerId masterContainerID = application.getMasterContainer().getId();
     
     containerMgrProxy =
-        getContainerMgrProxy(masterContainerID.getAppId());
+        getContainerMgrProxy(
+            masterContainerID.getApplicationAttemptId().getApplicationId());
   }
   
   private void launch() throws IOException {
@@ -170,12 +171,12 @@ public class AMLauncher implements Runna
 
     // Construct the actual Container
     ContainerLaunchContext container = recordFactory.newRecordInstance(ContainerLaunchContext.class);
-    container.addAllCommands(applicationMasterContext.getCommandList());
+    container.setCommands(applicationMasterContext.getCommandList());
     StringBuilder mergedCommand = new StringBuilder();
     String failCount = Integer.toString(application.getAppAttemptId()
         .getAttemptId());
     List<String> commandList = new ArrayList<String>();
-    for (String str : container.getCommandList()) {
+    for (String str : container.getCommands()) {
       // This is out-right wrong. AM FAIL count should be passed via env.
       String result =
           str.replaceFirst(ApplicationConstants.AM_FAIL_COUNT_STRING,
@@ -183,21 +184,21 @@ public class AMLauncher implements Runna
       mergedCommand.append(result).append(" ");
       commandList.add(result);
     }
-    container.clearCommands();
-    container.addAllCommands(commandList);
+    container.setCommands(commandList);
     /** add the failed count to the app master command line */
    
     LOG.info("Command to launch container " + 
         containerID + " : " + mergedCommand);
-    container.addAllEnv(applicationMasterContext.getAllEnvironment());
-
-    container.addAllEnv(setupTokensInEnv(applicationMasterContext));
+    Map<String, String> environment = 
+        applicationMasterContext.getAllEnvironment();
+    environment.putAll(setupTokensInEnv(applicationMasterContext));
+    container.setEnv(environment);
 
     // Construct the actual Container
     container.setContainerId(containerID);
     container.setUser(applicationMasterContext.getUser());
     container.setResource(applicationMasterContext.getMasterCapability());
-    container.addAllLocalResources(applicationMasterContext.getAllResourcesTodo());
+    container.setLocalResources(applicationMasterContext.getAllResourcesTodo());
     container.setContainerTokens(applicationMasterContext.getFsTokensTodo());
     return container;
   }
@@ -225,8 +226,8 @@ public class AMLauncher implements Runna
           new Token<ApplicationTokenIdentifier>(id,
               this.applicationTokenSecretManager);
       String schedulerAddressStr =
-          this.conf.get(YarnConfiguration.SCHEDULER_ADDRESS,
-              YarnConfiguration.DEFAULT_SCHEDULER_BIND_ADDRESS);
+          this.conf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+              YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS);
       InetSocketAddress unresolvedAddr =
           NetUtils.createSocketAddr(schedulerAddressStr);
       String resolvedAddr =

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java Tue Sep 13 22:49:27 2011
@@ -37,7 +37,6 @@ public class ApplicationMasterLauncher e
   private static final Log LOG = LogFactory.getLog(
       ApplicationMasterLauncher.class);
   private final ThreadPoolExecutor launcherPool;
-  private final EventHandler handler;
   private LauncherThread launcherHandlingThread;
   
   private final BlockingQueue<Runnable> masterEvents
@@ -52,7 +51,6 @@ public class ApplicationMasterLauncher e
       RMContext context) {
     super(ApplicationMasterLauncher.class.getName());
     this.context = context;
-    this.handler = context.getDispatcher().getEventHandler();
     /* register to dispatcher */
     this.context.getDispatcher().register(AMLauncherEventType.class, this);
     this.launcherPool = new ThreadPoolExecutor(1, 10, 1, 
@@ -67,14 +65,16 @@ public class ApplicationMasterLauncher e
     super.start();
   }
   
-  protected Runnable createRunnableLauncher(RMAppAttempt application, AMLauncherEventType event) {
+  protected Runnable createRunnableLauncher(RMAppAttempt application, 
+      AMLauncherEventType event) {
     Runnable launcher = new AMLauncher(context, application, event,
         applicationTokenSecretManager, clientToAMSecretManager, getConfig());
     return launcher;
   }
   
   private void launch(RMAppAttempt application) {
-    Runnable launcher = createRunnableLauncher(application, AMLauncherEventType.LAUNCH);
+    Runnable launcher = createRunnableLauncher(application, 
+        AMLauncherEventType.LAUNCH);
     masterEvents.add(launcher);
   }
   

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/StoreFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/StoreFactory.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/StoreFactory.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/StoreFactory.java Tue Sep 13 22:49:27 2011
@@ -23,14 +23,14 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationMaster;
 import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.server.resourcemanager.RMConfig;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
 
 public class StoreFactory {
   
   public static Store getStore(Configuration conf) {
     Store store = ReflectionUtils.newInstance(
-        conf.getClass(RMConfig.RM_STORE, 
+        conf.getClass(YarnConfiguration.RM_STORE, 
             MemStore.class, Store.class), 
             conf);
     return store;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKStore.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKStore.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKStore.java Tue Sep 13 22:49:27 2011
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMasterProto;
@@ -48,10 +49,7 @@ import org.apache.hadoop.yarn.proto.Yarn
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
-import org.apache.hadoop.yarn.server.resourcemanager.RMConfig;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
@@ -91,9 +89,9 @@ public class ZKStore implements Store {
 
   public ZKStore(Configuration conf) throws IOException {
     this.conf = conf;
-    this.ZK_ADDRESS = conf.get(RMConfig.ZK_ADDRESS);
-    this.ZK_TIMEOUT = conf.getInt(RMConfig.ZK_SESSION_TIMEOUT,
-        RMConfig.DEFAULT_ZK_TIMEOUT);
+    this.ZK_ADDRESS = conf.get(YarnConfiguration.RM_ZK_STORE_ADDRESS);
+    this.ZK_TIMEOUT = conf.getInt(YarnConfiguration.RM_ZK_STORE_TIMEOUT_MS,
+        YarnConfiguration.DEFAULT_RM_ZK_STORE_TIMEOUT_MS);
     zkClient = new ZooKeeper(this.ZK_ADDRESS, 
         this.ZK_TIMEOUT,
         createZKWatcher() 
@@ -123,10 +121,9 @@ public class ZKStore implements Store {
   public synchronized void storeNode(RMNode node) throws IOException {
     /** create a storage node and store it in zk **/
     if (!doneWithRecovery) return;
-    NodeReportPBImpl nodeManagerInfo = createNodeManagerInfo(node);
-    // TODO FinBugs - will be fixed after the subsequent fixme
-    byte[] bytes = nodeManagerInfo.getProto().toByteArray();
     // TODO: FIXMEVinodkv
+//    NodeReportPBImpl nodeManagerInfo = createNodeManagerInfo(node);
+//    byte[] bytes = nodeManagerInfo.getProto().toByteArray();
 //    try {
 //      zkClient.create(NODES + Integer.toString(node.getNodeID().getId()), bytes, null,
 //          CreateMode.PERSISTENT);
@@ -181,7 +178,8 @@ public class ZKStore implements Store {
   }
 
   private String containerPathFromContainerId(ContainerId containerId) {
-    String appString = ConverterUtils.toString(containerId.getAppId());
+    String appString = ConverterUtils.toString(
+        containerId.getApplicationAttemptId().getApplicationId());
     return appString + "/" + containerId.getId();
   }
 
@@ -198,7 +196,10 @@ public class ZKStore implements Store {
       
       ContainerPBImpl containerPBImpl = (ContainerPBImpl) container;
       try {
-        zkClient.setData(APPS + ConverterUtils.toString(container.getId().getAppId()) +
+        zkClient.setData(APPS + 
+            ConverterUtils.toString(
+                container.getId().getApplicationAttemptId().getApplicationId()) 
+                +
             ZK_PATH_SEPARATOR + APP_MASTER_CONTAINER
             , containerPBImpl.getProto().toByteArray(), -1);
       } catch(InterruptedException ie) {
@@ -476,12 +477,12 @@ public class ZKStore implements Store {
           continue;
         }
         int httpPort = Integer.valueOf(m.group(1));
-        // TODO: FindBugs Valid. Fix
-        RMNode nm = new RMNodeImpl(node.getNodeId(), null,
-            hostName, cmPort, httpPort,
-            ResourceTrackerService.resolve(node.getNodeId().getHost()), 
-            node.getCapability());
-        nodeManagers.add(nm);
+        // TODO: FindBugs warns passing null below. Commenting this for later.
+//        RMNode nm = new RMNodeImpl(node.getNodeId(), null,
+//            hostName, cmPort, httpPort,
+//            ResourceTrackerService.resolve(node.getNodeId().getHost()), 
+//            node.getCapability());
+//        nodeManagers.add(nm);
       }
       readLastNodeId();
       /* make sure we get all the applications */

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Tue Sep 13 22:49:27 2011
@@ -37,9 +37,9 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationState;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
-import org.apache.hadoop.yarn.server.resourcemanager.RMConfig;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
@@ -181,8 +181,8 @@ public class RMAppImpl implements RMApp 
     this.masterService = masterService;
     this.startTime = System.currentTimeMillis();
 
-    this.maxRetries = conf.getInt(RMConfig.AM_MAX_RETRIES,
-        RMConfig.DEFAULT_AM_MAX_RETRIES);
+    this.maxRetries = conf.getInt(YarnConfiguration.RM_AM_MAX_RETRIES,
+        YarnConfiguration.DEFAULT_RM_AM_MAX_RETRIES);
 
     ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     this.readLock = lock.readLock();

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java Tue Sep 13 22:49:27 2011
@@ -24,7 +24,6 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.server.resourcemanager.RMConfig;
 import org.apache.hadoop.yarn.util.AbstractLivelinessMonitor;
 
 public class AMLivelinessMonitor extends AbstractLivelinessMonitor<ApplicationAttemptId> {
@@ -38,10 +37,10 @@ public class AMLivelinessMonitor extends
 
   public void init(Configuration conf) {
     super.init(conf);
-    setExpireInterval(conf.getInt(YarnConfiguration.AM_EXPIRY_INTERVAL,
-        RMConfig.DEFAULT_AM_EXPIRY_INTERVAL));
-    setMonitorInterval(conf.getInt(RMConfig.AMLIVELINESS_MONITORING_INTERVAL,
-        RMConfig.DEFAULT_AMLIVELINESS_MONITORING_INTERVAL));
+    setExpireInterval(conf.getInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
+        YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS));
+    setMonitorInterval(conf.getInt(YarnConfiguration.RM_AM_LIVENESS_MONITOR_INTERVAL_MS,
+        YarnConfiguration.DEFAULT_RM_AM_LIVENESS_MONITOR_INTERVAL_MS));
   }
 
   @Override

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Tue Sep 13 22:49:27 2011
@@ -24,6 +24,7 @@ import java.util.Set;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.event.EventHandler;
 
@@ -47,9 +48,9 @@ public interface RMAppAttempt extends Ev
 
   Set<NodeId> getRanNodes();
 
-  List<Container> pullJustFinishedContainers();
+  List<ContainerStatus> pullJustFinishedContainers();
 
-  List<Container> getJustFinishedContainers();
+  List<ContainerStatus> getJustFinishedContainers();
 
   Container getMasterContainer();
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Tue Sep 13 22:49:27 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -97,8 +98,8 @@ public class RMAppAttemptImpl implements
   //nodes on while this attempt's containers ran
   private final Set<NodeId> ranNodes = 
     new HashSet<NodeId>();
-  private final List<Container> justFinishedContainers = 
-    new ArrayList<Container>();
+  private final List<ContainerStatus> justFinishedContainers = 
+    new ArrayList<ContainerStatus>();
   private Container masterContainer;
 
   private float progress = 0;
@@ -333,7 +334,7 @@ public class RMAppAttemptImpl implements
   }
 
   @Override
-  public List<Container> getJustFinishedContainers() {
+  public List<ContainerStatus> getJustFinishedContainers() {
     this.readLock.lock();
     try {
       return this.justFinishedContainers;
@@ -343,11 +344,11 @@ public class RMAppAttemptImpl implements
   }
 
   @Override
-  public List<Container> pullJustFinishedContainers() {
+  public List<ContainerStatus> pullJustFinishedContainers() {
     this.writeLock.lock();
 
     try {
-      List<Container> returnList = new ArrayList<Container>(
+      List<ContainerStatus> returnList = new ArrayList<ContainerStatus>(
           this.justFinishedContainers.size());
       returnList.addAll(this.justFinishedContainers);
       this.justFinishedContainers.clear();
@@ -705,11 +706,13 @@ public class RMAppAttemptImpl implements
 
       RMAppAttemptContainerFinishedEvent containerFinishedEvent
         = (RMAppAttemptContainerFinishedEvent) event;
-      Container container = containerFinishedEvent.getContainer();
+      ContainerStatus containerStatus = 
+          containerFinishedEvent.getContainerStatus();
 
       // Is this container the AmContainer? If the finished container is same as
       // the AMContainer, AppAttempt fails
-      if (appAttempt.masterContainer.getId().equals(container.getId())) {
+      if (appAttempt.masterContainer.getId().equals(
+          containerStatus.getContainerId())) {
         new FinalTransition(RMAppAttemptState.FAILED).transition(
             appAttempt, containerFinishedEvent);
         return RMAppAttemptState.FAILED;
@@ -718,7 +721,7 @@ public class RMAppAttemptImpl implements
       // Normal container.
 
       // Put it in completedcontainers list
-      appAttempt.justFinishedContainers.add(container);
+      appAttempt.justFinishedContainers.add(containerStatus);
       return RMAppAttemptState.RUNNING;
     }
   }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java Tue Sep 13 22:49:27 2011
@@ -19,22 +19,22 @@
 package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
 
 public class RMAppAttemptContainerFinishedEvent extends RMAppAttemptEvent {
 
-  private final Container container;
+  private final ContainerStatus containerStatus;
 
   public RMAppAttemptContainerFinishedEvent(ApplicationAttemptId appAttemptId, 
-      Container container) {
+      ContainerStatus containerStatus) {
     super(appAttemptId, RMAppAttemptEventType.CONTAINER_FINISHED);
-    this.container = container;
+    this.containerStatus = containerStatus;
   }
 
-  public Container getContainer() {
-    return this.container;
+  public ContainerStatus getContainerStatus() {
+    return this.containerStatus;
   }
 
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerAllocationExpirer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerAllocationExpirer.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerAllocationExpirer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerAllocationExpirer.java Tue Sep 13 22:49:27 2011
@@ -21,9 +21,9 @@ package org.apache.hadoop.yarn.server.re
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.SystemClock;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.server.resourcemanager.RMConfig;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent;
 import org.apache.hadoop.yarn.util.AbstractLivelinessMonitor;
 
@@ -40,10 +40,10 @@ public class ContainerAllocationExpirer 
   public void init(Configuration conf) {
     super.init(conf);
     setExpireInterval(conf.getInt(
-        RMConfig.CONTAINER_LIVELINESS_MONITORING_INTERVAL,
-        RMConfig.DEFAULT_CONTAINER_LIVELINESS_MONITORING_INTERVAL));
-    setMonitorInterval(conf.getInt(RMConfig.AMLIVELINESS_MONITORING_INTERVAL,
-        RMConfig.DEFAULT_AMLIVELINESS_MONITORING_INTERVAL));
+        YarnConfiguration.RM_CONTAINER_LIVENESS_MONITOR_INTERVAL_MS,
+        YarnConfiguration.DEFAULT_RM_CONTAINER_LIVENESS_MONITOR_INTERVAL_MS));
+    setMonitorInterval(conf.getInt(YarnConfiguration.RM_AM_LIVENESS_MONITOR_INTERVAL_MS,
+        YarnConfiguration.DEFAULT_RM_AM_LIVENESS_MONITOR_INTERVAL_MS));
   }
 
   @Override

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java Tue Sep 13 22:49:27 2011
@@ -26,8 +26,8 @@ public class RMContainerFinishedEvent ex
   private final ContainerStatus remoteContainerStatus;
 
   public RMContainerFinishedEvent(ContainerId containerId,
-      ContainerStatus containerStatus) {
-    super(containerId, RMContainerEventType.FINISHED);
+      ContainerStatus containerStatus, RMContainerEventType event) {
+    super(containerId, event);
     this.remoteContainerStatus = containerStatus;
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java Tue Sep 13 22:49:27 2011
@@ -92,7 +92,7 @@ public class RMContainerImpl implements 
 
     // Transitions from RUNNING state
     .addTransition(RMContainerState.RUNNING, RMContainerState.COMPLETED,
-        RMContainerEventType.FINISHED, new ContainerCompletedTransition())
+        RMContainerEventType.FINISHED, new FinishedTransition())
     .addTransition(RMContainerState.RUNNING, RMContainerState.KILLED,
         RMContainerEventType.KILL, new KillTransition())
 
@@ -273,10 +273,16 @@ public class RMContainerImpl implements 
 
     @Override
     public void transition(RMContainerImpl container, RMContainerEvent event) {
+      RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
+
+      // Update container-status for diagnostics. Today we completely
+      // replace it on finish. We may just need to update diagnostics.
+      container.container.setContainerStatus(finishedEvent
+          .getRemoteContainerStatus());
 
       // Inform AppAttempt
       container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
-          container.appAttemptId, container.container));
+          container.appAttemptId, container.container.getContainerStatus()));
     }
   }
 
@@ -312,22 +318,4 @@ public class RMContainerImpl implements 
     }
   }
 
-  private static final class ContainerCompletedTransition extends
-      FinishedTransition {
-
-    @Override
-    public void transition(RMContainerImpl container, RMContainerEvent event) {
-
-      RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
-
-      // Update container-status for diagnostics. Today we completely
-      // replace it on finish. We may just need to update diagnostics.
-      // ^TODO
-      container.container.setContainerStatus(finishedEvent
-          .getRemoteContainerStatus());
-
-      // Inform appAttempt
-      super.transition(container, event);
-    }
-  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java Tue Sep 13 22:49:27 2011
@@ -23,7 +23,6 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -36,9 +35,9 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -87,8 +86,8 @@ public class RMNodeImpl implements RMNod
       .newRecordInstance(NodeHealthStatus.class);
   
   /* set of containers that have just launched */
-  private final Map<ContainerId, Container> justLaunchedContainers = 
-    new HashMap<ContainerId, Container>();
+  private final Map<ContainerId, ContainerStatus> justLaunchedContainers = 
+    new HashMap<ContainerId, ContainerStatus>();
   
 
   /* set of containers that need to be cleaned */
@@ -355,43 +354,29 @@ public class RMNodeImpl implements RMNod
 
       // Filter the map to only obtain just launched containers and finished
       // containers.
-      Map<ApplicationId, List<Container>> remoteAppContainersMap = statusEvent
-          .getContainersCollection();
-      Map<ApplicationId, List<Container>> containersMapForScheduler = new HashMap<ApplicationId, List<Container>>(
-          remoteAppContainersMap.size());
-      for (Entry<ApplicationId, List<Container>> entrySet : remoteAppContainersMap
-          .entrySet()) {
-
-        ApplicationId appId = entrySet.getKey();
-        List<Container> remoteContainerList = entrySet.getValue();
-
-        if (!containersMapForScheduler.containsKey(appId)) {
-          containersMapForScheduler.put(appId, new ArrayList<Container>(
-              remoteContainerList.size()));
-        }
-        List<Container> entryForThisApp = containersMapForScheduler
-            .get(appId);
-
-        for (Container remoteContainer : remoteContainerList) {
-
-          // Process running containers
-          ContainerId containerId = remoteContainer.getId();
-          if (remoteContainer.getState() == ContainerState.RUNNING) {
-            if (!rmNode.justLaunchedContainers.containsKey(containerId)) {
-              // Just launched container. RM knows about it the first time.
-              rmNode.justLaunchedContainers.put(containerId, remoteContainer);
-              entryForThisApp.add(remoteContainer);
-            }
-          } else {
-            // A finished container
-            rmNode.justLaunchedContainers.remove(containerId);
-            entryForThisApp.add(remoteContainer);
+      List<ContainerStatus> newlyLaunchedContainers = 
+          new ArrayList<ContainerStatus>();
+      List<ContainerStatus> completedContainers = 
+          new ArrayList<ContainerStatus>();
+      for (ContainerStatus remoteContainer : statusEvent.getContainers()) {
+        // Process running containers
+        ContainerId containerId = remoteContainer.getContainerId();
+        if (remoteContainer.getState() == ContainerState.RUNNING) {
+          if (!rmNode.justLaunchedContainers.containsKey(containerId)) {
+            // Just launched container. RM knows about it the first time.
+            rmNode.justLaunchedContainers.put(containerId, remoteContainer);
+            newlyLaunchedContainers.add(remoteContainer);
           }
+        } else {
+          // A finished container
+          rmNode.justLaunchedContainers.remove(containerId);
+          completedContainers.add(remoteContainer);
         }
       }
 
       rmNode.context.getDispatcher().getEventHandler().handle(
-          new NodeUpdateSchedulerEvent(rmNode, containersMapForScheduler));
+          new NodeUpdateSchedulerEvent(rmNode, newlyLaunchedContainers, 
+              completedContainers));
 
       return RMNodeState.RUNNING;
     }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java Tue Sep 13 22:49:27 2011
@@ -19,10 +19,8 @@
 package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
 
 import java.util.List;
-import java.util.Map;
 
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
@@ -30,11 +28,11 @@ import org.apache.hadoop.yarn.server.api
 public class RMNodeStatusEvent extends RMNodeEvent {
 
   private final NodeHealthStatus nodeHealthStatus;
-  private Map<ApplicationId, List<Container>> containersCollection;
+  private List<ContainerStatus> containersCollection;
   private final HeartbeatResponse latestResponse;
 
   public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus,
-      Map<ApplicationId, List<Container>> collection,
+      List<ContainerStatus> collection,
       HeartbeatResponse latestResponse) {
     super(nodeId, RMNodeEventType.STATUS_UPDATE);
     this.nodeHealthStatus = nodeHealthStatus;
@@ -46,7 +44,7 @@ public class RMNodeStatusEvent extends R
     return this.nodeHealthStatus;
   }
 
-  public Map<ApplicationId, List<Container>> getContainersCollection() {
+  public List<ContainerStatus> getContainers() {
     return this.containersCollection;
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java Tue Sep 13 22:49:27 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -161,26 +162,21 @@ public class SchedulerApp {
             RMContainerEventType.LAUNCHED));
   }
 
-  public synchronized void killContainers(
-      SchedulerApp application) {
-  }
-
   synchronized public void containerCompleted(RMContainer rmContainer,
-      RMContainerEventType event) {
+      ContainerStatus containerStatus, RMContainerEventType event) {
     
     Container container = rmContainer.getContainer();
     ContainerId containerId = container.getId();
     
     // Inform the container
-    if (event.equals(RMContainerEventType.FINISHED)) {
-      // Have to send diagnostics for finished containers.
-      rmContainer.handle(new RMContainerFinishedEvent(containerId,
-          container.getContainerStatus()));
-    } else {
-      rmContainer.handle(new RMContainerEvent(containerId, event));
-    }
+    rmContainer.handle(
+        new RMContainerFinishedEvent(
+            containerId,
+            containerStatus, 
+            event)
+        );
     LOG.info("Completed container: " + rmContainer.getContainerId() + 
-        " in state: " + rmContainer.getState());
+        " in state: " + rmContainer.getState() + " event:" + event);
     
     // Remove from the list of containers
     liveContainers.remove(rmContainer.getContainerId());
@@ -219,7 +215,8 @@ public class SchedulerApp {
 
     Resources.addTo(currentConsumption, container.getResource());
     if (LOG.isDebugEnabled()) {
-      LOG.debug("allocate: applicationId=" + container.getId().getAppId()
+      LOG.debug("allocate: applicationAttemptId=" 
+          + container.getId().getApplicationAttemptId() 
           + " container=" + container.getId() + " host="
           + container.getNodeId().getHost() + " type=" + type);
     }
@@ -278,10 +275,7 @@ public class SchedulerApp {
   }
 
   synchronized public void resetSchedulingOpportunities(Priority priority) {
-    Integer schedulingOpportunities = 
-        this.schedulingOpportunities.get(priority);
-    schedulingOpportunities = 0;
-    this.schedulingOpportunities.put(priority, schedulingOpportunities);
+    this.schedulingOpportunities.put(priority, Integer.valueOf(0));
   }
 
   synchronized public void addSchedulingOpportunity(Priority priority) {
@@ -305,9 +299,7 @@ public class SchedulerApp {
   }
 
   synchronized void resetReReservations(Priority priority) {
-    Integer reReservations = this.reReservations.get(priority);
-    reReservations = 0;
-    this.reReservations.put(priority, reReservations);
+    this.reReservations.put(priority, Integer.valueOf(0));
   }
 
   synchronized void addReReservation(Priority priority) {

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java Tue Sep 13 22:49:27 2011
@@ -198,8 +198,8 @@ public class SchedulerNode {
       }
       
       // Cannot reserve more than one application on a given node!
-      if (!this.reservedContainer.getContainer().getId().getAppAttemptId().equals(
-          reservedContainer.getContainer().getId().getAppAttemptId())) {
+      if (!this.reservedContainer.getContainer().getId().getApplicationAttemptId().equals(
+          reservedContainer.getContainer().getId().getApplicationAttemptId())) {
         throw new IllegalStateException("Trying to reserve" +
         		" container " + reservedContainer + 
             " for application " + application.getApplicationId() + 
@@ -221,7 +221,7 @@ public class SchedulerNode {
   public synchronized void unreserveResource(SchedulerApp application) {
     // Cannot unreserve for wrong application...
     ApplicationAttemptId reservedApplication = 
-        reservedContainer.getContainer().getId().getAppAttemptId(); 
+        reservedContainer.getContainer().getId().getApplicationAttemptId(); 
     if (!reservedApplication.equals(
         application.getApplicationAttemptId())) {
       throw new IllegalStateException("Trying to unreserve " +  

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Tue Sep 13 22:49:27 2011
@@ -36,10 +36,9 @@ import org.apache.hadoop.security.Access
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.Lock;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -59,11 +58,11 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent;
@@ -80,14 +79,14 @@ implements ResourceScheduler, CapacitySc
 
   private static final Log LOG = LogFactory.getLog(CapacityScheduler.class);
 
-  private Queue root;
+  private CSQueue root;
 
   private final static List<Container> EMPTY_CONTAINER_LIST = 
     new ArrayList<Container>();
 
-  static final Comparator<Queue> queueComparator = new Comparator<Queue>() {
+  static final Comparator<CSQueue> queueComparator = new Comparator<CSQueue>() {
     @Override
-    public int compare(Queue q1, Queue q2) {
+    public int compare(CSQueue q1, CSQueue q2) {
       if (q1.getUtilization() < q2.getUtilization()) {
         return -1;
       } else if (q1.getUtilization() > q2.getUtilization()) {
@@ -110,7 +109,7 @@ implements ResourceScheduler, CapacitySc
   private ContainerTokenSecretManager containerTokenSecretManager;
   private RMContext rmContext;
 
-  private Map<String, Queue> queues = new ConcurrentHashMap<String, Queue>();
+  private Map<String, CSQueue> queues = new ConcurrentHashMap<String, CSQueue>();
 
   private Map<NodeId, SchedulerNode> nodes = 
       new ConcurrentHashMap<NodeId, SchedulerNode>();
@@ -127,7 +126,9 @@ implements ResourceScheduler, CapacitySc
 
   private boolean initialized = false;
 
-  public Queue getRootQueue() {
+  public CapacityScheduler() {}
+
+  public CSQueue getRootQueue() {
     return root;
   }
 
@@ -207,7 +208,7 @@ implements ResourceScheduler, CapacitySc
     CapacitySchedulerConfiguration.PREFIX + ROOT;
 
   static class QueueHook {
-    public Queue hook(Queue queue) {
+    public CSQueue hook(CSQueue queue) {
       return queue;
     }
   }
@@ -225,8 +226,8 @@ implements ResourceScheduler, CapacitySc
   private void reinitializeQueues(CapacitySchedulerConfiguration conf) 
   throws IOException {
     // Parse new queues
-    Map<String, Queue> newQueues = new HashMap<String, Queue>();
-    Queue newRoot = 
+    Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
+    CSQueue newRoot = 
         parseQueue(this, conf, null, ROOT, newQueues, queues, 
             queueComparator, applicationComparator, noop);
     
@@ -247,7 +248,7 @@ implements ResourceScheduler, CapacitySc
    */
   @Lock(CapacityScheduler.class)
   private void validateExistingQueues(
-      Map<String, Queue> queues, Map<String, Queue> newQueues) 
+      Map<String, CSQueue> queues, Map<String, CSQueue> newQueues) 
   throws IOException {
     for (String queue : queues.keySet()) {
       if (!newQueues.containsKey(queue)) {
@@ -264,11 +265,11 @@ implements ResourceScheduler, CapacitySc
    */
   @Lock(CapacityScheduler.class)
   private void addNewQueues(
-      Map<String, Queue> queues, Map<String, Queue> newQueues) 
+      Map<String, CSQueue> queues, Map<String, CSQueue> newQueues) 
   {
-    for (Map.Entry<String, Queue> e : newQueues.entrySet()) {
+    for (Map.Entry<String, CSQueue> e : newQueues.entrySet()) {
       String queueName = e.getKey();
-      Queue queue = e.getValue();
+      CSQueue queue = e.getValue();
       if (!queues.containsKey(queueName)) {
         queues.put(queueName, queue);
       }
@@ -276,15 +277,15 @@ implements ResourceScheduler, CapacitySc
   }
   
   @Lock(CapacityScheduler.class)
-  static Queue parseQueue(
+  static CSQueue parseQueue(
       CapacitySchedulerContext csContext, 
       CapacitySchedulerConfiguration conf, 
-      Queue parent, String queueName, Map<String, Queue> queues,
-      Map<String, Queue> oldQueues, 
-      Comparator<Queue> queueComparator,
+      CSQueue parent, String queueName, Map<String, CSQueue> queues,
+      Map<String, CSQueue> oldQueues, 
+      Comparator<CSQueue> queueComparator,
       Comparator<SchedulerApp> applicationComparator,
       QueueHook hook) {
-    Queue queue;
+    CSQueue queue;
     String[] childQueueNames = 
       conf.getQueues((parent == null) ? 
           queueName : (parent.getQueuePath()+"."+queueName));
@@ -306,9 +307,9 @@ implements ResourceScheduler, CapacitySc
       // Used only for unit tests
       queue = hook.hook(parentQueue);
       
-      List<Queue> childQueues = new ArrayList<Queue>();
+      List<CSQueue> childQueues = new ArrayList<CSQueue>();
       for (String childQueueName : childQueueNames) {
-        Queue childQueue = 
+        CSQueue childQueue = 
           parseQueue(csContext, conf, queue, childQueueName, 
               queues, oldQueues, queueComparator, applicationComparator, hook);
         childQueues.add(childQueue);
@@ -322,7 +323,7 @@ implements ResourceScheduler, CapacitySc
     return queue;
   }
 
-  synchronized Queue getQueue(String queueName) {
+  synchronized CSQueue getQueue(String queueName) {
     return queues.get(queueName);
   }
   
@@ -331,7 +332,7 @@ implements ResourceScheduler, CapacitySc
           String queueName, String user) {
 
     // Sanity checks
-    Queue queue = getQueue(queueName);
+    CSQueue queue = getQueue(queueName);
     if (queue == null) {
       String message = "Application " + applicationAttemptId + 
       " submitted by user " + user + " to unknown queue: " + queueName;
@@ -392,12 +393,20 @@ implements ResourceScheduler, CapacitySc
     
     // Release all the running containers 
     for (RMContainer rmContainer : application.getLiveContainers()) {
-      completedContainer(rmContainer, RMContainerEventType.KILL);
+      completedContainer(rmContainer, 
+          SchedulerUtils.createAbnormalContainerStatus(
+              rmContainer.getContainerId(), 
+              SchedulerUtils.COMPLETED_APPLICATION), 
+          RMContainerEventType.KILL);
     }
     
      // Release all reserved containers
     for (RMContainer rmContainer : application.getAllReservedContainers()) {
-      completedContainer(rmContainer, RMContainerEventType.KILL);
+      completedContainer(rmContainer, 
+          SchedulerUtils.createAbnormalContainerStatus(
+              rmContainer.getContainerId(), 
+              "Application Complete"), 
+          RMContainerEventType.KILL);
     }
     
     // Clean up pending requests, metrics etc.
@@ -405,7 +414,7 @@ implements ResourceScheduler, CapacitySc
     
     // Inform the queue
     String queueName = application.getQueue().getQueueName();
-    Queue queue = queues.get(queueName);
+    CSQueue queue = queues.get(queueName);
     if (!(queue instanceof LeafQueue)) {
       LOG.error("Cannot finish application " + "from non-leaf queue: "
           + queueName);
@@ -445,7 +454,11 @@ implements ResourceScheduler, CapacitySc
              "Trying to release container not owned by app or with invalid id",
              application.getApplicationId(), releasedContainerId);
       }
-      completedContainer(rmContainer, RMContainerEventType.RELEASED);
+      completedContainer(rmContainer,
+          SchedulerUtils.createAbnormalContainerStatus(
+              releasedContainerId, 
+              SchedulerUtils.RELEASED_CONTAINER),
+          RMContainerEventType.RELEASED);
     }
 
     synchronized (application) {
@@ -479,7 +492,7 @@ implements ResourceScheduler, CapacitySc
   public QueueInfo getQueueInfo(String queueName, 
       boolean includeChildQueues, boolean recursive) 
   throws IOException {
-    Queue queue = null;
+    CSQueue queue = null;
 
     synchronized (this) {
       queue = this.queues.get(queueName); 
@@ -521,22 +534,23 @@ implements ResourceScheduler, CapacitySc
   }
 
   private synchronized void nodeUpdate(RMNode nm, 
-      Map<ApplicationId, List<Container>> containers ) {
+      List<ContainerStatus> newlyLaunchedContainers,
+      List<ContainerStatus> completedContainers) {
     LOG.info("nodeUpdate: " + nm + " clusterResources: " + clusterResource);
     
     SchedulerNode node = getNode(nm.getNodeID());
 
-    // Processing the current containers running/finished on node
-    for (List<Container> appContainers : containers.values()) {
-      for (Container container : appContainers) {
-        if (container.getState() == ContainerState.RUNNING) {
-          containerLaunchedOnNode(container, node);
-        } else { // has to be 'COMPLETE'
-          LOG.info("DEBUG --- Container FINISHED: " + container.getId());
-          completedContainer(getRMContainer(container.getId()), 
-              RMContainerEventType.FINISHED);
-        }
-      }
+    // Processing the newly launched containers
+    for (ContainerStatus launchedContainer : newlyLaunchedContainers) {
+      containerLaunchedOnNode(launchedContainer.getContainerId(), node);
+    }
+
+    // Process completed containers
+    for (ContainerStatus completedContainer : completedContainers) {
+      ContainerId containerId = completedContainer.getContainerId();
+      LOG.info("DEBUG --- Container FINISHED: " + containerId);
+      completedContainer(getRMContainer(containerId), 
+          completedContainer, RMContainerEventType.FINISHED);
     }
 
     // Now node data structures are upto date and ready for scheduling.
@@ -566,23 +580,24 @@ implements ResourceScheduler, CapacitySc
     } else {
       LOG.info("Skipping scheduling since node " + nm + 
           " is reserved by application " + 
-          node.getReservedContainer().getContainerId().getAppId());
+          node.getReservedContainer().getContainerId().getApplicationAttemptId()
+          );
     }
 
   }
 
-  private void containerLaunchedOnNode(Container container, SchedulerNode node) {
+  private void containerLaunchedOnNode(ContainerId containerId, SchedulerNode node) {
     // Get the application for the finished container
-    ApplicationAttemptId applicationAttemptId = container.getId().getAppAttemptId();
+    ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
     SchedulerApp application = getApplication(applicationAttemptId);
     if (application == null) {
       LOG.info("Unknown application: " + applicationAttemptId + 
-          " launched container " + container.getId() +
+          " launched container " + containerId +
           " on node: " + node);
       return;
     }
     
-    application.containerLaunchedOnNode(container.getId());
+    application.containerLaunchedOnNode(containerId);
   }
 
   @Override
@@ -604,7 +619,8 @@ implements ResourceScheduler, CapacitySc
     {
       NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event;
       nodeUpdate(nodeUpdatedEvent.getRMNode(), 
-          nodeUpdatedEvent.getContainers());
+          nodeUpdatedEvent.getNewlyLaunchedContainers(),
+          nodeUpdatedEvent.getCompletedContainers());
     }
     break;
     case APP_ADDED:
@@ -625,7 +641,11 @@ implements ResourceScheduler, CapacitySc
     {
       ContainerExpiredSchedulerEvent containerExpiredEvent = 
           (ContainerExpiredSchedulerEvent) event;
-      completedContainer(getRMContainer(containerExpiredEvent.getContainerId()), 
+      ContainerId containerId = containerExpiredEvent.getContainerId();
+      completedContainer(getRMContainer(containerId), 
+          SchedulerUtils.createAbnormalContainerStatus(
+              containerId, 
+              SchedulerUtils.EXPIRED_CONTAINER), 
           RMContainerEventType.EXPIRE);
     }
     break;
@@ -652,13 +672,21 @@ implements ResourceScheduler, CapacitySc
     // Remove running containers
     List<RMContainer> runningContainers = node.getRunningContainers();
     for (RMContainer container : runningContainers) {
-      completedContainer(container, RMContainerEventType.KILL);
+      completedContainer(container, 
+          SchedulerUtils.createAbnormalContainerStatus(
+              container.getContainerId(), 
+              SchedulerUtils.LOST_CONTAINER), 
+          RMContainerEventType.KILL);
     }
     
     // Remove reservations, if any
     RMContainer reservedContainer = node.getReservedContainer();
     if (reservedContainer != null) {
-      completedContainer(reservedContainer, RMContainerEventType.KILL);
+      completedContainer(reservedContainer, 
+          SchedulerUtils.createAbnormalContainerStatus(
+              reservedContainer.getContainerId(), 
+              SchedulerUtils.LOST_CONTAINER), 
+          RMContainerEventType.KILL);
     }
 
     this.nodes.remove(nodeInfo.getNodeID());
@@ -667,8 +695,8 @@ implements ResourceScheduler, CapacitySc
   }
   
   @Lock(CapacityScheduler.class)
-  private synchronized void completedContainer(RMContainer rmContainer, 
-      RMContainerEventType event) {
+  private synchronized void completedContainer(RMContainer rmContainer,
+      ContainerStatus containerStatus, RMContainerEventType event) {
     if (rmContainer == null) {
       LOG.info("Null container completed...");
       return;
@@ -677,7 +705,7 @@ implements ResourceScheduler, CapacitySc
     Container container = rmContainer.getContainer();
     
     // Get the application for the finished container
-    ApplicationAttemptId applicationAttemptId = container.getId().getAppAttemptId();
+    ApplicationAttemptId applicationAttemptId = container.getId().getApplicationAttemptId();
     SchedulerApp application = getApplication(applicationAttemptId);
     if (application == null) {
       LOG.info("Container " + container + " of" +
@@ -692,7 +720,7 @@ implements ResourceScheduler, CapacitySc
     // Inform the queue
     LeafQueue queue = (LeafQueue)application.getQueue();
     queue.completedContainer(clusterResource, application, node, 
-        rmContainer, event);
+        rmContainer, containerStatus, event);
 
     LOG.info("Application " + applicationAttemptId + 
         " released container " + container.getId() +
@@ -712,7 +740,7 @@ implements ResourceScheduler, CapacitySc
 
   private RMContainer getRMContainer(ContainerId containerId) {
     SchedulerApp application = 
-        getApplication(containerId.getAppAttemptId());
+        getApplication(containerId.getApplicationAttemptId());
     return (application == null) ? null : application.getRMContainer(containerId);
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Tue Sep 13 22:49:27 2011
@@ -40,7 +40,7 @@ public class CapacitySchedulerConfigurat
   private static final String CS_CONFIGURATION_FILE = "capacity-scheduler.xml";
   
   @Private
-  public static final String PREFIX = "yarn.capacity-scheduler.";
+  public static final String PREFIX = "yarn.scheduler.capacity.";
   
   @Private
   public static final String DOT = ".";

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Tue Sep 13 22:49:27 2011
@@ -39,6 +39,7 @@ import org.apache.hadoop.security.UserGr
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.ContainerToken;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -59,16 +60,17 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 
 @Private
 @Unstable
-public class LeafQueue implements Queue {
+public class LeafQueue implements CSQueue {
   private static final Log LOG = LogFactory.getLog(LeafQueue.class);
 
   private final String queueName;
-  private Queue parent;
+  private CSQueue parent;
   private float capacity;
   private float absoluteCapacity;
   private float maximumCapacity;
@@ -119,8 +121,8 @@ public class LeafQueue implements Queue 
   final static int DEFAULT_AM_RESOURCE = 2 * 1024;
   
   public LeafQueue(CapacitySchedulerContext cs, 
-      String queueName, Queue parent, 
-      Comparator<SchedulerApp> applicationComparator, Queue old) {
+      String queueName, CSQueue parent, 
+      Comparator<SchedulerApp> applicationComparator, CSQueue old) {
     this.scheduler = cs;
     this.queueName = queueName;
     this.parent = parent;
@@ -192,7 +194,7 @@ public class LeafQueue implements Queue 
       float maxAMResourcePercent, float absoluteCapacity) {
     return 
         Math.max(
-            (int)((clusterResource.getMemory() / DEFAULT_AM_RESOURCE) * 
+            (int)((clusterResource.getMemory() / (float)DEFAULT_AM_RESOURCE) * 
                    maxAMResourcePercent * absoluteCapacity), 
             1);
   }
@@ -271,7 +273,7 @@ public class LeafQueue implements Queue 
   }
 
   @Override
-  public Queue getParent() {
+  public CSQueue getParent() {
     return parent;
   }
 
@@ -313,15 +315,15 @@ public class LeafQueue implements Queue 
     return maxApplications;
   }
 
-  public int getMaxApplicationsPerUser() {
+  public synchronized int getMaxApplicationsPerUser() {
     return maxApplicationsPerUser;
   }
 
-  public int getMaximumActiveApplications() {
+  public synchronized int getMaximumActiveApplications() {
     return maxActiveApplications;
   }
 
-  public int getMaximumActiveApplicationsPerUser() {
+  public synchronized int getMaximumActiveApplicationsPerUser() {
     return maxActiveApplicationsPerUser;
   }
 
@@ -341,7 +343,7 @@ public class LeafQueue implements Queue 
   }
 
   @Override
-  public List<Queue> getChildQueues() {
+  public List<CSQueue> getChildQueues() {
     return null;
   }
 
@@ -381,7 +383,7 @@ public class LeafQueue implements Queue 
     this.userLimitFactor = userLimitFactor;
   }
 
-  synchronized void setParentQueue(Queue parent) {
+  synchronized void setParentQueue(CSQueue parent) {
     this.parent = parent;
   }
   
@@ -423,12 +425,12 @@ public class LeafQueue implements Queue 
   }
 
   @Private
-  public int getUserLimit() {
+  public synchronized int getUserLimit() {
     return userLimit;
   }
 
   @Private
-  public float getUserLimitFactor() {
+  public synchronized float getUserLimitFactor() {
     return userLimitFactor;
   }
 
@@ -480,7 +482,7 @@ public class LeafQueue implements Queue 
   }
 
   @Override
-  public synchronized void reinitialize(Queue queue, Resource clusterResource) 
+  public synchronized void reinitialize(CSQueue queue, Resource clusterResource) 
   throws IOException {
     // Sanity check
     if (!(queue instanceof LeafQueue) || 
@@ -493,9 +495,10 @@ public class LeafQueue implements Queue 
     setupQueueConfigs(leafQueue.capacity, leafQueue.absoluteCapacity, 
         leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity, 
         leafQueue.userLimit, leafQueue.userLimitFactor, 
-        leafQueue.maxApplications, leafQueue.maxApplicationsPerUser,
-        leafQueue.maxActiveApplications, 
-        leafQueue.maxActiveApplicationsPerUser,
+        leafQueue.maxApplications,
+        leafQueue.getMaxApplicationsPerUser(),
+        leafQueue.getMaximumActiveApplications(), 
+        leafQueue.getMaximumActiveApplicationsPerUser(),
         leafQueue.state, leafQueue.acls);
     
     updateResource(clusterResource);
@@ -761,7 +764,11 @@ public class LeafQueue implements Queue 
       // Release
       Container container = rmContainer.getContainer();
       completedContainer(clusterResource, application, node, 
-          rmContainer, RMContainerEventType.RELEASED);
+          rmContainer, 
+          SchedulerUtils.createAbnormalContainerStatus(
+              container.getId(), 
+              SchedulerUtils.UNRESERVED_CONTAINER), 
+          RMContainerEventType.RELEASED);
       return container.getResource();
     }
 
@@ -900,7 +907,7 @@ public class LeafQueue implements Queue 
       // Protect against corner case where you need the whole node with
       // Math.min(nodeFactor, minimumAllocationFactor)
       starvation = 
-          (int)((application.getReReservations(priority) / reservedContainers) * 
+          (int)((application.getReReservations(priority) / (float)reservedContainers) * 
                 (1.0f - (Math.min(nodeFactor, getMinimumAllocationFactor())))
                );
       
@@ -1174,7 +1181,7 @@ public class LeafQueue implements Queue 
   @Override
   public void completedContainer(Resource clusterResource, 
       SchedulerApp application, SchedulerNode node, RMContainer rmContainer, 
-      RMContainerEventType event) {
+      ContainerStatus containerStatus, RMContainerEventType event) {
     if (application != null) {
       // Careful! Locking order is important!
       synchronized (this) {
@@ -1189,7 +1196,7 @@ public class LeafQueue implements Queue 
           application.unreserve(node, rmContainer.getReservedPriority());
           node.unreserveResource(application);
         } else {
-          application.containerCompleted(rmContainer, event);
+          application.containerCompleted(rmContainer, containerStatus, event);
           node.releaseContainer(container);
         }
 
@@ -1209,7 +1216,7 @@ public class LeafQueue implements Queue 
 
       // Inform the parent queue
       parent.completedContainer(clusterResource, application, 
-          node, rmContainer, event);
+          node, rmContainer, null, event);
     }
   }
 



Mime
View raw message