hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1084143 [1/2] - in /hadoop/mapreduce/branches/MR-279/yarn/yarn-server: yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/contain...
Date Tue, 22 Mar 2011 11:59:34 GMT
Author: vinodkv
Date: Tue Mar 22 11:59:33 2011
New Revision: 1084143

URL: http://svn.apache.org/viewvc?rev=1084143&view=rev
Log:
Implementation of cleanup of container and application level file resources on the NodeManager. Contributed by Vinod Kumar Vavilapalli.

Added:
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedAppsEvent.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEvent.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/resources/
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/resources/krb5.conf
Modified:
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationState.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEvent.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/AppLocalizationRunnerImpl.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java

Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedAppsEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedAppsEvent.java?rev=1084143&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedAppsEvent.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedAppsEvent.java Tue Mar 22 11:59:33 2011
@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import java.util.List;
+
+import org.apache.hadoop.yarn.ApplicationID;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerManagerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerManagerEventType;
+
+public class CMgrCompletedAppsEvent extends ContainerManagerEvent {
+
+  private List<ApplicationID> appsToCleanup;
+
+  public CMgrCompletedAppsEvent(List<ApplicationID> appsToCleanup) {
+    super(ContainerManagerEventType.FINISH_APPS);
+    this.appsToCleanup = appsToCleanup;
+  }
+
+  public List<ApplicationID> getAppsToCleanup() {
+    return this.appsToCleanup;
+  }
+}

Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java?rev=1084143&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java Tue Mar 22 11:59:33 2011
@@ -0,0 +1,37 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import java.util.List;
+
+import org.apache.hadoop.yarn.Container;
+
+public class CMgrCompletedContainersEvent extends ContainerManagerEvent {
+
+  private List<Container> containerToCleanup;
+
+  public CMgrCompletedContainersEvent(List<Container> containersToCleanup) {
+    super(ContainerManagerEventType.FINISH_CONTAINERS);
+    this.containerToCleanup = containersToCleanup;
+  }
+
+  public List<Container> getContainersToCleanup() {
+    return this.containerToCleanup;
+  }
+}

Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEvent.java?rev=1084143&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEvent.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEvent.java Tue Mar 22 11:59:33 2011
@@ -0,0 +1,29 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class ContainerManagerEvent extends
+    AbstractEvent<ContainerManagerEventType> {
+
+  public ContainerManagerEvent(ContainerManagerEventType type) {
+    super(type);
+  }
+}

Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java?rev=1084143&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java Tue Mar 22 11:59:33 2011
@@ -0,0 +1,24 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+public enum ContainerManagerEventType {
+  FINISH_APPS,
+  FINISH_CONTAINERS
+}

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Tue Mar 22 11:59:33 2011
@@ -46,7 +46,7 @@ public class DefaultContainerExecutor ex
 
   private final FileContext lfs;
 
-  DefaultContainerExecutor() {
+  public DefaultContainerExecutor() {
     try {
       this.lfs = FileContext.getLocalFSFileContext();
     } catch (UnsupportedFileSystemException e) {

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Tue Mar 22 11:59:33 2011
@@ -58,7 +58,7 @@ public class LinuxContainerExecutor exte
   enum Commands {
     INITIALIZE_JOB(0),
     LAUNCH_CONTAINER(1),
-    SIGNAL_TASK(2),
+    SIGNAL_CONTAINER(2),
     DELETE_AS_USER(3),
     DELETE_LOG_AS_USER(4);
 
@@ -180,9 +180,6 @@ public class LinuxContainerExecutor exte
         logOutput(shExec.getOutput());
       }
     } catch (ExitCodeException e) {
-      if (shExec == null) {
-        return -1;
-      }
       int exitCode = shExec.getExitCode();
       LOG.warn("Exit code from task is : " + exitCode);
       // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the task was
@@ -197,7 +194,7 @@ public class LinuxContainerExecutor exte
       launchCommandObjs.remove(container.getLaunchContext().id);
     }
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Output from LinuxTaskController's launchTask follows:");
+      LOG.debug("Output from LinuxContainerExecutor's launchTask follows:");
       logOutput(shExec.getOutput());
     }
     return 0;
@@ -210,7 +207,7 @@ public class LinuxContainerExecutor exte
     String[] command =
         new String[] { containerExecutorExe,
                    user,
-                   Integer.toString(Commands.SIGNAL_TASK.getValue()),
+                   Integer.toString(Commands.SIGNAL_CONTAINER.getValue()),
                    pid,
                    Integer.toString(signal.getValue()) };
     ShellCommandExecutor shExec = new ShellCommandExecutor(command);
@@ -232,8 +229,47 @@ public class LinuxContainerExecutor exte
   }
 
   @Override
-  public void deleteAsUser(String user, Path subDir, Path... basedirs)
+  public void deleteAsUser(String user, Path subDir, Path... baseDirs)
       throws IOException, InterruptedException {
+
+    if (baseDirs == null || baseDirs.length == 0) {
+      LOG.info("Deleting absolute path : " + subDir);
+      deleteAsUser(user, subDir);
+      return;
+    }
+    for (Path baseDir : baseDirs) {
+      Path del = new Path(baseDir, subDir);
+      LOG.info("Deleting path : " + del);
+      deleteAsUser(user, del);
+    }
   }
 
+  private void deleteAsUser(String user, Path dir) {
+    List<String> command = new ArrayList<String>(
+        Arrays.asList(containerExecutorExe,
+                    user,
+                    Integer.toString(Commands.DELETE_AS_USER.getValue()),
+                    dir.toUri().getPath()));
+    String[] commandArray = command.toArray(new String[command.size()]);
+    ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
+    LOG.info(" -- DEBUG -- deleteAsUser: " + Arrays.toString(commandArray));
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("deleteAsUser: " + Arrays.toString(commandArray));
+    }
+    try {
+      shExec.execute();
+      if (LOG.isDebugEnabled()) {
+        logOutput(shExec.getOutput());
+      }
+    } catch (IOException e) {
+      int exitCode = shExec.getExitCode();
+      LOG.warn("Exit code from task is : " + exitCode);
+      if (exitCode != 0) {
+        LOG.error("DeleteAsUser for " + dir.toUri().getPath()
+            + " returned with non-zero exit code" + exitCode);
+        LOG.error("Output from LinuxContainerExecutor's deleteAsUser follows:");
+        logOutput(shExec.getOutput());
+      }
+    }
+  }
 }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Tue Mar 22 11:59:33 2011
@@ -18,33 +18,31 @@
 
 package org.apache.hadoop.yarn.server.nodemanager;
 
-import java.io.IOException;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_CONTAINER_EXECUTOR_CLASS;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_KEYTAB;
 
+import java.io.IOException;
 import java.util.Comparator;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.ApplicationID;
+import org.apache.hadoop.yarn.ContainerID;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.server.YarnServerConfig;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.service.CompositeService;
 import org.apache.hadoop.yarn.service.Service;
 
-
-import org.apache.hadoop.yarn.ApplicationID;
-import org.apache.hadoop.yarn.ContainerID;
-
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.*;
-
 public class NodeManager extends CompositeService {
 
   public NodeManager() {
@@ -59,32 +57,39 @@ public class NodeManager extends Composi
     DeletionService del = new DeletionService(exec);
     addService(del);
 
+    // NodeManager level dispatcher
+    Dispatcher dispatcher = new AsyncDispatcher();
+
     // StatusUpdater should be added first so that it can start first. Once it
     // contacts RM, does registration and gets tokens, then only
     // ContainerManager can start.
-    NodeStatusUpdater nodeStatusUpdater = createNodeStatusUpdater(context);
+    NodeStatusUpdater nodeStatusUpdater =
+        createNodeStatusUpdater(context, dispatcher);
     addService(nodeStatusUpdater);
 
     NodeResourceMonitor nodeResourceMonitor = createNodeResourceMonitor();
     addService(nodeResourceMonitor);
 
-    Service containerManager =
+    ContainerManagerImpl containerManager =
         createContainerManager(context, exec, del, nodeStatusUpdater);
     addService(containerManager);
 
     Service webServer = createWebServer();
     addService(webServer);
+
+    dispatcher.register(ContainerManagerEventType.class, containerManager);
   }
 
-  protected NodeStatusUpdater createNodeStatusUpdater(Context context) {
-    return new NodeStatusUpdaterImpl(context);
+  protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+      Dispatcher dispatcher) {
+    return new NodeStatusUpdaterImpl(context, dispatcher);
   }
 
   protected NodeResourceMonitor createNodeResourceMonitor() {
     return new NodeResourceMonitorImpl();
   }
 
-  protected Service createContainerManager(Context context,
+  protected ContainerManagerImpl createContainerManager(Context context,
       ContainerExecutor exec, DeletionService del,
       NodeStatusUpdater nodeStatusUpdater) {
     return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater);

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Tue Mar 22 11:59:33 2011
@@ -35,11 +35,7 @@ import org.apache.hadoop.fs.CommonConfig
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityInfo;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.server.RMNMSecurityInfoClass;
-import org.apache.hadoop.yarn.server.YarnServerConfig;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.hadoop.yarn.ApplicationID;
 import org.apache.hadoop.yarn.ContainerID;
 import org.apache.hadoop.yarn.ContainerState;
 import org.apache.hadoop.yarn.HeartbeatResponse;
@@ -48,6 +44,12 @@ import org.apache.hadoop.yarn.NodeStatus
 import org.apache.hadoop.yarn.RegistrationResponse;
 import org.apache.hadoop.yarn.Resource;
 import org.apache.hadoop.yarn.ResourceTracker;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.RMNMSecurityInfoClass;
+import org.apache.hadoop.yarn.server.YarnServerConfig;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.service.AbstractService;
 
 public class NodeStatusUpdaterImpl extends AbstractService implements
     NodeStatusUpdater {
@@ -57,6 +59,8 @@ public class NodeStatusUpdaterImpl exten
   private final Object heartbeatMonitor = new Object();
 
   private Context context;
+  private Dispatcher dispatcher;
+
   private long heartBeatInterval;
   private ResourceTracker resourceTracker;
   private String rmAddress;
@@ -66,9 +70,10 @@ public class NodeStatusUpdaterImpl exten
   private byte[] secretKeyBytes = new byte[0];
   private boolean isStopped;
 
-  public NodeStatusUpdaterImpl(Context context) {
+  public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher) {
     super(NodeStatusUpdaterImpl.class.getName());
     this.context = context;
+    this.dispatcher = dispatcher;
   }
 
   @Override
@@ -173,7 +178,7 @@ public class NodeStatusUpdaterImpl exten
         }
 
         // Clone the container to send it to the RM
-        org.apache.hadoop.yarn.Container c = container.getContainer();
+        org.apache.hadoop.yarn.Container c = container.cloneAndGetContainer();
         c.hostName = this.nodeName;
         applicationContainers.add(c);
         ++numActiveContainers;
@@ -219,6 +224,18 @@ public class NodeStatusUpdaterImpl exten
             HeartbeatResponse response =
               resourceTracker.nodeHeartbeat(nodeStatus);
             lastHeartBeatID = response.responseId;
+            List<org.apache.hadoop.yarn.Container> containersToCleanup =
+                response.containersToCleanup;
+            if (containersToCleanup.size() != 0) {
+              dispatcher.getEventHandler().handle(
+                  new CMgrCompletedContainersEvent(containersToCleanup));
+            }
+            List<ApplicationID> appsToCleanup =
+                response.appplicationsToCleanup;
+            if (appsToCleanup.size() != 0) {
+              dispatcher.getEventHandler().handle(
+                  new CMgrCompletedAppsEvent(appsToCleanup));
+            }
           } catch (AvroRemoteException e) {
             LOG.error("Caught exception in status-updater", e);
             break;

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Tue Mar 22 11:59:33 2011
@@ -33,13 +33,21 @@ import org.apache.hadoop.fs.CommonConfig
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityInfo;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.ApplicationID;
+import org.apache.hadoop.yarn.ContainerID;
+import org.apache.hadoop.yarn.ContainerLaunchContext;
+import org.apache.hadoop.yarn.ContainerManager;
+import org.apache.hadoop.yarn.ContainerStatus;
+import org.apache.hadoop.yarn.YarnRemoteException;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.ipc.RPCUtil;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo;
+import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
+import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedContainersEvent;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerManagerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
 import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
@@ -62,15 +70,10 @@ import org.apache.hadoop.yarn.server.sec
 import org.apache.hadoop.yarn.service.CompositeService;
 import org.apache.hadoop.yarn.service.Service;
 import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
-import org.apache.hadoop.yarn.ApplicationID;
-import org.apache.hadoop.yarn.ContainerID;
-import org.apache.hadoop.yarn.ContainerLaunchContext;
-import org.apache.hadoop.yarn.ContainerManager;
-import org.apache.hadoop.yarn.ContainerStatus;
-import org.apache.hadoop.yarn.YarnRemoteException;
 
 public class ContainerManagerImpl extends CompositeService implements
-    ServiceStateChangeListener, ContainerManager, EventHandler<Event> {
+    ServiceStateChangeListener, ContainerManager,
+    EventHandler<ContainerManagerEvent> {
 
   private static final Log LOG = LogFactory.getLog(ContainerManagerImpl.class);
 
@@ -86,11 +89,14 @@ public class ContainerManagerImpl extend
 
   protected AsyncDispatcher dispatcher;
 
+  private DeletionService deletionService;
+
   public ContainerManagerImpl(Context context, ContainerExecutor exec,
       DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater) {
     super(ContainerManagerImpl.class.getName());
     this.context = context;
     dispatcher = new AsyncDispatcher();
+    this.deletionService = deletionContext;
 
     rsrcLocalizationSrvc =
         createResourceLocalizationService(exec, deletionContext);
@@ -139,7 +145,6 @@ public class ContainerManagerImpl extend
 
   @Override
   public void init(Configuration conf) {
-    // TODO enqueue user dirs in deletion context
     cmAddr = NetUtils.createSocketAddr(
         conf.get(NM_BIND_ADDRESS, DEFAULT_NM_BIND_ADDRESS));
     Configuration cmConf = new Configuration(conf);
@@ -150,6 +155,9 @@ public class ContainerManagerImpl extend
 
   @Override
   public void start() {
+
+    // Enqueue user dirs in deletion context
+
     YarnRPC rpc = YarnRPC.create(getConfig());
     if (UserGroupInformation.isSecurityEnabled()) {
       // This is fine as status updater is started before ContainerManager and
@@ -210,8 +218,7 @@ public class ContainerManagerImpl extend
     }
 
     // TODO: Validate the request
-    dispatcher.getEventHandler().handle(
-        new ApplicationInitEvent(containerID));
+    dispatcher.getEventHandler().handle(new ApplicationInitEvent(container));
     return null;
   }
 
@@ -241,7 +248,7 @@ public class ContainerManagerImpl extend
     LOG.info("Getting container-status for " + containerID);
     Container container = this.context.getContainers().get(containerID);
     if (container != null) {
-      ContainerStatus containerStatus = container.getContainerStatus();
+      ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
       LOG.info("Returning " + containerStatus);
       return containerStatus;
     } else {
@@ -283,8 +290,30 @@ public class ContainerManagerImpl extend
   }
 
   @Override
-  public void handle(Event event) {
-    dispatcher.getEventHandler().handle(event);
+  public void handle(ContainerManagerEvent event) {
+    switch (event.getType()) {
+    case FINISH_APPS:
+      CMgrCompletedAppsEvent appsFinishedEvent =
+          (CMgrCompletedAppsEvent) event;
+      for (ApplicationID appID : appsFinishedEvent.getAppsToCleanup()) {
+        this.dispatcher.getEventHandler().handle(
+            new ApplicationEvent(appID,
+                ApplicationEventType.FINISH_APPLICATION));
+      }
+      break;
+    case FINISH_CONTAINERS:
+      CMgrCompletedContainersEvent containersFinishedEvent =
+          (CMgrCompletedContainersEvent) event;
+      for (org.apache.hadoop.yarn.Container container : containersFinishedEvent
+          .getContainersToCleanup()) {
+        this.dispatcher.getEventHandler().handle(
+            new ContainerEvent(container.id,
+                ContainerEventType.KILL_CONTAINER));
+      }
+      break;
+    default:
+      LOG.warn("Invalid event " + event.getType() + ". Ignoring.");
+    }
   }
 
   @Override

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java Tue Mar 22 11:59:33 2011
@@ -42,5 +42,7 @@ public interface Application extends Eve
 
   ApplicationID getAppId();
 
+  ApplicationState getApplicationState();
+
   Credentials getCredentials() throws IOException;
 }

Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java?rev=1084143&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java Tue Mar 22 11:59:33 2011
@@ -0,0 +1,36 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.application;
+
+import org.apache.hadoop.yarn.ContainerID;
+
+public class ApplicationContainerFinishedEvent extends ApplicationEvent {
+  private ContainerID containerID;
+
+  public ApplicationContainerFinishedEvent(
+      ContainerID containerID) {
+    super(containerID.appID,
+        ApplicationEventType.APPLICATION_CONTAINER_FINISHED);
+    this.containerID = containerID;
+  }
+
+  public ContainerID getContainerID() {
+    return this.containerID;
+  }
+}

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java Tue Mar 22 11:59:33 2011
@@ -23,5 +23,7 @@ public enum ApplicationEventType {
   FINISH_APPLICATION,
 
   APPLICATION_INITED,
+  APPLICATION_CONTAINER_FINISHED,
+  APPLICATION_RESOURCES_CLEANEDUP,
   APPLICATION_FINISHED,
 }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Tue Mar 22 11:59:33 2011
@@ -20,59 +20,59 @@ package org.apache.hadoop.yarn.server.no
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.text.NumberFormat;
-import java.util.ArrayList;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
-import java.util.List;
+import java.util.Iterator;
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputByteBuffer;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.ApplicationID;
+import org.apache.hadoop.yarn.ContainerID;
+import org.apache.hadoop.yarn.LocalResource;
+import org.apache.hadoop.yarn.LocalResourceVisibility;
 import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType;
 import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.hadoop.yarn.util.AvroUtil;
 
-import org.apache.hadoop.yarn.ApplicationID;
-import org.apache.hadoop.yarn.ContainerID;
-import org.apache.hadoop.yarn.LocalResource;
-import org.apache.hadoop.yarn.LocalResourceVisibility;
-
 public class ApplicationImpl implements Application {
 
   final Dispatcher dispatcher;
   final String user;
   final ApplicationID appId;
-  final Map<CharSequence,CharSequence> env;
-  final Map<CharSequence,LocalResource> resources;
+  final Map<CharSequence, CharSequence> env;
+  final Map<CharSequence, LocalResource> resources;
   final ByteBuffer containerTokens;
-  Map<Path,String> localizedResources;
+  Map<Path, String> localizedResources;
 
   private static final Log LOG = LogFactory.getLog(Application.class);
 
-  private List<ContainerID> containersWaitingForAppInit =
-      new ArrayList<ContainerID>();
+  Map<ContainerID, Container> containers =
+      new HashMap<ContainerID, Container>();
 
   // TODO check for suitability of symlink name
-  static Map<String,LocalResource>
-    filterResources(Map<CharSequence,LocalResource> resources,
-        LocalResourceVisibility state) {
-    Map<String,LocalResource> ret =
-      new HashMap<String,LocalResource>();
-    for (Map.Entry<CharSequence,LocalResource> rsrc : resources.entrySet()) {
+  static Map<String, LocalResource>
+      filterResources(Map<CharSequence, LocalResource> resources,
+          LocalResourceVisibility state) {
+    Map<String, LocalResource> ret =
+        new HashMap<String, LocalResource>();
+    for (Map.Entry<CharSequence, LocalResource> rsrc : resources.entrySet()) {
       if (state.equals(rsrc.getValue().state)) {
         ret.put(rsrc.getKey().toString(), rsrc.getValue());
       }
@@ -83,16 +83,16 @@ public class ApplicationImpl implements 
   public ApplicationImpl(Dispatcher dispatcher,
       CharSequence user,
       ApplicationID appId,
-      Map<CharSequence,CharSequence> env,
-      Map<CharSequence,LocalResource> resources,
+      Map<CharSequence, CharSequence> env,
+      Map<CharSequence, LocalResource> resources,
       ByteBuffer containerTokens) {
     this.dispatcher = dispatcher;
     this.user = user.toString();
     this.appId = appId;
     this.env = env;
     this.resources = null == resources
-      ? new HashMap<CharSequence,LocalResource>()
-      : resources;
+        ? new HashMap<CharSequence, LocalResource>()
+        : resources;
     this.containerTokens = containerTokens;
     stateMachine = stateMachineFactory.make(this);
   }
@@ -108,27 +108,34 @@ public class ApplicationImpl implements 
   }
 
   @Override
-  public Map<CharSequence,CharSequence> getEnvironment() {
+  public synchronized ApplicationState getApplicationState() {
+    // TODO: Synchro should be at statemachine level.
+    return this.stateMachine.getCurrentState();
+  }
+
+  @Override
+  public Map<CharSequence, CharSequence> getEnvironment() {
     return env;
   }
 
   @Override
-  public Map<String,LocalResource>
+  public Map<String, LocalResource>
       getResources(LocalResourceVisibility vis) {
-    final Map<String,LocalResource> ret;
+    final Map<String, LocalResource> ret;
     if (LocalResourceVisibility.PUBLIC.equals(vis)) {
       ret = filterResources(resources, LocalResourceVisibility.PUBLIC);
     } else {
       // TODO separate these
       ret = filterResources(resources, LocalResourceVisibility.PRIVATE);
-      ret.putAll(filterResources(resources, LocalResourceVisibility.APPLICATION));
+      ret.putAll(filterResources(resources,
+          LocalResourceVisibility.APPLICATION));
     }
     return Collections.unmodifiableMap(ret);
   }
 
   @Override
-  public Map<Path,String> getLocalizedResources() {
-    if (ApplicationState.INITED.equals(stateMachine.getCurrentState())) {
+  public Map<Path, String> getLocalizedResources() {
+    if (ApplicationState.RUNNING.equals(stateMachine.getCurrentState())) {
       return localizedResources;
     }
     throw new IllegalStateException(
@@ -149,48 +156,80 @@ public class ApplicationImpl implements 
     return ret;
   }
 
-  private static StateMachineFactory
-         <ApplicationImpl, ApplicationState, ApplicationEventType, ApplicationEvent>
-       stateMachineFactory =
+  private static final ContainerDoneTransition CONTAINER_DONE_TRANSITION =
+      new ContainerDoneTransition();
+
+  private static StateMachineFactory<ApplicationImpl, ApplicationState, ApplicationEventType, ApplicationEvent> stateMachineFactory =
       new StateMachineFactory
          <ApplicationImpl, ApplicationState, ApplicationEventType, ApplicationEvent>
        (ApplicationState.NEW)
-    .addTransition(ApplicationState.NEW, ApplicationState.INITING,
-        ApplicationEventType.INIT_APPLICATION, new AppInitTransition())
-    .addTransition(ApplicationState.INITING, ApplicationState.INITING,
-        ApplicationEventType.INIT_APPLICATION, new AppIsInitingTransition())
-    .addTransition(ApplicationState.INITING, ApplicationState.FINISHING,
-        ApplicationEventType.FINISH_APPLICATION, new AppFinishingTransition())
-    .addTransition(ApplicationState.INITING, ApplicationState.INITED,
-        ApplicationEventType.APPLICATION_INITED, new AppInitedTransition())
-    .addTransition(ApplicationState.INITED,
-        ApplicationState.INITED, ApplicationEventType.INIT_APPLICATION,
-        new AppHasInitedTransition())
-    .addTransition(ApplicationState.INITED,
-        ApplicationState.FINISHING, ApplicationEventType.FINISH_APPLICATION,
-        new AppFinishingTransition())
-    .addTransition(ApplicationState.FINISHING,
-        ApplicationState.FINISHED, ApplicationEventType.APPLICATION_FINISHED)
-    // TODO failure transitions are completely broken
 
-    // create the topology tables
-    .installTopology();
+           // Transitions from NEW state
+           .addTransition(ApplicationState.NEW, ApplicationState.INITING,
+               ApplicationEventType.INIT_APPLICATION, new AppInitTransition())
+
+           // Transitions from INITING state
+           .addTransition(ApplicationState.INITING, ApplicationState.INITING,
+               ApplicationEventType.INIT_APPLICATION,
+               new AppIsInitingTransition())
+           .addTransition(ApplicationState.INITING,
+               EnumSet.of(ApplicationState.FINISHING_CONTAINERS_WAIT,
+                   ApplicationState.APPLICATION_RESOURCES_CLEANINGUP),
+               ApplicationEventType.FINISH_APPLICATION,
+               new AppFinishTriggeredTransition())
+           .addTransition(ApplicationState.INITING, ApplicationState.RUNNING,
+               ApplicationEventType.APPLICATION_INITED,
+               new AppInitDoneTransition())
+
+           // Transitions from RUNNING state
+           .addTransition(ApplicationState.RUNNING,
+               ApplicationState.RUNNING,
+               ApplicationEventType.INIT_APPLICATION,
+               new DuplicateAppInitTransition())
+           .addTransition(ApplicationState.RUNNING,
+               ApplicationState.RUNNING,
+               ApplicationEventType.APPLICATION_CONTAINER_FINISHED,
+               CONTAINER_DONE_TRANSITION)
+           .addTransition(
+               ApplicationState.RUNNING,
+               EnumSet.of(ApplicationState.FINISHING_CONTAINERS_WAIT,
+                   ApplicationState.APPLICATION_RESOURCES_CLEANINGUP),
+               ApplicationEventType.FINISH_APPLICATION,
+               new AppFinishTriggeredTransition())
+
+           // Transitions from FINISHING_CONTAINERS_WAIT state.
+           .addTransition(
+               ApplicationState.FINISHING_CONTAINERS_WAIT,
+               EnumSet.of(ApplicationState.FINISHING_CONTAINERS_WAIT,
+                   ApplicationState.APPLICATION_RESOURCES_CLEANINGUP),
+               ApplicationEventType.APPLICATION_CONTAINER_FINISHED,
+               new AppFinishTransition())
+
+           // Transitions from APPLICATION_RESOURCES_CLEANINGUP state
+           .addTransition(ApplicationState.APPLICATION_RESOURCES_CLEANINGUP,
+               ApplicationState.FINISHED,
+               ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP)
 
-  private final StateMachine<ApplicationState, ApplicationEventType, ApplicationEvent>
-    stateMachine;
+           // TODO failure transitions are completely broken
+
+           // create the topology tables
+           .installTopology();
+
+  private final StateMachine<ApplicationState, ApplicationEventType, ApplicationEvent> stateMachine;
 
   static class AppInitTransition implements
       SingleArcTransition<ApplicationImpl, ApplicationEvent> {
     @Override
     public void transition(ApplicationImpl app, ApplicationEvent event) {
       ApplicationInitEvent initEvent = (ApplicationInitEvent) event;
-      ContainerID cId = initEvent.getContainerRequestingAppInit();
-      app.containersWaitingForAppInit.add(cId);
+      Container container = initEvent.getContainer();
+      app.containers.put(container.getContainerID(), container);
       app.dispatcher.getEventHandler().handle(
-          new ContainerEvent(cId, ContainerEventType.INIT_CONTAINER));
+          new ContainerEvent(container.getContainerID(),
+              ContainerEventType.INIT_CONTAINER));
       app.dispatcher.getEventHandler().handle(
           new ApplicationLocalizerEvent(
-            LocalizerEventType.INIT_APPLICATION_RESOURCES, app));
+              LocalizerEventType.INIT_APPLICATION_RESOURCES, app));
     }
   }
 
@@ -199,14 +238,15 @@ public class ApplicationImpl implements 
     @Override
     public void transition(ApplicationImpl app, ApplicationEvent event) {
       ApplicationInitEvent initEvent = (ApplicationInitEvent) event;
-      ContainerID cId = initEvent.getContainerRequestingAppInit();
-      app.containersWaitingForAppInit.add(cId);
+      Container container = initEvent.getContainer();
+      app.containers.put(container.getContainerID(), container);
       app.dispatcher.getEventHandler().handle(
-          new ContainerEvent(cId, ContainerEventType.INIT_CONTAINER));
+          new ContainerEvent(container.getContainerID(),
+              ContainerEventType.INIT_CONTAINER));
     }
   }
 
-  static class AppInitedTransition implements
+  static class AppInitDoneTransition implements
       SingleArcTransition<ApplicationImpl, ApplicationEvent> {
     @Override
     public void transition(ApplicationImpl app, ApplicationEvent event) {
@@ -214,48 +254,100 @@ public class ApplicationImpl implements 
       ApplicationInitedEvent initedEvent = (ApplicationInitedEvent) event;
       app.localizedResources = initedEvent.getLocalizedResources();
       // Start all the containers waiting for ApplicationInit
-      for (ContainerID containerID : app.containersWaitingForAppInit) {
-        app.dispatcher.getEventHandler().handle(
-            new ContainerEvent(containerID,
-              ContainerEventType.CONTAINER_RESOURCES_LOCALIZED));
+      Iterator<Container> it = app.containers.values().iterator();
+      while (it.hasNext()) {
+        Container container = it.next();
+        if (container.getContainerState().equals(ContainerState.LOCALIZING)) {
+          app.dispatcher.getEventHandler().handle(
+              new ContainerEvent(container.getContainerID(),
+                  ContainerEventType.CONTAINER_RESOURCES_LOCALIZED));
+        }
       }
     }
   }
 
-  static class AppHasInitedTransition implements
+  static class DuplicateAppInitTransition implements
       SingleArcTransition<ApplicationImpl, ApplicationEvent> {
     @Override
     public void transition(ApplicationImpl app, ApplicationEvent event) {
       ApplicationInitEvent initEvent = (ApplicationInitEvent) event;
-      ContainerID containerID = initEvent.getContainerRequestingAppInit();
+      ContainerID containerID = initEvent.getContainer().getContainerID();
       app.dispatcher.getEventHandler().handle(
           new ContainerEvent(containerID, ContainerEventType.INIT_CONTAINER));
       app.dispatcher.getEventHandler().handle(
           new ContainerEvent(containerID,
-            ContainerEventType.CONTAINER_RESOURCES_LOCALIZED));
+              ContainerEventType.CONTAINER_RESOURCES_LOCALIZED));
     }
   }
 
-  static class AppFinishingTransition implements
+  static final class ContainerDoneTransition implements
       SingleArcTransition<ApplicationImpl, ApplicationEvent> {
     @Override
     public void transition(ApplicationImpl app, ApplicationEvent event) {
+      ApplicationContainerFinishedEvent containerEvent =
+          (ApplicationContainerFinishedEvent) event;
+      LOG.info("Removing " + containerEvent.getContainerID()
+          + " from application " + app.toString());
+      app.containers.remove(containerEvent.getContainerID());
+    }
+  }
+
+  void handleAppFinishWithContainersCleanedup() {
+    // Delete Application level resources
+    this.dispatcher.getEventHandler().handle(
+        new ApplicationLocalizerEvent(
+            LocalizerEventType.DESTROY_APPLICATION_RESOURCES, this));
+
+    // TODO: Trigger the LogsManager
+  }
+
+  static class AppFinishTriggeredTransition
+      implements
+      MultipleArcTransition<ApplicationImpl, ApplicationEvent, ApplicationState> {
+    @Override
+    public ApplicationState transition(ApplicationImpl app,
+        ApplicationEvent event) {
+
+      if (app.containers.isEmpty()) {
+        // No container to cleanup. Cleanup app level resources.
+        app.handleAppFinishWithContainersCleanedup();
+        return ApplicationState.APPLICATION_RESOURCES_CLEANINGUP;
+      }
 
       // Send event to ContainersLauncher to finish all the containers of this
       // application.
-      for (ContainerID containerID : app.containersWaitingForAppInit) {
+      for (ContainerID containerID : app.containers.keySet()) {
         app.dispatcher.getEventHandler().handle(
             new ContainerEvent(containerID,
-              ContainerEventType.KILL_CONTAINER));
+                ContainerEventType.KILL_CONTAINER));
       }
+      return ApplicationState.FINISHING_CONTAINERS_WAIT;
+    }
+  }
 
-      // Delete Application level resources
-      app.dispatcher.getEventHandler().handle(
-          new ApplicationLocalizerEvent(
-            LocalizerEventType.DESTROY_APPLICATION_RESOURCES, app));
+  static class AppFinishTransition
+      implements
+      MultipleArcTransition<ApplicationImpl, ApplicationEvent, ApplicationState> {
 
-      // TODO: Trigger the LogsManager
+    @Override
+    public ApplicationState transition(ApplicationImpl app,
+        ApplicationEvent event) {
+
+      ApplicationContainerFinishedEvent containerFinishEvent =
+          (ApplicationContainerFinishedEvent) event;
+      LOG.info("Removing " + containerFinishEvent.getContainerID()
+          + " from application " + app.toString());
+      app.containers.remove(containerFinishEvent.getContainerID());
+
+      if (app.containers.isEmpty()) {
+        // All containers are cleanedup.
+        app.handleAppFinishWithContainersCleanedup();
+        return ApplicationState.FINISHING_CONTAINERS_WAIT;
+      }
+
+      return ApplicationState.APPLICATION_RESOURCES_CLEANINGUP;
     }
+
   }
 
   @Override

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java Tue Mar 22 11:59:33 2011
@@ -18,19 +18,20 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.application;
 
-import org.apache.hadoop.yarn.ContainerID;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 
 public class ApplicationInitEvent extends ApplicationEvent {
 
-  private final ContainerID containerRequestingAppInit;
+  private final Container container;
 
-  public ApplicationInitEvent(ContainerID containerID) {
-    super(containerID.appID, ApplicationEventType.INIT_APPLICATION);
-    this.containerRequestingAppInit = containerID;
+  public ApplicationInitEvent(Container container) {
+    super(container.getContainerID().appID,
+        ApplicationEventType.INIT_APPLICATION);
+    this.container = container;
   }
 
-  public ContainerID getContainerRequestingAppInit() {
-    return this.containerRequestingAppInit;
+  public Container getContainer() {
+    return this.container;
   }
 
 }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationState.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationState.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationState.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationState.java Tue Mar 22 11:59:33 2011
@@ -19,5 +19,5 @@
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.application;
 
 public enum ApplicationState {
-  NEW, INITING, INITED, FINISHING, FINISHED 
+  NEW, INITING, RUNNING, FINISHING_CONTAINERS_WAIT, APPLICATION_RESOURCES_CLEANINGUP, FINISHED 
 }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java Tue Mar 22 11:59:33 2011
@@ -24,12 +24,17 @@ import org.apache.hadoop.yarn.ContainerS
 
 public interface Container extends EventHandler<ContainerEvent> {
 
-  org.apache.hadoop.yarn.Container getContainer();
+  org.apache.hadoop.yarn.ContainerID getContainerID();
+
+  String getUser();
+
+  ContainerState getContainerState();
 
-  // TODO overly-general interface
   ContainerLaunchContext getLaunchContext();
 
-  ContainerStatus getContainerStatus();
+  org.apache.hadoop.yarn.Container cloneAndGetContainer();
+
+  ContainerStatus cloneAndGetContainerStatus();
 
   String toString();
 }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEvent.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEvent.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEvent.java Tue Mar 22 11:59:33 2011
@@ -18,9 +18,8 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
 
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.ContainerID;
+import org.apache.hadoop.yarn.event.AbstractEvent;
 
 public class ContainerEvent extends AbstractEvent<ContainerEventType> {
 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Tue Mar 22 11:59:33 2011
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.ContainerS
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizerEvent;
@@ -57,6 +58,9 @@ public class ContainerImpl implements Co
     stateMachine = stateMachineFactory.make(this);
   }
 
+  private static final ContainerDoneTransition CONTAINER_DONE_TRANSITION =
+    new ContainerDoneTransition();
+
   // State Machine for each container.
   private static StateMachineFactory
            <ContainerImpl, ContainerState, ContainerEventType, ContainerEvent>
@@ -66,7 +70,7 @@ public class ContainerImpl implements Co
     .addTransition(ContainerState.NEW, ContainerState.LOCALIZING,
         ContainerEventType.INIT_CONTAINER)
     .addTransition(ContainerState.NEW, ContainerState.DONE,
-        ContainerEventType.KILL_CONTAINER)
+        ContainerEventType.KILL_CONTAINER, CONTAINER_DONE_TRANSITION)
 
     // From LOCALIZING State
     .addTransition(ContainerState.LOCALIZING,
@@ -99,14 +103,16 @@ public class ContainerImpl implements Co
 
     // From CONTAINER_EXITED_WITH_SUCCESS State
     .addTransition(ContainerState.EXITED_WITH_SUCCESS, ContainerState.DONE,
-            ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)
+            ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP,
+            CONTAINER_DONE_TRANSITION)
     .addTransition(ContainerState.EXITED_WITH_SUCCESS,
                    ContainerState.EXITED_WITH_SUCCESS,
                    ContainerEventType.KILL_CONTAINER)
 
     // From EXITED_WITH_FAILURE State
     .addTransition(ContainerState.EXITED_WITH_FAILURE, ContainerState.DONE,
-            ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)
+            ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP,
+            CONTAINER_DONE_TRANSITION)
     .addTransition(ContainerState.EXITED_WITH_FAILURE,
                    ContainerState.EXITED_WITH_FAILURE,
                    ContainerEventType.KILL_CONTAINER)
@@ -128,11 +134,12 @@ public class ContainerImpl implements Co
     // From CONTAINER_CLEANEDUP_AFTER_KILL State.
     .addTransition(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
             ContainerState.DONE,
-            ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)
+            ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP,
+            CONTAINER_DONE_TRANSITION)
 
     // From DONE
     .addTransition(ContainerState.DONE, ContainerState.DONE,
-        ContainerEventType.KILL_CONTAINER)
+        ContainerEventType.KILL_CONTAINER, CONTAINER_DONE_TRANSITION)
 
     // create the topology tables
     .installTopology();
@@ -160,7 +167,22 @@ public class ContainerImpl implements Co
   }
 
   @Override
-  public synchronized org.apache.hadoop.yarn.Container getContainer() {
+  public ContainerID getContainerID() {
+    return this.launchContext.id;
+  }
+
+  @Override
+  public String getUser() {
+    return this.launchContext.user.toString();
+  }
+
+  @Override
+  public ContainerState getContainerState() {
+    return stateMachine.getCurrentState();
+  }
+
+  @Override
+  public synchronized org.apache.hadoop.yarn.Container cloneAndGetContainer() {
     org.apache.hadoop.yarn.Container c = new org.apache.hadoop.yarn.Container();
     c.id = this.launchContext.id;
     c.resource = this.launchContext.resource;
@@ -174,7 +196,7 @@ public class ContainerImpl implements Co
   }
 
   @Override
-  public ContainerStatus getContainerStatus() {
+  public ContainerStatus cloneAndGetContainerStatus() {
     ContainerStatus containerStatus = new ContainerStatus();
     containerStatus.state = getCurrentState();
     containerStatus.containerID = this.launchContext.id;
@@ -294,6 +316,16 @@ public class ContainerImpl implements Co
     }
   }
 
+  static class ContainerDoneTransition implements
+      SingleArcTransition<ContainerImpl, ContainerEvent> {
+    @Override
+    public void transition(ContainerImpl container, ContainerEvent event) {
+      // Inform the application
+      container.dispatcher.getEventHandler().handle(
+          new ApplicationContainerFinishedEvent(container.getContainerID()));
+    }
+  }
+
   @Override
   public synchronized void handle(ContainerEvent event) {
 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Tue Mar 22 11:59:33 2011
@@ -114,7 +114,7 @@ public class ContainerLaunch implements 
         }
       }
       dispatcher.getEventHandler().handle(new ContainerEvent(
-            launchContext.id,
+            container.getContainerID(),
             ContainerEventType.CONTAINER_LAUNCHED));
       ret =
         exec.launchContainer(container, launchSysDir, user, app.toString(),

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java Tue Mar 22 11:59:33 2011
@@ -129,8 +129,8 @@ public class ContainersLauncher extends 
   public void handle(ContainersLauncherEvent event) {
     // TODO: ContainersLauncher launches containers one by one!!
     Container container = event.getContainer();
-    ContainerID containerId = container.getLaunchContext().id;
-    String userName = container.getLaunchContext().user.toString();
+    ContainerID containerId = container.getContainerID();
+    String userName = container.getUser();
     switch (event.getType()) {
       case LAUNCH_CONTAINER:
         Application app =

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/AppLocalizationRunnerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/AppLocalizationRunnerImpl.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/AppLocalizationRunnerImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/AppLocalizationRunnerImpl.java Tue Mar 22 11:59:33 2011
@@ -135,7 +135,8 @@ public class AppLocalizationRunnerImpl i
       Path appTokens = new Path(sysDir, ApplicationLocalizer.APPTOKEN_FILE);
       tokenOut = lfs.create(appTokens, EnumSet.of(CREATE, OVERWRITE));
       Credentials appCreds = app.getCredentials();
-      LOG.info("Writing credentials again. Credentials list: ");
+      LOG.info("Writing credentials again to " + appTokens.toString()
+          + ". Credentials list: ");
       for (Token<? extends TokenIdentifier> tk : appCreds.getAllTokens()) {
         LOG.info(tk.getService() + " : " + tk.encodeToUrlString());
       }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Tue Mar 22 11:59:33 2011
@@ -18,6 +18,13 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
 
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_LOCALIZER_BIND_ADDRESS;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_LOCAL_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_LOG_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_LOCALIZER_BIND_ADDRESS;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_LOCAL_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_LOG_DIR;
+
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
@@ -39,7 +46,10 @@ import org.apache.hadoop.fs.permission.F
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityInfo;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.LocalizationProtocol;
+import org.apache.hadoop.yarn.URL;
 import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRemoteException;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.ipc.RPCUtil;
@@ -47,6 +57,8 @@ import org.apache.hadoop.yarn.ipc.YarnRP
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
@@ -57,11 +69,6 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenSecretManager;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.util.AvroUtil;
-import org.apache.hadoop.yarn.LocalizationProtocol;
-import org.apache.hadoop.yarn.URL;
-import org.apache.hadoop.yarn.YarnRemoteException;
-
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.*;
 
 public class ResourceLocalizationService extends AbstractService
     implements EventHandler<LocalizerEvent>, LocalizationProtocol {
@@ -96,7 +103,7 @@ public class ResourceLocalizationService
 
   public ResourceLocalizationService(Dispatcher dispatcher,
       ContainerExecutor exec, DeletionService delService) {
-    super("localizer");
+    super(ResourceLocalizationService.class.getName());
     this.exec = exec;
     this.dispatcher = dispatcher;
     this.delService = delService;
@@ -222,6 +229,9 @@ public class ResourceLocalizationService
 
   @Override
   public void handle(LocalizerEvent event) {
+    String userName;
+    String appIDStr;
+    
     switch (event.getType()) {
     case INIT_APPLICATION_RESOURCES:
       Application app = ((ApplicationLocalizerEvent)event).getApplication();
@@ -257,13 +267,60 @@ public class ResourceLocalizationService
       break;
     case CLEANUP_CONTAINER_RESOURCES:
       Container container = ((ContainerLocalizerEvent)event).getContainer();
-      // TODO: delete the container dir
+
+      // Delete the container directories
+      userName = container.getUser();;
+      String containerIDStr = container.toString();
+      appIDStr = AvroUtil.toString(container.getContainerID().appID);
+      for (Path localDir : localDirs) {
+        Path usersdir = new Path(localDir, ApplicationLocalizer.USERCACHE);
+        Path userdir =
+            new Path(usersdir, userName);
+        Path allAppsdir = new Path(userdir, ApplicationLocalizer.APPCACHE);
+        Path appDir = new Path(allAppsdir, appIDStr);
+        Path containerDir =
+            new Path(appDir, containerIDStr);
+        delService.delete(userName,
+            containerDir, null);
+
+        Path sysDir = new Path(localDir, NM_PRIVATE_DIR);
+        Path appSysDir = new Path(sysDir, appIDStr);
+        Path containerSysDir = new Path(appSysDir, containerIDStr);
+        delService.delete(null, containerSysDir, null);
+      }
+
       dispatcher.getEventHandler().handle(new ContainerEvent(
-            container.getLaunchContext().id,
+            container.getContainerID(),
             ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
       break;
     case DESTROY_APPLICATION_RESOURCES:
-      // decrement reference counts of all resources associated with this app
+
+      Application application =
+          ((ApplicationLocalizerEvent) event).getApplication();
+
+      // Delete the application directories
+      userName = application.getUser();
+      appIDStr = application.toString();
+      for (Path localDir : localDirs) {
+        Path usersdir = new Path(localDir, ApplicationLocalizer.USERCACHE);
+        Path userdir =
+            new Path(usersdir, userName);
+        Path allAppsdir = new Path(userdir, ApplicationLocalizer.APPCACHE);
+        Path appDir = new Path(allAppsdir, appIDStr);
+        delService.delete(userName,
+            appDir, null);
+
+        Path sysDir = new Path(localDir, NM_PRIVATE_DIR);
+        Path appSysDir = new Path(sysDir, appIDStr);
+        delService.delete(null, appSysDir, null);
+      }
+
+      // TODO: decrement reference counts of all resources associated with this
+      // app
+
+      dispatcher.getEventHandler().handle(new ApplicationEvent(
+            application.getAppId(),
+            ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP));
       break;
     }
   }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Tue Mar 22 11:59:33 2011
@@ -26,28 +26,27 @@ import org.apache.avro.ipc.AvroRemoteExc
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
-import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.ContainerID;
+import org.apache.hadoop.yarn.ContainerManager;
+import org.apache.hadoop.yarn.ContainerState;
+import org.apache.hadoop.yarn.ContainerStatus;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationInitedEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent;
-import org.apache.hadoop.yarn.ContainerID;
-import org.apache.hadoop.yarn.ContainerManager;
-import org.apache.hadoop.yarn.ContainerState;
-import org.apache.hadoop.yarn.ContainerStatus;
 
-class DummyContainerManager extends ContainerManagerImpl {
+public class DummyContainerManager extends ContainerManagerImpl {
 
   private static final Log LOG = LogFactory
       .getLog(DummyContainerManager.class);
@@ -77,12 +76,18 @@ class DummyContainerManager extends Cont
               ((ContainerLocalizerEvent) event).getContainer();
           // TODO: delete the container dir
           this.dispatcher.getEventHandler().handle(
-              new ContainerEvent(container.getLaunchContext().id,
+              new ContainerEvent(container.getContainerID(),
                   ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
           break;
         case DESTROY_APPLICATION_RESOURCES:
+          Application application =
+            ((ApplicationLocalizerEvent) event).getApplication();
+
           // decrement reference counts of all resources associated with this
           // app
+          this.dispatcher.getEventHandler().handle(
+              new ApplicationEvent(application.getAppId(),
+                  ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP));
           break;
         }
       }
@@ -96,7 +101,7 @@ class DummyContainerManager extends Cont
       @Override
       public void handle(ContainersLauncherEvent event) {
         Container container = event.getContainer();
-        ContainerID containerId = container.getLaunchContext().id;
+        ContainerID containerId = container.getContainerID();
         switch (event.getType()) {
         case LAUNCH_CONTAINER:
           dispatcher.getEventHandler().handle(
@@ -105,15 +110,15 @@ class DummyContainerManager extends Cont
           break;
         case CLEANUP_CONTAINER:
           dispatcher.getEventHandler().handle(
-              new ContainerEvent(containerId,
-                  ContainerEventType.CONTAINER_KILLED_ON_REQUEST));
+              new ContainerExitEvent(containerId,
+                  ContainerEventType.CONTAINER_KILLED_ON_REQUEST, 0));
           break;
         }
       }
     };
   }
 
-  static void waitForContainerState(ContainerManager containerManager,
+  public static void waitForContainerState(ContainerManager containerManager,
         ContainerID containerID, ContainerState finalState)
         throws InterruptedException, AvroRemoteException {
       ContainerStatus containerStatus =

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java Tue Mar 22 11:59:33 2011
@@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.Registrati
 import org.apache.hadoop.yarn.Resource;
 import org.apache.hadoop.yarn.ResourceTracker;
 
-class LocalRMInterface implements ResourceTracker {
+public class LocalRMInterface implements ResourceTracker {
 
   @Override
   public RegistrationResponse registerNodeManager(CharSequence node,

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java?rev=1084143&r1=1084142&r2=1084143&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java Tue Mar 22 11:59:33 2011
@@ -23,10 +23,11 @@ import java.io.IOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
+import org.junit.After;
 
 public class TestContainerManagerWithLCE extends TestContainerManager {
 
@@ -62,21 +63,34 @@ public class TestContainerManagerWithLCE
             (short) 0777));
   }
 
+  @After
   @Override
   public void tearDown() throws IOException, InterruptedException {
-    super.tearDown();
-    FileUtil.chmod(localDir.getAbsolutePath(), "777", true);
-    localFS.delete(new Path(localDir.getCanonicalPath()), true);
+    if (shouldRunTest()) {
+      super.tearDown();
+    }
   }
 
   @Override
   public void testContainerSetup() throws IOException, InterruptedException {
-    // No cleanup as of now. Only one test for now. TODO: FIX
+    // Don't run the test if the binary is not available.
+    if (!shouldRunTest()) {
+      LOG.info("LCE binary path is not passed. Not running the test");
+      return;
+    }
+    LOG.info("Running testContainerSetup");
+    super.testContainerSetup();
   }
 
   @Override
   public void testContainerManagerInitialization() throws IOException {
-    // No cleanup as of now. Only one test for now. TODO: FIX
+    // Don't run the test if the binary is not available.
+    if (!shouldRunTest()) {
+      LOG.info("LCE binary path is not passed. Not running the test");
+      return;
+    }
+    LOG.info("Running testContainerManagerInitialization");
+    super.testContainerManagerInitialization();
   }
 
   @Override
@@ -87,9 +101,22 @@ public class TestContainerManagerWithLCE
       LOG.info("LCE binary path is not passed. Not running the test");
       return;
     }
+    LOG.info("Running testContainerLaunchAndStop");
     super.testContainerLaunchAndStop();
   }
 
+  @Override
+  public void testLocalFilesCleanup() throws InterruptedException,
+      IOException {
+    // Don't run the test if the binary is not available.
+    if (!shouldRunTest()) {
+      LOG.info("LCE binary path is not passed. Not running the test");
+      return;
+    }
+    LOG.info("Running testLocalFilesCleanup");
+    super.testLocalFilesCleanup();
+  }
+
   private boolean shouldRunTest() {
     return System
         .getProperty(LinuxContainerExecutor.CONTAINER_EXECUTOR_EXEC_KEY) != null;



Mime
View raw message