hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1170378 [7/12] - in /hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/ dev-support/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-mapreduce-clie...
Date Tue, 13 Sep 2011 22:49:38 GMT
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Tue Sep 13 22:49:27 2011
@@ -18,9 +18,6 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation;
 
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_BIND_ADDRESS;
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_BIND_ADDRESS;
-
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
@@ -43,12 +40,9 @@ import org.apache.hadoop.security.token.
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorAppFinishedEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorAppStartedEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorContainerFinishedEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorEvent;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -65,13 +59,6 @@ public class LogAggregationService exten
   Path remoteRootLogDir;
   private String nodeFile;
 
-  static final String LOG_COMPRESSION_TYPE = NMConfig.NM_PREFIX
-      + "logaggregation.log_compression_type";
-  static final String DEFAULT_COMPRESSION_TYPE = "none";
-
-  private static final String LOG_RENTENTION_POLICY_CONFIG_KEY =
-      NMConfig.NM_PREFIX + "logaggregation.retain-policy";
-
   private final ConcurrentMap<ApplicationId, AppLogAggregator> appLogAggregators;
 
   private final ExecutorService threadPool;
@@ -86,17 +73,17 @@ public class LogAggregationService exten
 
   public synchronized void init(Configuration conf) {
     this.localRootLogDirs =
-        conf.getStrings(NMConfig.NM_LOG_DIR, NMConfig.DEFAULT_NM_LOG_DIR);
+        conf.getStrings(YarnConfiguration.NM_LOG_DIRS, YarnConfiguration.DEFAULT_NM_LOG_DIRS);
     this.remoteRootLogDir =
-        new Path(conf.get(NMConfig.REMOTE_USER_LOG_DIR,
-            NMConfig.DEFAULT_REMOTE_APP_LOG_DIR));
+        new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+            YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
     super.init(conf);
   }
 
   @Override
   public synchronized void start() {
     String address =
-        getConfig().get(NM_BIND_ADDRESS, DEFAULT_NM_BIND_ADDRESS);
+        getConfig().get(YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS);
     InetSocketAddress cmBindAddress = NetUtils.createSocketAddr(address);
     try {
       this.nodeFile =
@@ -185,11 +172,13 @@ public class LogAggregationService exten
     // A container is complete. Put this containers' logs up for aggregation if
     // this containers' logs are needed.
 
-    if (!this.appLogAggregators.containsKey(containerId.getAppId())) {
+    if (!this.appLogAggregators.containsKey(
+        containerId.getApplicationAttemptId().getApplicationId())) {
       throw new YarnException("Application is not initialized yet for "
           + containerId);
     }
-    this.appLogAggregators.get(containerId.getAppId())
+    this.appLogAggregators.get(
+        containerId.getApplicationAttemptId().getApplicationId())
         .startContainerLogAggregation(containerId, exitCode.equals("0"));
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java Tue Sep 13 22:49:27 2011
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogKey;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogReader;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -106,8 +105,8 @@ public class LogDumper extends Configure
       return -1;
     } else {
       Path remoteRootLogDir =
-        new Path(getConf().get(NMConfig.REMOTE_USER_LOG_DIR,
-            NMConfig.DEFAULT_REMOTE_APP_LOG_DIR));
+        new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+            YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
       AggregatedLogFormat.LogReader reader =
           new AggregatedLogFormat.LogReader(getConf(),
               LogAggregationService.getRemoteNodeLogFileForApp(
@@ -151,8 +150,8 @@ public class LogDumper extends Configure
       dumpAllContainersLogs(ApplicationId appId, DataOutputStream out)
           throws IOException {
     Path remoteRootLogDir =
-        new Path(getConf().get(NMConfig.REMOTE_USER_LOG_DIR,
-            NMConfig.DEFAULT_REMOTE_APP_LOG_DIR));
+        new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+            YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
     Path remoteAppLogDir =
         LogAggregationService.getRemoteAppLogDir(remoteRootLogDir, appId);
     RemoteIterator<FileStatus> nodeFiles =

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/event/LogAggregatorContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/event/LogAggregatorContainerFinishedEvent.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/event/LogAggregatorContainerFinishedEvent.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/event/LogAggregatorContainerFinishedEvent.java Tue Sep 13 22:49:27 2011
@@ -23,10 +23,10 @@ import org.apache.hadoop.yarn.api.record
 public class LogAggregatorContainerFinishedEvent extends LogAggregatorEvent {
 
   private final ContainerId containerId;
-  private final String exitCode;
+  private final int exitCode;
 
   public LogAggregatorContainerFinishedEvent(ContainerId containerId,
-      String exitCode) {
+      int exitCode) {
     super(LogAggregatorEventType.CONTAINER_FINISHED);
     this.containerId = containerId;
     this.exitCode = exitCode;
@@ -36,7 +36,7 @@ public class LogAggregatorContainerFinis
     return this.containerId;
   }
 
-  public String getExitCode() {
+  public int getExitCode() {
     return this.exitCode;
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Tue Sep 13 22:49:27 2011
@@ -29,11 +29,11 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
@@ -45,14 +45,6 @@ public class ContainersMonitorImpl exten
   final static Log LOG = LogFactory
       .getLog(ContainersMonitorImpl.class);
 
-  private final static String MONITORING_INTERVAL_CONFIG_KEY =
-      NMConfig.NM_PREFIX + "containers-monitor.monitoring-interval";
-  public static final String RESOURCE_CALCULATOR_PLUGIN_CONFIG_KEY =
-      NMConfig.NM_PREFIX + "containers-monitor.resourcecalculatorplugin";
-  public static final String NM_RESERVED_PHYSICALMEMORY_MB =
-      NMConfig.NM_PREFIX + "reserved-physical-memory.mb";
-
-  private final static int MONITORING_INTERVAL_DEFAULT = 3000;
   private long monitoringInterval;
   private MonitoringThread monitoringThread;
 
@@ -96,11 +88,11 @@ public class ContainersMonitorImpl exten
   @Override
   public synchronized void init(Configuration conf) {
     this.monitoringInterval =
-        conf.getLong(MONITORING_INTERVAL_CONFIG_KEY,
-            MONITORING_INTERVAL_DEFAULT);
+        conf.getLong(YarnConfiguration.NM_CONTAINER_MON_INTERVAL_MS,
+            YarnConfiguration.DEFAULT_NM_CONTAINER_MON_INTERVAL_MS);
 
     Class<? extends ResourceCalculatorPlugin> clazz =
-        conf.getClass(RESOURCE_CALCULATOR_PLUGIN_CONFIG_KEY, null,
+        conf.getClass(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, null,
             ResourceCalculatorPlugin.class);
     this.resourceCalculatorPlugin =
         ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
@@ -120,7 +112,7 @@ public class ContainersMonitorImpl exten
 
     // ///////// Virtual memory configuration //////
     this.maxVmemAllottedForContainers =
-        conf.getLong(NMConfig.NM_VMEM_GB, NMConfig.DEFAULT_NM_VMEM_GB);
+        conf.getLong(YarnConfiguration.NM_VMEM_GB, YarnConfiguration.DEFAULT_NM_VMEM_GB);
     this.maxVmemAllottedForContainers =
         this.maxVmemAllottedForContainers * 1024 * 1024 * 1024L; //Normalize
 
@@ -131,7 +123,7 @@ public class ContainersMonitorImpl exten
 
     // ///////// Physical memory configuration //////
     long reservedPmemOnNM =
-        conf.getLong(NM_RESERVED_PHYSICALMEMORY_MB, DISABLED_MEMORY_LIMIT);
+        conf.getLong(YarnConfiguration.NM_RESERVED_MEMORY_MB, DISABLED_MEMORY_LIMIT);
     reservedPmemOnNM =
         reservedPmemOnNM == DISABLED_MEMORY_LIMIT
             ? DISABLED_MEMORY_LIMIT

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java Tue Sep 13 22:49:27 2011
@@ -18,9 +18,6 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.webapp;
 
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_LOG_DIR;
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_LOG_DIR;
-
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
@@ -33,10 +30,10 @@ import org.apache.hadoop.fs.LocalDirAllo
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -64,7 +61,7 @@ public class ContainerLogsPage extends N
     @Inject
     public ContainersLogsBlock(Configuration conf, Context context) {
       this.conf = conf;
-      this.logsSelector = new LocalDirAllocator(NMConfig.NM_LOG_DIR);
+      this.logsSelector = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS);
       this.nmContext = context;
       this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
     }
@@ -98,7 +95,8 @@ public class ContainerLogsPage extends N
             logFile =
                 new File(this.logsSelector
                     .getLocalPathToRead(
-                        ConverterUtils.toString(containerId.getAppId())
+                        ConverterUtils.toString(
+                            containerId.getApplicationAttemptId().getApplicationId())
                             + Path.SEPARATOR + $(CONTAINER_ID)
                             + Path.SEPARATOR
                             + $(CONTAINER_LOG_TYPE), this.conf).toUri()
@@ -176,10 +174,12 @@ public class ContainerLogsPage extends N
     static List<File>
         getContainerLogDirs(Configuration conf, ContainerId containerId) {
       String[] logDirs =
-          conf.getStrings(NM_LOG_DIR, DEFAULT_NM_LOG_DIR);
+          conf.getStrings(YarnConfiguration.NM_LOG_DIRS, YarnConfiguration.DEFAULT_NM_LOG_DIRS);
       List<File> containerLogDirs = new ArrayList<File>(logDirs.length);
       for (String logDir : logDirs) {
-        String appIdStr = ConverterUtils.toString(containerId.getAppId());
+        String appIdStr = 
+            ConverterUtils.toString(
+                containerId.getApplicationAttemptId().getApplicationId());
         File appLogDir = new File(logDir, appIdStr);
         String containerIdStr = ConverterUtils.toString(containerId);
         containerLogDirs.add(new File(appLogDir, containerIdStr));

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java Tue Sep 13 22:49:27 2011
@@ -25,6 +25,7 @@ import static org.apache.hadoop.yarn.uti
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -69,10 +70,14 @@ public class ContainerPage extends NMVie
         ConverterUtils.toContainerId(this.recordFactory, $(CONTAINER_ID));
       Container container = this.nmContext.getContainers().get(containerID);
       ContainerStatus containerData = container.cloneAndGetContainerStatus();
+      int exitCode = containerData.getExitStatus();
+      String exiStatus = 
+          (exitCode == YarnConfiguration.INVALID_CONTAINER_EXIT_STATUS) ? 
+              "N/A" : String.valueOf(exitCode);
       info("Container information")
         ._("ContainerID", $(CONTAINER_ID))
         ._("ContainerState", container.getContainerState())
-        ._("ExitStatus", containerData.getExitStatus())
+        ._("ExitStatus", exiStatus)
         ._("Diagnostics", containerData.getDiagnostics())
         ._("User", container.getUser())
         ._("TotalMemoryNeeded",

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java Tue Sep 13 22:49:27 2011
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.no
 import static org.apache.hadoop.yarn.util.StringHelper.join;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
 import org.apache.hadoop.yarn.webapp.Controller;
 
 import com.google.inject.Inject;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java Tue Sep 13 22:49:27 2011
@@ -24,8 +24,8 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
 import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.webapp.WebApp;
@@ -52,8 +52,8 @@ public class WebServer extends AbstractS
 
   @Override
   public synchronized void start() {
-    String bindAddress = getConfig().get(NMConfig.NM_HTTP_BIND_ADDRESS,
-        NMConfig.DEFAULT_NM_HTTP_BIND_ADDRESS);
+    String bindAddress = getConfig().get(YarnConfiguration.NM_WEBAPP_ADDRESS,
+        YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
     LOG.info("Instantiating NMWebApp at " + bindAddress);
     try {
       this.webApp =

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Tue Sep 13 22:49:27 2011
@@ -20,12 +20,15 @@ package org.apache.hadoop.yarn.server.no
 
 import static org.junit.Assert.fail;
 
+import java.util.Collection;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -57,8 +60,8 @@ public class DummyContainerManager exten
   
   public DummyContainerManager(Context context, ContainerExecutor exec,
       DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
-      NodeManagerMetrics metrics) {
-    super(context, exec, deletionContext, nodeStatusUpdater, metrics);
+      NodeManagerMetrics metrics, ContainerTokenSecretManager containerTokenSecretManager) {
+    super(context, exec, deletionContext, nodeStatusUpdater, metrics, containerTokenSecretManager);
   }
 
   @Override
@@ -79,14 +82,17 @@ public class DummyContainerManager exten
           ContainerLocalizationRequestEvent rsrcReqs =
             (ContainerLocalizationRequestEvent) event;
           // simulate localization of all requested resources
-          for (LocalResourceRequest req : rsrcReqs.getRequestedResources()) {
-            LOG.info("DEBUG: " + req + ":" +
-                rsrcReqs.getContainer().getContainerID());
-            dispatcher.getEventHandler().handle(
-                new ContainerResourceLocalizedEvent(
-                  rsrcReqs.getContainer().getContainerID(), req,
-                  new Path("file:///local" + req.getPath().toUri().getPath())));
-          }
+            for (Collection<LocalResourceRequest> rc : rsrcReqs
+                .getRequestedResources().values()) {
+              for (LocalResourceRequest req : rc) {
+                LOG.info("DEBUG: " + req + ":"
+                    + rsrcReqs.getContainer().getContainerID());
+                dispatcher.getEventHandler().handle(
+                    new ContainerResourceLocalizedEvent(rsrcReqs.getContainer()
+                        .getContainerID(), req, new Path("file:///local"
+                        + req.getPath().toUri().getPath())));
+              }
+            }
           break;
         case CLEANUP_CONTAINER_RESOURCES:
           Container container =

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java Tue Sep 13 22:49:27 2011
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
 import org.junit.After;
 
@@ -119,13 +120,13 @@ public class TestContainerManagerWithLCE
 
   private boolean shouldRunTest() {
     return System
-        .getProperty(LinuxContainerExecutor.CONTAINER_EXECUTOR_EXEC_KEY) != null;
+        .getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null;
   }
 
   @Override
   protected ContainerExecutor createContainerExecutor() {
-    super.conf.set(LinuxContainerExecutor.CONTAINER_EXECUTOR_EXEC_KEY, System
-        .getProperty(LinuxContainerExecutor.CONTAINER_EXECUTOR_EXEC_KEY));
+    super.conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, System
+        .getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH));
     LinuxContainerExecutor linuxContainerExecutor =
         new LinuxContainerExecutor();
     linuxContainerExecutor.setConf(super.conf);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Tue Sep 13 22:49:27 2011
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.no
 import java.io.File;
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.NodeHealthCheckerService;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
@@ -37,9 +35,9 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.api.ResourceTracker;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -48,8 +46,8 @@ import org.junit.Test;
 
 public class TestEventFlow {
 
-  private static final Log LOG = LogFactory.getLog(TestEventFlow.class);
-  private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+  private static final RecordFactory recordFactory =
+      RecordFactoryProvider.getRecordFactory(null);
 
   private static File localDir = new File("target",
       TestEventFlow.class.getName() + "-localDir").getAbsoluteFile();
@@ -74,9 +72,10 @@ public class TestEventFlow {
     Context context = new NMContext();
 
     YarnConfiguration conf = new YarnConfiguration();
-    conf.set(NMConfig.NM_LOCAL_DIR, localDir.getAbsolutePath());
-    conf.set(NMConfig.NM_LOG_DIR, localLogDir.getAbsolutePath());
-    conf.set(NMConfig.REMOTE_USER_LOG_DIR, remoteLogDir.getAbsolutePath());
+    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
+    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, 
+        remoteLogDir.getAbsolutePath());
 
     ContainerExecutor exec = new DefaultContainerExecutor();
     exec.setConf(conf);
@@ -84,8 +83,9 @@ public class TestEventFlow {
     Dispatcher dispatcher = new AsyncDispatcher();
     NodeHealthCheckerService healthChecker = null;
     NodeManagerMetrics metrics = NodeManagerMetrics.create();
+    ContainerTokenSecretManager containerTokenSecretManager =  new ContainerTokenSecretManager();
     NodeStatusUpdater nodeStatusUpdater =
-        new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) {
+        new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics, containerTokenSecretManager) {
       @Override
       protected ResourceTracker getRMClient() {
         return new LocalRMInterface();
@@ -98,27 +98,36 @@ public class TestEventFlow {
     };
 
     DummyContainerManager containerManager =
-        new DummyContainerManager(context, exec, del, nodeStatusUpdater, metrics);
+        new DummyContainerManager(context, exec, del, nodeStatusUpdater, 
+            metrics, containerTokenSecretManager);
     containerManager.init(conf);
     containerManager.start();
 
-    ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+    ContainerLaunchContext launchContext = 
+        recordFactory.newRecordInstance(ContainerLaunchContext.class);
     ContainerId cID = recordFactory.newRecordInstance(ContainerId.class);
-    cID.setAppId(recordFactory.newRecordInstance(ApplicationId.class));
-    ApplicationAttemptId atId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    atId.setApplicationId(cID.getAppId());
-    cID.setAppAttemptId(atId);
+    ApplicationId applicationId =
+        recordFactory.newRecordInstance(ApplicationId.class);
+    applicationId.setClusterTimestamp(0);
+    applicationId.setId(0);
+    ApplicationAttemptId applicationAttemptId = 
+        recordFactory.newRecordInstance(ApplicationAttemptId.class);
+    applicationAttemptId.setApplicationId(applicationId);
+    applicationAttemptId.setAttemptId(0);
+    cID.setApplicationAttemptId(applicationAttemptId);
     launchContext.setContainerId(cID);
     launchContext.setUser("testing");
     launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
-    StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
+    StartContainerRequest request = 
+        recordFactory.newRecordInstance(StartContainerRequest.class);
     request.setContainerLaunchContext(launchContext);
     containerManager.startContainer(request);
 
     BaseContainerManagerTest.waitForContainerState(containerManager, cID,
         ContainerState.RUNNING);
 
-    StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
+    StopContainerRequest stopRequest = 
+        recordFactory.newRecordInstance(StopContainerRequest.class);
     stopRequest.setContainerId(cID);
     containerManager.stopContainer(stopRequest);
     BaseContainerManagerTest.waitForContainerState(containerManager, cID,

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java Tue Sep 13 22:49:27 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
 import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 
 import org.apache.hadoop.net.NetUtils;
 
@@ -211,7 +212,8 @@ public class TestNMAuditLogger {
   public void testNMAuditLoggerWithIP() throws Exception {
     Configuration conf = new Configuration();
     // start the IPC server
-    Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf);
+    Server server = RPC.getServer(TestProtocol.class,
+        new MyTestRPCServer(), "0.0.0.0", 0, 5, true, conf, null);
     server.start();
 
     InetSocketAddress addr = NetUtils.getConnectAddress(server);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Sep 13 22:49:27 2011
@@ -21,6 +21,10 @@ package org.apache.hadoop.yarn.server.no
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentMap;
 
 import org.apache.commons.logging.Log;
@@ -28,10 +32,13 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.NodeHealthCheckerService;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -47,9 +54,12 @@ import org.apache.hadoop.yarn.server.api
 import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
 import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
+import org.apache.hadoop.yarn.service.Service;
 import org.apache.hadoop.yarn.service.Service.STATE;
 import org.junit.After;
 import org.junit.Assert;
@@ -58,24 +68,38 @@ import org.junit.Test;
 
 public class TestNodeStatusUpdater {
 
+  // temp fix until metrics system can auto-detect itself running in unit test:
+  static {
+    DefaultMetricsSystem.setMiniClusterMode(true);
+  }
+
   static final Log LOG = LogFactory.getLog(TestNodeStatusUpdater.class);
   static final Path basedir =
       new Path("target", TestNodeStatusUpdater.class.getName());
-  private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+  private static final RecordFactory recordFactory = RecordFactoryProvider
+      .getRecordFactory(null);
 
   int heartBeatID = 0;
   volatile Error nmStartError = null;
+  private final List<NodeId> registeredNodes = new ArrayList<NodeId>();
+
+  @After
+  public void tearDown() {
+    this.registeredNodes.clear();
+    DefaultMetricsSystem.shutdown();
+  }
 
   private class MyResourceTracker implements ResourceTracker {
 
-    private Context context;
+    private final Context context;
 
     public MyResourceTracker(Context context) {
       this.context = context;
     }
 
     @Override
-    public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnRemoteException {
+    public RegisterNodeManagerResponse registerNodeManager(
+        RegisterNodeManagerRequest request) throws YarnRemoteException {
       NodeId nodeId = request.getNodeId();
       Resource resource = request.getResource();
       LOG.info("Registering " + nodeId.toString());
@@ -86,33 +110,60 @@ public class TestNodeStatusUpdater {
         Assert.fail(e.getMessage());
       }
       Assert.assertEquals(5 * 1024, resource.getMemory());
-      RegistrationResponse regResponse = recordFactory.newRecordInstance(RegistrationResponse.class);
-      
-      RegisterNodeManagerResponse response = recordFactory.newRecordInstance(RegisterNodeManagerResponse.class);
+      registeredNodes.add(nodeId);
+      RegistrationResponse regResponse = recordFactory
+          .newRecordInstance(RegistrationResponse.class);
+
+      RegisterNodeManagerResponse response = recordFactory
+          .newRecordInstance(RegisterNodeManagerResponse.class);
       response.setRegistrationResponse(regResponse);
       return response;
     }
 
-    ApplicationId applicationID = recordFactory.newRecordInstance(ApplicationId.class);
-    ApplicationAttemptId appAttemptID = recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    ContainerId firstContainerID = recordFactory.newRecordInstance(ContainerId.class);
-    ContainerId secondContainerID = recordFactory.newRecordInstance(ContainerId.class);
+    ApplicationId applicationID = recordFactory
+        .newRecordInstance(ApplicationId.class);
+    ApplicationAttemptId appAttemptID = recordFactory
+        .newRecordInstance(ApplicationAttemptId.class);
+    ContainerId firstContainerID = recordFactory
+        .newRecordInstance(ContainerId.class);
+    ContainerId secondContainerID = recordFactory
+        .newRecordInstance(ContainerId.class);
+
+    private Map<ApplicationId, List<ContainerStatus>> getAppToContainerStatusMap(
+        List<ContainerStatus> containers) {
+      Map<ApplicationId, List<ContainerStatus>> map =
+          new HashMap<ApplicationId, List<ContainerStatus>>();
+      for (ContainerStatus cs : containers) {
+        ApplicationId applicationId = 
+            cs.getContainerId().getApplicationAttemptId().getApplicationId();
+        List<ContainerStatus> appContainers = map.get(applicationId);
+        if (appContainers == null) {
+          appContainers = new ArrayList<ContainerStatus>();
+          map.put(applicationId, appContainers);
+        }
+        appContainers.add(cs);
+      }
+      return map;
+    }
 
     @Override
-    public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) throws YarnRemoteException {
+    public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
+        throws YarnRemoteException {
       NodeStatus nodeStatus = request.getNodeStatus();
       LOG.info("Got heartbeat number " + heartBeatID);
       nodeStatus.setResponseId(heartBeatID++);
+      Map<ApplicationId, List<ContainerStatus>> appToContainers =
+          getAppToContainerStatusMap(nodeStatus.getContainersStatuses());
       if (heartBeatID == 1) {
-        Assert.assertEquals(0, nodeStatus.getAllContainers().size());
+        Assert.assertEquals(0, nodeStatus.getContainersStatuses().size());
 
         // Give a container to the NM.
         applicationID.setId(heartBeatID);
         appAttemptID.setApplicationId(applicationID);
-        firstContainerID.setAppId(applicationID);
-        firstContainerID.setAppAttemptId(appAttemptID);
+        firstContainerID.setApplicationAttemptId(appAttemptID);
         firstContainerID.setId(heartBeatID);
-        ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+        ContainerLaunchContext launchContext = recordFactory
+            .newRecordInstance(ContainerLaunchContext.class);
         launchContext.setContainerId(firstContainerID);
         launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
         launchContext.getResource().setMemory(2);
@@ -121,11 +172,9 @@ public class TestNodeStatusUpdater {
       } else if (heartBeatID == 2) {
         // Checks on the RM end
         Assert.assertEquals("Number of applications should only be one!", 1,
-            nodeStatus.getAllContainers().size());
+            nodeStatus.getContainersStatuses().size());
         Assert.assertEquals("Number of container for the app should be one!",
-            1, nodeStatus.getContainers(applicationID).size());
-        Assert.assertEquals(2, nodeStatus.getContainers(applicationID).get(0)
-            .getResource().getMemory());
+            1, appToContainers.get(applicationID).size());
 
         // Checks on the NM end
         ConcurrentMap<ContainerId, Container> activeContainers =
@@ -135,10 +184,10 @@ public class TestNodeStatusUpdater {
         // Give another container to the NM.
         applicationID.setId(heartBeatID);
         appAttemptID.setApplicationId(applicationID);
-        secondContainerID.setAppId(applicationID);
-        secondContainerID.setAppAttemptId(appAttemptID);
+        secondContainerID.setApplicationAttemptId(appAttemptID);
         secondContainerID.setId(heartBeatID);
-        ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+        ContainerLaunchContext launchContext = recordFactory
+            .newRecordInstance(ContainerLaunchContext.class);
         launchContext.setContainerId(secondContainerID);
         launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
         launchContext.getResource().setMemory(3);
@@ -147,23 +196,21 @@ public class TestNodeStatusUpdater {
       } else if (heartBeatID == 3) {
         // Checks on the RM end
         Assert.assertEquals("Number of applications should only be one!", 1,
-            nodeStatus.getAllContainers().size());
+            appToContainers.size());
         Assert.assertEquals("Number of container for the app should be two!",
-            2, nodeStatus.getContainers(applicationID).size());
-        Assert.assertEquals(2, nodeStatus.getContainers(applicationID).get(0)
-            .getResource().getMemory());
-        Assert.assertEquals(3, nodeStatus.getContainers(applicationID).get(1)
-            .getResource().getMemory());
+            2, appToContainers.get(applicationID).size());
 
         // Checks on the NM end
         ConcurrentMap<ContainerId, Container> activeContainers =
             this.context.getContainers();
         Assert.assertEquals(2, activeContainers.size());
       }
-      HeartbeatResponse response = recordFactory.newRecordInstance(HeartbeatResponse.class);
+      HeartbeatResponse response = recordFactory
+          .newRecordInstance(HeartbeatResponse.class);
       response.setResponseId(heartBeatID);
       
-      NodeHeartbeatResponse nhResponse = recordFactory.newRecordInstance(NodeHeartbeatResponse.class);
+      NodeHeartbeatResponse nhResponse = recordFactory
+          .newRecordInstance(NodeHeartbeatResponse.class);
       nhResponse.setHeartbeatResponse(response);
       return nhResponse;
     }
@@ -173,8 +220,10 @@ public class TestNodeStatusUpdater {
     private Context context;
 
     public MyNodeStatusUpdater(Context context, Dispatcher dispatcher,
-        NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
-      super(context, dispatcher, healthChecker, metrics);
+        NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
+        ContainerTokenSecretManager containerTokenSecretManager) {
+      super(context, dispatcher, healthChecker, metrics,
+          containerTokenSecretManager);
       this.context = context;
     }
 
@@ -200,21 +249,23 @@ public class TestNodeStatusUpdater {
     final NodeManager nm = new NodeManager() {
       @Override
       protected NodeStatusUpdater createNodeStatusUpdater(Context context,
-          Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+          Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
+          ContainerTokenSecretManager containerTokenSecretManager) {
         return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
-                                       metrics);
+                                       metrics, containerTokenSecretManager);
       }
     };
 
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.setInt(NMConfig.NM_VMEM_GB, 5); // 5GB
-    conf.set(NMConfig.NM_BIND_ADDRESS, "127.0.0.1:12345");
-    conf.set(NMConfig.NM_LOCALIZER_BIND_ADDRESS, "127.0.0.1:12346");
-    conf.set(NMConfig.NM_LOG_DIR, new Path(basedir, "logs").toUri().getPath());
-    conf.set(NMConfig.REMOTE_USER_LOG_DIR, new Path(basedir, "remotelogs")
-        .toUri().getPath());
-    conf.set(NMConfig.NM_LOCAL_DIR, new Path(basedir, "nm0").toUri().getPath());
+    YarnConfiguration conf = createNMConfig();
     nm.init(conf);
+
+    // verify that the last service is the nodeStatusUpdater (ie registration
+    // with RM)
+    Object[] services  = nm.getServices().toArray();
+    Object lastService = services[services.length-1];
+    Assert.assertTrue("last service is NOT the node status updater",
+        lastService instanceof NodeStatusUpdater);
+
     new Thread() {
       public void run() {
         try {
@@ -244,7 +295,75 @@ public class TestNodeStatusUpdater {
     while (heartBeatID <= 3) {
       Thread.sleep(500);
     }
+    Assert.assertEquals("Number of registered NMs is wrong!!", 1,
+        this.registeredNodes.size());
 
     nm.stop();
   }
+
+  /**
+   * Verifies that if for some reason NM fails to start ContainerManager RPC
+   * server, RM is oblivious to NM's presence. The behaviour is like this
+   * because otherwise, NM will report to RM even if all its servers are not
+   * started properly, RM will think that the NM is alive and will retire the NM
+   * only after NM_EXPIRY interval. See MAPREDUCE-2749.
+   */
+  @Test
+  public void testNoRegistrationWhenNMServicesFail() {
+
+    final NodeManager nm = new NodeManager() {
+      @Override
+      protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+          Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
+          ContainerTokenSecretManager containerTokenSecretManager) {
+        return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
+                                       metrics, containerTokenSecretManager);
+      }
+
+      @Override
+      protected ContainerManagerImpl createContainerManager(Context context,
+          ContainerExecutor exec, DeletionService del,
+          NodeStatusUpdater nodeStatusUpdater,
+          ContainerTokenSecretManager containerTokenSecretManager) {
+        return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
+            metrics, containerTokenSecretManager) {
+          @Override
+          public void start() {
+            // Simulating failure of starting RPC server
+            throw new YarnException("Starting of RPC Server failed");
+          }
+        };
+      }
+    };
+
+    YarnConfiguration conf = createNMConfig();
+    nm.init(conf);
+    try {
+      nm.start();
+      Assert.fail("NM should have failed to start. Didn't get exception!!");
+    } catch (Exception e) {
+      Assert.assertEquals("Starting of RPC Server failed", e.getCause()
+          .getMessage());
+    }
+
+    Assert.assertEquals("NM state is wrong!", Service.STATE.INITED, nm
+        .getServiceState());
+
+    Assert.assertEquals("Number of registered nodes is wrong!", 0,
+        this.registeredNodes.size());
+  }
+
+  private YarnConfiguration createNMConfig() {
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.setInt(YarnConfiguration.NM_VMEM_GB, 5); // 5GB
+    conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
+    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
+    conf.set(YarnConfiguration.NM_LOG_DIRS, new Path(basedir, "logs").toUri()
+        .getPath());
+    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, new Path(basedir,
+        "remotelogs").toUri().getPath());
+    conf.set(YarnConfiguration.NM_LOCAL_DIRS, new Path(basedir, "nm0")
+        .toUri().getPath());
+    return conf;
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBLocalizerRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBLocalizerRPC.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBLocalizerRPC.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBLocalizerRPC.java Tue Sep 13 22:49:27 2011
@@ -22,6 +22,7 @@ import java.net.InetSocketAddress;
 
 import org.apache.avro.ipc.Server;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -39,7 +40,7 @@ public class TestPBLocalizerRPC {
 
   static RecordFactory createPBRecordFactory() {
     Configuration conf = new Configuration();
-    conf.set(RecordFactoryProvider.RPC_SERIALIZER_KEY, "protocolbuffers");
+    conf.set(YarnConfiguration.IPC_SERIALIZER_TYPE, "protocolbuffers");
     return RecordFactoryProvider.getRecordFactory(conf);
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java Tue Sep 13 22:49:27 2011
@@ -24,6 +24,7 @@ import org.apache.hadoop.io.DataOutputBu
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.RPCUtil;
@@ -46,7 +47,7 @@ public class TestPBRecordImpl {
 
   static RecordFactory createPBRecordFactory() {
     Configuration conf = new Configuration();
-    conf.set(RecordFactoryProvider.RPC_SERIALIZER_KEY, "protocolbuffers");
+    conf.set(YarnConfiguration.IPC_SERIALIZER_TYPE, "protocolbuffers");
     return RecordFactoryProvider.getRecordFactory(conf);
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Tue Sep 13 22:49:27 2011
@@ -41,13 +41,13 @@ import org.apache.hadoop.yarn.event.Asyn
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.api.ResourceTracker;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
 import org.apache.hadoop.yarn.server.nodemanager.LocalRMInterface;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
 import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
 import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
@@ -68,6 +68,7 @@ public abstract class BaseContainerManag
   protected static File localLogDir;
   protected static File remoteLogDir;
   protected static File tmpDir;
+  protected ContainerTokenSecretManager containerTokenSecretManager = new ContainerTokenSecretManager();
 
   protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
 
@@ -95,7 +96,7 @@ public abstract class BaseContainerManag
   protected String user = "nobody";
 
   protected NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl(
-      context, new AsyncDispatcher(), null, metrics) {
+      context, new AsyncDispatcher(), null, metrics, this.containerTokenSecretManager) {
     @Override
     protected ResourceTracker getRMClient() {
       return new LocalRMInterface();
@@ -129,10 +130,10 @@ public abstract class BaseContainerManag
     LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());
 
     String bindAddress = "0.0.0.0:5555";
-    conf.set(NMConfig.NM_BIND_ADDRESS, bindAddress);
-    conf.set(NMConfig.NM_LOCAL_DIR, localDir.getAbsolutePath());
-    conf.set(NMConfig.NM_LOG_DIR, localLogDir.getAbsolutePath());
-    conf.set(NMConfig.REMOTE_USER_LOG_DIR, remoteLogDir.getAbsolutePath());
+    conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
+    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
+    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());
 
     // Default delSrvc
     delSrvc = new DeletionService(exec) {
@@ -148,7 +149,7 @@ public abstract class BaseContainerManag
     exec = createContainerExecutor();
     containerManager =
         new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
-                                 metrics);
+                                 metrics, this.containerTokenSecretManager);
     containerManager.init(conf);
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java Tue Sep 13 22:49:27 2011
@@ -27,6 +27,7 @@ import java.util.Map;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@@ -95,10 +96,10 @@ public class TestAuxServices {
   @Test
   public void testAuxEventDispatch() {
     Configuration conf = new Configuration();
-    conf.setStrings(AuxServices.AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
-    conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Asrv"),
+    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
+    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
         ServiceA.class, Service.class);
-    conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Bsrv"),
+    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
         ServiceB.class, Service.class);
     conf.setInt("A.expected.init", 1);
     conf.setInt("B.expected.stop", 1);
@@ -123,10 +124,10 @@ public class TestAuxServices {
   @Test
   public void testAuxServices() {
     Configuration conf = new Configuration();
-    conf.setStrings(AuxServices.AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
-    conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Asrv"),
+    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
+    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
         ServiceA.class, Service.class);
-    conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Bsrv"),
+    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
         ServiceB.class, Service.class);
     final AuxServices aux = new AuxServices();
     aux.init(conf);
@@ -154,10 +155,10 @@ public class TestAuxServices {
   @Test
   public void testAuxServicesMeta() {
     Configuration conf = new Configuration();
-    conf.setStrings(AuxServices.AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
-    conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Asrv"),
+    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
+    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
         ServiceA.class, Service.class);
-    conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Bsrv"),
+    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
         ServiceB.class, Service.class);
     final AuxServices aux = new AuxServices();
     aux.init(conf);
@@ -191,10 +192,10 @@ public class TestAuxServices {
   @Test
   public void testAuxUnexpectedStop() {
     Configuration conf = new Configuration();
-    conf.setStrings(AuxServices.AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
-    conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Asrv"),
+    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
+    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
         ServiceA.class, Service.class);
-    conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Bsrv"),
+    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
         ServiceB.class, Service.class);
     final AuxServices aux = new AuxServices();
     aux.init(conf);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Tue Sep 13 22:49:27 2011
@@ -23,7 +23,11 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import junit.framework.Assert;
 
@@ -53,6 +57,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.junit.Test;
 
@@ -66,6 +71,20 @@ public class TestContainerManager extend
     LOG = LogFactory.getLog(TestContainerManager.class);
   }
 
+  private ContainerId createContainerId() {
+    ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
+    appId.setClusterTimestamp(0);
+    appId.setId(0);
+    ApplicationAttemptId appAttemptId = 
+        recordFactory.newRecordInstance(ApplicationAttemptId.class);
+    appAttemptId.setApplicationId(appId);
+    appAttemptId.setAttemptId(1);
+    ContainerId containerId = 
+        recordFactory.newRecordInstance(ContainerId.class);
+    containerId.setApplicationAttemptId(appAttemptId);
+    return containerId;
+  }
+  
   @Test
   public void testContainerManagerInitialization() throws IOException {
 
@@ -74,14 +93,9 @@ public class TestContainerManager extend
     // Just do a query for a non-existing container.
     boolean throwsException = false;
     try {
-      GetContainerStatusRequest request = recordFactory.newRecordInstance(GetContainerStatusRequest.class);
-      ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
-      ApplicationAttemptId appAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
-      appAttemptId.setApplicationId(appId);
-      appAttemptId.setAttemptId(1);
-      ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
-      cId.setAppId(appId);
-      cId.setAppAttemptId(appAttemptId);
+      GetContainerStatusRequest request = 
+          recordFactory.newRecordInstance(GetContainerStatusRequest.class);
+      ContainerId cId = createContainerId();
       request.setContainerId(cId);
       containerManager.getContainerStatus(request);
     } catch (YarnRemoteException e) {
@@ -106,20 +120,14 @@ public class TestContainerManager extend
     ContainerLaunchContext container = recordFactory.newRecordInstance(ContainerLaunchContext.class);
 
     // ////// Construct the Container-id
-    ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
-    ApplicationAttemptId appAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    appAttemptId.setApplicationId(appId);
-    appAttemptId.setAttemptId(1);
-    ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
-    cId.setAppId(appId);
-    cId.setAppAttemptId(appAttemptId);
+    ContainerId cId = createContainerId();
     container.setContainerId(cId);
 
     container.setUser(user);
 
     // ////// Construct the container-spec.
-    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
-//    containerLaunchContext.resources = new HashMap<CharSequence, LocalResource>();
+    ContainerLaunchContext containerLaunchContext = 
+        recordFactory.newRecordInstance(ContainerLaunchContext.class);
     URL resource_alpha =
         ConverterUtils.getYarnUrlFromPath(localFS
             .makeQualified(new Path(file.getAbsolutePath())));
@@ -130,14 +138,17 @@ public class TestContainerManager extend
     rsrc_alpha.setType(LocalResourceType.FILE);
     rsrc_alpha.setTimestamp(file.lastModified());
     String destinationFile = "dest_file";
-    containerLaunchContext.setLocalResource(destinationFile, rsrc_alpha);
+    Map<String, LocalResource> localResources = 
+        new HashMap<String, LocalResource>();
+    localResources.put(destinationFile, rsrc_alpha);
+    containerLaunchContext.setLocalResources(localResources);
     containerLaunchContext.setUser(container.getUser());
     containerLaunchContext.setContainerId(container.getContainerId());
     containerLaunchContext.setResource(recordFactory
         .newRecordInstance(Resource.class));
-//    containerLaunchContext.command = new ArrayList<CharSequence>();
 
-    StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
+    StartContainerRequest startRequest = 
+        recordFactory.newRecordInstance(StartContainerRequest.class);
     startRequest.setContainerLaunchContext(containerLaunchContext);
     
     containerManager.startContainer(startRequest);
@@ -146,7 +157,7 @@ public class TestContainerManager extend
         ContainerState.COMPLETE);
 
     // Now ascertain that the resources are localised correctly.
-    // TODO: Don't we need clusterStamp in localDir?
+    ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
     String appIDStr = ConverterUtils.toString(appId);
     String containerIDStr = ConverterUtils.toString(cId);
     File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
@@ -186,41 +197,41 @@ public class TestContainerManager extend
     PrintWriter fileWriter = new PrintWriter(scriptFile);
     File processStartFile =
         new File(tmpDir, "start_file.txt").getAbsoluteFile();
-    fileWriter.write("\numask 0"); // So that start file is readable by the test.
+    fileWriter.write("\numask 0"); // So that start file is readable by the test
     fileWriter.write("\necho Hello World! > " + processStartFile);
     fileWriter.write("\necho $$ >> " + processStartFile);
     fileWriter.write("\nexec sleep 100");
     fileWriter.close();
 
-    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+    ContainerLaunchContext containerLaunchContext = 
+        recordFactory.newRecordInstance(ContainerLaunchContext.class);
 
     // ////// Construct the Container-id
-    ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
-    ApplicationAttemptId appAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    appAttemptId.setApplicationId(appId);
-    appAttemptId.setAttemptId(1);
-    ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
-    cId.setAppId(appId);
-    cId.setAppAttemptId(appAttemptId);
+    ContainerId cId = createContainerId();
     containerLaunchContext.setContainerId(cId);
 
     containerLaunchContext.setUser(user);
 
-//    containerLaunchContext.resources =new HashMap<CharSequence, LocalResource>();
     URL resource_alpha =
         ConverterUtils.getYarnUrlFromPath(localFS
             .makeQualified(new Path(scriptFile.getAbsolutePath())));
-    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
+    LocalResource rsrc_alpha =
+        recordFactory.newRecordInstance(LocalResource.class);
     rsrc_alpha.setResource(resource_alpha);
     rsrc_alpha.setSize(-1);
     rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
     rsrc_alpha.setType(LocalResourceType.FILE);
     rsrc_alpha.setTimestamp(scriptFile.lastModified());
     String destinationFile = "dest_file";
-    containerLaunchContext.setLocalResource(destinationFile, rsrc_alpha);
+    Map<String, LocalResource> localResources = 
+        new HashMap<String, LocalResource>();
+    localResources.put(destinationFile, rsrc_alpha);
+    containerLaunchContext.setLocalResources(localResources);
     containerLaunchContext.setUser(containerLaunchContext.getUser());
-    containerLaunchContext.addCommand("/bin/bash");
-    containerLaunchContext.addCommand(scriptFile.getAbsolutePath());
+    List<String> commands = new ArrayList<String>();
+    commands.add("/bin/bash");
+    commands.add(scriptFile.getAbsolutePath());
+    containerLaunchContext.setCommands(commands);
     containerLaunchContext.setResource(recordFactory
         .newRecordInstance(Resource.class));
     containerLaunchContext.getResource().setMemory(100 * 1024 * 1024);
@@ -263,10 +274,12 @@ public class TestContainerManager extend
     BaseContainerManagerTest.waitForContainerState(containerManager, cId,
         ContainerState.COMPLETE);
     
-    GetContainerStatusRequest gcsRequest = recordFactory.newRecordInstance(GetContainerStatusRequest.class);
+    GetContainerStatusRequest gcsRequest = 
+        recordFactory.newRecordInstance(GetContainerStatusRequest.class);
     gcsRequest.setContainerId(cId);
-    ContainerStatus containerStatus = containerManager.getContainerStatus(gcsRequest).getStatus();
-    Assert.assertEquals(String.valueOf(ExitCode.KILLED.getExitCode()),
+    ContainerStatus containerStatus = 
+        containerManager.getContainerStatus(gcsRequest).getStatus();
+    Assert.assertEquals(ExitCode.KILLED.getExitCode(),
         containerStatus.getExitStatus());
 
     // Assert that the process is not alive anymore
@@ -281,8 +294,10 @@ public class TestContainerManager extend
     // Real del service
     delSrvc = new DeletionService(exec);
     delSrvc.init(conf);
+    ContainerTokenSecretManager containerTokenSecretManager = new 
+        ContainerTokenSecretManager();
     containerManager = new ContainerManagerImpl(context, exec, delSrvc,
-        nodeStatusUpdater, metrics);
+        nodeStatusUpdater, metrics, containerTokenSecretManager);
     containerManager.init(conf);
     containerManager.start();
 
@@ -297,13 +312,8 @@ public class TestContainerManager extend
     ContainerLaunchContext container = recordFactory.newRecordInstance(ContainerLaunchContext.class);
 
     // ////// Construct the Container-id
-    ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
-    ApplicationAttemptId appAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    appAttemptId.setApplicationId(appId);
-    appAttemptId.setAttemptId(1);
-    ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
-    cId.setAppId(appId);
-    cId.setAppAttemptId(appAttemptId);
+    ContainerId cId = createContainerId();
+    ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
     container.setContainerId(cId);
 
     container.setUser(user);
@@ -322,7 +332,10 @@ public class TestContainerManager extend
     rsrc_alpha.setType(LocalResourceType.FILE);
     rsrc_alpha.setTimestamp(file.lastModified());
     String destinationFile = "dest_file";
-    containerLaunchContext.setLocalResource(destinationFile, rsrc_alpha);
+    Map<String, LocalResource> localResources = 
+        new HashMap<String, LocalResource>();
+    localResources.put(destinationFile, rsrc_alpha);
+    containerLaunchContext.setLocalResources(localResources);
     containerLaunchContext.setUser(container.getUser());
     containerLaunchContext.setContainerId(container.getContainerId());
     containerLaunchContext.setResource(recordFactory
@@ -337,7 +350,8 @@ public class TestContainerManager extend
     BaseContainerManagerTest.waitForContainerState(containerManager, cId,
         ContainerState.COMPLETE);
 
-    BaseContainerManagerTest.waitForApplicationState(containerManager, cId.getAppId(),
+    BaseContainerManagerTest.waitForApplicationState(containerManager, 
+        cId.getApplicationAttemptId().getApplicationId(),
         ApplicationState.RUNNING);
 
     // Now ascertain that the resources are localised correctly.
@@ -369,7 +383,8 @@ public class TestContainerManager extend
     containerManager.handle(new CMgrCompletedAppsEvent(Arrays
         .asList(new ApplicationId[] { appId })));
 
-    BaseContainerManagerTest.waitForApplicationState(containerManager, cId.getAppId(),
+    BaseContainerManagerTest.waitForApplicationState(containerManager, 
+        cId.getApplicationAttemptId().getApplicationId(),
         ApplicationState.FINISHED);
 
     // Now ascertain that the resources are localised correctly.



Mime
View raw message