hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1196458 [10/19] - in /hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ assembly/ bin/ conf/ dev-support/ hadoop-mapreduce-client/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/ hadoop-mapreduce-client/hadoop-mapreduce-cl...
Date Wed, 02 Nov 2011 05:35:03 GMT
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java Wed Nov  2 05:34:31 2011
@@ -84,20 +84,23 @@ public class AsyncDispatcher extends Abs
     //start all the components
     super.start();
     eventHandlingThread = new Thread(createThread());
+    eventHandlingThread.setName("AsyncDispatcher event handler");
     eventHandlingThread.start();
   }
 
   @Override
   public void stop() {
     stopped = true;
-    eventHandlingThread.interrupt();
-    try {
-      eventHandlingThread.join();
-    } catch (InterruptedException ie) {
-      LOG.debug("Interruped Exception while stopping", ie);
+    if (eventHandlingThread != null) {
+      eventHandlingThread.interrupt();
+      try {
+        eventHandlingThread.join();
+      } catch (InterruptedException ie) {
+        LOG.debug("Interrupted Exception while stopping", ie);
+      }
     }
 
-    //stop all the components
+    // stop all the components
     super.stop();
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java Wed Nov  2 05:34:31 2011
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.factories
 import java.net.InetSocketAddress;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.YarnException;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java Wed Nov  2 05:34:31 2011
@@ -27,8 +27,8 @@ import java.util.concurrent.ConcurrentHa
 import java.util.concurrent.ConcurrentMap;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RPC.Server;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.YarnException;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java Wed Nov  2 05:34:31 2011
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.ipc;
 
 import java.net.InetSocketAddress;
 
-import org.apache.avro.ipc.Server;
+import org.apache.hadoop.ipc.Server;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -58,34 +58,9 @@ public class HadoopYarnProtoRPC extends 
     LOG.info("Creating a HadoopYarnProtoRpc server for protocol " + protocol + 
         " with " + numHandlers + " handlers");
     
-    final RPC.Server hadoopServer;
-    hadoopServer = 
-      RpcFactoryProvider.getServerFactory(conf).getServer(protocol, instance, 
+    return RpcFactoryProvider.getServerFactory(conf).getServer(protocol, instance, 
           addr, conf, secretManager, numHandlers);
 
-    Server server = new Server() {
-      @Override
-      public void close() {
-        hadoopServer.stop();
-      }
-
-      @Override
-      public int getPort() {
-        return hadoopServer.getListenerAddress().getPort();
-      }
-
-      @Override
-      public void join() throws InterruptedException {
-        hadoopServer.join();
-      }
-
-      @Override
-      public void start() {
-        hadoopServer.start();
-      }
-    };
-    return server;
-
   }
 
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java Wed Nov  2 05:34:31 2011
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.ipc;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 
-import org.apache.avro.ipc.Server;
+import org.apache.hadoop.ipc.Server;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -30,7 +30,6 @@ import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
 /**
  * This uses Hadoop RPC. Uses a tunnel AvroSpecificRpcEngine over 
@@ -70,29 +69,7 @@ public class HadoopYarnRPC extends YarnR
     } catch (IOException e) {
       throw new YarnException(e);
     }
-    Server server = new Server() {
-      @Override
-      public void close() {
-        hadoopServer.stop();
-      }
-
-      @Override
-      public int getPort() {
-        return hadoopServer.getListenerAddress().getPort();
-      }
-
-      @Override
-      public void join() throws InterruptedException {
-        hadoopServer.join();
-      }
-
-      @Override
-      public void start() {
-        hadoopServer.start();
-      }
-    };
-    return server;
-
+    return hadoopServer;
   }
 
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java Wed Nov  2 05:34:31 2011
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.ipc;
 
 import java.net.InetSocketAddress;
 
-import org.apache.avro.ipc.Server;
+import org.apache.hadoop.ipc.Server;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java Wed Nov  2 05:34:31 2011
@@ -22,43 +22,35 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 
-// TODO: Make it avro-ish. TokenIdentifier really isn't serialized
-// as writable but simply uses readFields method in SaslRpcServer
-// for deserializatoin.
 public class ApplicationTokenIdentifier extends TokenIdentifier {
 
   public static final Text KIND_NAME = new Text("YARN_APPLICATION_TOKEN");
 
-  private Text appId;
-
-  // TODO: Add more information in the tokenID such that it is not
-  // transferrable, more secure etc.
-
-  public ApplicationTokenIdentifier(ApplicationId id) {
-    this.appId = new Text(Integer.toString(id.getId()));
-  }
+  private String applicationAttemptId;
 
   public ApplicationTokenIdentifier() {
-    this.appId = new Text();
   }
 
-  public Text getApplicationID() {
-    return appId;
+  public ApplicationTokenIdentifier(ApplicationAttemptId appAttemptId) {
+    this();
+    this.applicationAttemptId = appAttemptId.toString();
   }
 
   @Override
   public void write(DataOutput out) throws IOException {
-    appId.write(out);
+    Text.writeString(out, this.applicationAttemptId);
   }
 
   @Override
   public void readFields(DataInput in) throws IOException {
-    appId.readFields(in);
+    this.applicationAttemptId = Text.readString(in);
   }
 
   @Override
@@ -68,10 +60,19 @@ public class ApplicationTokenIdentifier 
 
   @Override
   public UserGroupInformation getUser() {
-    if (appId == null || "".equals(appId.toString())) {
+    if (this.applicationAttemptId == null
+        || "".equals(this.applicationAttemptId.toString())) {
       return null;
     }
-    return UserGroupInformation.createRemoteUser(appId.toString());
+    return UserGroupInformation.createRemoteUser(this.applicationAttemptId
+        .toString());
   }
 
+  @InterfaceAudience.Private
+  public static class Renewer extends Token.TrivialRenewer {
+    @Override
+    protected Text getKind() {
+      return KIND_NAME;
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java Wed Nov  2 05:34:31 2011
@@ -24,30 +24,31 @@ import java.io.IOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.util.BuilderUtils;
 
 public class ContainerTokenIdentifier extends TokenIdentifier {
 
-  private static Log LOG = LogFactory
-      .getLog(ContainerTokenIdentifier.class);
+  private static Log LOG = LogFactory.getLog(ContainerTokenIdentifier.class);
 
   public static final Text KIND = new Text("ContainerToken");
 
   private ContainerId containerId;
-  private String nmHostName;
+  private String nmHostAddr;
   private Resource resource;
 
   public ContainerTokenIdentifier(ContainerId containerID, String hostName,
       Resource r) {
     this.containerId = containerID;
-    this.nmHostName = hostName;
+    this.nmHostAddr = hostName;
     this.resource = r;
   }
 
@@ -55,59 +56,46 @@ public class ContainerTokenIdentifier ex
   }
 
   public ContainerId getContainerID() {
-    return containerId;
+    return this.containerId;
   }
 
-  public String getNmHostName() {
-    return nmHostName;
+  public String getNmHostAddress() {
+    return this.nmHostAddr;
   }
 
   public Resource getResource() {
-    return resource;
+    return this.resource;
   }
 
   @Override
   public void write(DataOutput out) throws IOException {
-    LOG.debug("Writing ContainerTokenIdentifier to RPC layer");
-    ApplicationAttemptId applicationAttemptId = 
-        containerId.getApplicationAttemptId();
+    LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this);
+    ApplicationAttemptId applicationAttemptId = this.containerId
+        .getApplicationAttemptId();
     ApplicationId applicationId = applicationAttemptId.getApplicationId();
     out.writeLong(applicationId.getClusterTimestamp());
     out.writeInt(applicationId.getId());
     out.writeInt(applicationAttemptId.getAttemptId());
     out.writeInt(this.containerId.getId());
-    out.writeUTF(this.nmHostName);
+    out.writeUTF(this.nmHostAddr);
     out.writeInt(this.resource.getMemory());
   }
 
   @Override
   public void readFields(DataInput in) throws IOException {
-    this.containerId = 
-        RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
-            ContainerId.class);
-    ApplicationAttemptId applicationAttemptId =
-        RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
-            ApplicationAttemptId.class);
-    ApplicationId applicationId =
-        RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
-            ApplicationId.class);
-    applicationId.setClusterTimestamp(in.readLong());
-    applicationId.setId(in.readInt());
-    applicationAttemptId.setApplicationId(applicationId);
-    applicationAttemptId.setAttemptId(in.readInt());
-    this.containerId.setApplicationAttemptId(applicationAttemptId);
-    this.containerId.setId(in.readInt());
-    this.nmHostName = in.readUTF();
-    this.resource = 
-        RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
-            Resource.class);
-    this.resource.setMemory(in.readInt());
+    ApplicationId applicationId = BuilderUtils.newApplicationId(
+        in.readLong(), in.readInt());
+    ApplicationAttemptId applicationAttemptId = BuilderUtils
+        .newApplicationAttemptId(applicationId, in.readInt());
+    this.containerId = BuilderUtils.newContainerId(applicationAttemptId, in
+        .readInt());
+    this.nmHostAddr = in.readUTF();
+    this.resource = BuilderUtils.newResource(in.readInt());
   }
 
-  @SuppressWarnings("static-access")
   @Override
   public Text getKind() {
-    return this.KIND;
+    return KIND;
   }
 
   @Override
@@ -115,4 +103,11 @@ public class ContainerTokenIdentifier ex
     return UserGroupInformation.createRemoteUser(this.containerId.toString());
   }
 
+  @InterfaceAudience.Private
+  public static class Renewer extends Token.TrivialRenewer {
+    @Override
+    protected Text getKind() {
+      return KIND;
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java Wed Nov  2 05:34:31 2011
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.security;
 
 import java.util.Collection;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -28,6 +30,10 @@ import org.apache.hadoop.security.token.
 public class ContainerTokenSelector implements
     TokenSelector<ContainerTokenIdentifier> {
 
+  private static final Log LOG = LogFactory
+      .getLog(ContainerTokenSelector.class);
+
+  @SuppressWarnings("unchecked")
   @Override
   public Token<ContainerTokenIdentifier> selectToken(Text service,
       Collection<Token<? extends TokenIdentifier>> tokens) {
@@ -35,6 +41,10 @@ public class ContainerTokenSelector impl
       return null;
     }
     for (Token<? extends TokenIdentifier> token : tokens) {
+      if (LOG.isDebugEnabled()) {
+        LOG.info("Looking for service: " + service + ". Current token is "
+            + token);
+      }
       if (ContainerTokenIdentifier.KIND.equals(token.getKind()) && 
           service.equals(token.getService())) {
         return (Token<ContainerTokenIdentifier>) token;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMSecretManager.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMSecretManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMSecretManager.java Wed Nov  2 05:34:31 2011
@@ -28,17 +28,16 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.token.SecretManager;
-import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
 
 public class ClientToAMSecretManager extends
-    SecretManager<ApplicationTokenIdentifier> {
+    SecretManager<ClientTokenIdentifier> {
 
   private static Log LOG = LogFactory.getLog(ClientToAMSecretManager.class);
 
   // Per application masterkeys for managing client-tokens
   private Map<Text, SecretKey> masterKeys = new HashMap<Text, SecretKey>();
 
-  public void setMasterKey(ApplicationTokenIdentifier identifier, byte[] key) {
+  public void setMasterKey(ClientTokenIdentifier identifier, byte[] key) {
     SecretKey sk = SecretManager.createSecretKey(key);
     Text applicationID = identifier.getApplicationID();
     this.masterKeys.put(applicationID, sk);
@@ -51,7 +50,7 @@ public class ClientToAMSecretManager ext
     }
   }
 
-  private void addMasterKey(ApplicationTokenIdentifier identifier) {
+  private void addMasterKey(ClientTokenIdentifier identifier) {
     Text applicationID = identifier.getApplicationID();
     this.masterKeys.put(applicationID, generateSecret());
     if (LOG.isDebugEnabled()) {
@@ -64,7 +63,7 @@ public class ClientToAMSecretManager ext
 
   // TODO: Handle the masterKey invalidation.
   public synchronized SecretKey getMasterKey(
-      ApplicationTokenIdentifier identifier) {
+      ClientTokenIdentifier identifier) {
     Text applicationID = identifier.getApplicationID();
     if (!this.masterKeys.containsKey(applicationID)) {
       addMasterKey(identifier);
@@ -74,7 +73,7 @@ public class ClientToAMSecretManager ext
 
   @Override
   public synchronized byte[] createPassword(
-      ApplicationTokenIdentifier identifier) {
+      ClientTokenIdentifier identifier) {
     byte[] password =
         createPassword(identifier.getBytes(), getMasterKey(identifier));
     if (LOG.isDebugEnabled()) {
@@ -85,7 +84,7 @@ public class ClientToAMSecretManager ext
   }
 
   @Override
-  public byte[] retrievePassword(ApplicationTokenIdentifier identifier)
+  public byte[] retrievePassword(ClientTokenIdentifier identifier)
       throws SecretManager.InvalidToken {
     byte[] password =
         createPassword(identifier.getBytes(), getMasterKey(identifier));
@@ -97,8 +96,8 @@ public class ClientToAMSecretManager ext
   }
 
   @Override
-  public ApplicationTokenIdentifier createIdentifier() {
-    return new ApplicationTokenIdentifier();
+  public ClientTokenIdentifier createIdentifier() {
+    return new ClientTokenIdentifier();
   }
 
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/AbstractService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/AbstractService.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/AbstractService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/AbstractService.java Wed Nov  2 05:34:31 2011
@@ -63,8 +63,12 @@ public abstract class AbstractService im
 
   @Override
   public synchronized void stop() {
-    if (state == STATE.STOPPED) {
-      return;//already stopped
+    if (state == STATE.STOPPED ||
+        state == STATE.INITED ||
+        state == STATE.NOTINITED) {
+      // already stopped, or else it was never
+      // started (eg another service failing canceled startup)
+      return;
     }
     ensureCurrentState(STATE.STARTED);
     changeState(STATE.STOPPED);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/CompositeService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/CompositeService.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/CompositeService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/CompositeService.java Wed Nov  2 05:34:31 2011
@@ -108,6 +108,7 @@ public class CompositeService extends Ab
     private CompositeService compositeService;
 
     public CompositeServiceShutdownHook(CompositeService compositeService) {
+      super("CompositeServiceShutdownHook for " + compositeService.getName());
       this.compositeService = compositeService;
     }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AbstractLivelinessMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AbstractLivelinessMonitor.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AbstractLivelinessMonitor.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AbstractLivelinessMonitor.java Wed Nov  2 05:34:31 2011
@@ -50,7 +50,9 @@ public abstract class AbstractLiveliness
 
   @Override
   public void start() {
+    assert !stopped : "starting when already stopped";
     checkerThread = new Thread(new PingChecker());
+    checkerThread.setName("Ping Checker");
     checkerThread.start();
     super.start();
   }
@@ -58,7 +60,9 @@ public abstract class AbstractLiveliness
   @Override
   public void stop() {
     stopped = true;
-    checkerThread.interrupt();
+    if (checkerThread != null) {
+      checkerThread.interrupt();
+    }
     super.stop();
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java Wed Nov  2 05:34:31 2011
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.util;
 
 import java.util.Iterator;
+import java.util.Map;
 
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -59,4 +60,55 @@ public class Apps {
   public static void throwParseException(String name, String s) {
     throw new YarnException(join("Error parsing ", name, ": ", s));
   }
+
+  public static void setEnvFromInputString(Map<String, String> env,
+      String envString) {
+    if (envString != null && envString.length() > 0) {
+      String childEnvs[] = envString.split(",");
+      for (String cEnv : childEnvs) {
+        String[] parts = cEnv.split("="); // split on '='
+        String value = env.get(parts[0]);
+
+        if (value != null) {
+          // Replace $env with the child's env constructed by NM's
+          // For example: LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp
+          value = parts[1].replace("$" + parts[0], value);
+        } else {
+          // example PATH=$PATH:/tmp
+          value = System.getenv(parts[0]);
+          if (value != null) {
+            // the env key is present in the tt's env
+            value = parts[1].replace("$" + parts[0], value);
+          } else {
+            // check for simple variable substitution
+            // for e.g. ROOT=$HOME
+            String envValue = System.getenv(parts[1].substring(1));
+            if (envValue != null) {
+              value = envValue;
+            } else {
+              // the env key is note present anywhere .. simply set it
+              // example X=$X:/tmp or X=/tmp
+              value = parts[1].replace("$" + parts[0], "");
+            }
+          }
+        }
+        addToEnvironment(env, parts[0], value);
+      }
+    }
+  }
+
+  private static final String SYSTEM_PATH_SEPARATOR =
+      System.getProperty("path.separator");
+
+  public static void addToEnvironment(
+      Map<String, String> environment,
+      String variable, String value) {
+    String val = environment.get(variable);
+    if (val == null) {
+      val = value;
+    } else {
+      val = val + SYSTEM_PATH_SEPARATOR + value;
+    }
+    environment.put(variable, val);
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Wed Nov  2 05:34:31 2011
@@ -18,20 +18,27 @@
 
 package org.apache.hadoop.yarn.util;
 
+import java.net.InetSocketAddress;
 import java.net.URI;
+import java.nio.ByteBuffer;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 
+import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -40,8 +47,10 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
 
 /**
  * Builder utilities to construct various objects.
@@ -194,18 +203,9 @@ public class BuilderUtils {
     return nodeId;
   }
 
-  public static Container newContainer(RecordFactory recordFactory,
-      ApplicationAttemptId appAttemptId, int containerId, NodeId nodeId,
-      String nodeHttpAddress, Resource resource, Priority priority) {
-    ContainerId containerID =
-        newContainerId(recordFactory, appAttemptId, containerId);
-    return newContainer(containerID, nodeId, nodeHttpAddress,
-        resource, priority);
-  }
-
   public static Container newContainer(ContainerId containerId,
       NodeId nodeId, String nodeHttpAddress,
-      Resource resource, Priority priority) {
+      Resource resource, Priority priority, ContainerToken containerToken) {
     Container container = recordFactory.newRecordInstance(Container.class);
     container.setId(containerId);
     container.setNodeId(nodeId);
@@ -217,9 +217,51 @@ public class BuilderUtils {
     containerStatus.setContainerId(containerId);
     containerStatus.setState(ContainerState.NEW);
     container.setContainerStatus(containerStatus);
+    container.setContainerToken(containerToken);
     return container;
   }
 
+  public static ContainerToken newContainerToken(NodeId nodeId,
+      ByteBuffer password, ContainerTokenIdentifier tokenIdentifier) {
+    ContainerToken containerToken = recordFactory
+        .newRecordInstance(ContainerToken.class);
+    containerToken.setIdentifier(ByteBuffer.wrap(tokenIdentifier.getBytes()));
+    containerToken.setKind(ContainerTokenIdentifier.KIND.toString());
+    containerToken.setPassword(password);
+    // RPC layer client expects ip:port as service for tokens
+    InetSocketAddress addr = NetUtils.createSocketAddr(nodeId.getHost(),
+        nodeId.getPort());
+    containerToken.setService(addr.getAddress().getHostAddress() + ":"
+        + addr.getPort());
+    return containerToken;
+  }
+
+  public static ContainerLaunchContext newContainerLaunchContext(
+      ContainerId containerID, String user, Resource assignedCapability,
+      Map<String, LocalResource> localResources,
+      Map<String, String> environment, List<String> commands,
+      Map<String, ByteBuffer> serviceData, ByteBuffer containerTokens,
+      Map<ApplicationAccessType, String> acls) {
+    ContainerLaunchContext container = recordFactory
+        .newRecordInstance(ContainerLaunchContext.class);
+    container.setContainerId(containerID);
+    container.setUser(user);
+    container.setResource(assignedCapability);
+    container.setLocalResources(localResources);
+    container.setEnvironment(environment);
+    container.setCommands(commands);
+    container.setServiceData(serviceData);
+    container.setContainerTokens(containerTokens);
+    container.setApplicationACLs(acls);
+    return container;
+  }
+
+  public static Priority newPriority(int p) {
+    Priority priority = recordFactory.newRecordInstance(Priority.class);
+    priority.setPriority(p);
+    return priority;
+  }
+
   public static ResourceRequest newResourceRequest(Priority priority,
       String hostName, Resource capability, int numContainers) {
     ResourceRequest request = recordFactory
@@ -245,7 +287,8 @@ public class BuilderUtils {
       ApplicationId applicationId, String user, String queue, String name,
       String host, int rpcPort, String clientToken, YarnApplicationState state,
       String diagnostics, String url, long startTime, long finishTime,
-      FinalApplicationStatus finalStatus) {
+      FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources,
+      String origTrackingUrl) {
     ApplicationReport report = recordFactory
         .newRecordInstance(ApplicationReport.class);
     report.setApplicationId(applicationId);
@@ -261,6 +304,8 @@ public class BuilderUtils {
     report.setStartTime(startTime);
     report.setFinishTime(finishTime);
     report.setFinalApplicationStatus(finalStatus);
+    report.setApplicationResourceUsageReport(appResources);
+    report.setOriginalTrackingUrl(origTrackingUrl);
     return report;
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Wed Nov  2 05:34:31 2011
@@ -20,10 +20,8 @@ package org.apache.hadoop.yarn.util;
 
 import static org.apache.hadoop.yarn.util.StringHelper._split;
 
-import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.text.NumberFormat;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -33,9 +31,8 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 
@@ -97,27 +94,8 @@ public class ConverterUtils {
     return url;
   }
 
-  // TODO: Why thread local?
-  // ^ NumberFormat instances are not threadsafe
-  private static final ThreadLocal<NumberFormat> appIdFormat =
-    new ThreadLocal<NumberFormat>() {
-      @Override
-      public NumberFormat initialValue() {
-        NumberFormat fmt = NumberFormat.getInstance();
-        fmt.setGroupingUsed(false);
-        fmt.setMinimumIntegerDigits(4);
-        return fmt;
-      }
-    };
-
-  
-
   public static String toString(ApplicationId appId) {
-    StringBuilder sb = new StringBuilder();
-    sb.append(APPLICATION_PREFIX + "_").append(appId.getClusterTimestamp())
-        .append("_");
-    sb.append(appIdFormat.get().format(appId.getId()));
-    return sb.toString();
+    return appId.toString();
   }
 
   public static ApplicationId toApplicationId(RecordFactory recordFactory,
@@ -148,15 +126,38 @@ public class ConverterUtils {
     return appAttemptId;
   }
 
+  private static ApplicationId toApplicationId(
+      Iterator<String> it) throws NumberFormatException {
+    ApplicationId appId = Records.newRecord(ApplicationId.class);
+    appId.setClusterTimestamp(Long.parseLong(it.next()));
+    appId.setId(Integer.parseInt(it.next()));
+    return appId;
+  }
+
   public static String toString(ContainerId cId) {
     return cId.toString();
   }
 
-  public static ContainerId toContainerId(String containerIdStr)
-      throws IOException {
+  public static NodeId toNodeId(String nodeIdStr) {
+    String[] parts = nodeIdStr.split(":");
+    if (parts.length != 2) {
+      throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
+          + "]. Expected host:port");
+    }
+    try {
+      NodeId nodeId =
+          BuilderUtils.newNodeId(parts[0], Integer.parseInt(parts[1]));
+      return nodeId;
+    } catch (NumberFormatException e) {
+      throw new IllegalArgumentException("Invalid port: " + parts[1], e);
+    }
+  }
+
+  public static ContainerId toContainerId(String containerIdStr) {
     Iterator<String> it = _split(containerIdStr).iterator();
     if (!it.next().equals(CONTAINER_PREFIX)) {
-      throw new IOException("Invalid ContainerId prefix: " + containerIdStr);
+      throw new IllegalArgumentException("Invalid ContainerId prefix: "
+          + containerIdStr);
     }
     try {
       ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
@@ -165,23 +166,38 @@ public class ConverterUtils {
       containerId.setId(Integer.parseInt(it.next()));
       return containerId;
     } catch (NumberFormatException n) {
-      throw new IOException("Invalid ContainerId: " + containerIdStr, n);
+      throw new IllegalArgumentException("Invalid ContainerId: "
+          + containerIdStr, n);
     }
   }
 
   public static ApplicationAttemptId toApplicationAttemptId(
-      String applicationAttmeptIdStr) throws IOException {
+      String applicationAttmeptIdStr) {
     Iterator<String> it = _split(applicationAttmeptIdStr).iterator();
     if (!it.next().equals(APPLICATION_ATTEMPT_PREFIX)) {
-      throw new IOException("Invalid AppAttemptId prefix: "
+      throw new IllegalArgumentException("Invalid AppAttemptId prefix: "
           + applicationAttmeptIdStr);
     }
     try {
       return toApplicationAttemptId(it);
     } catch (NumberFormatException n) {
-      throw new IOException("Invalid AppAttemptId: "
+      throw new IllegalArgumentException("Invalid AppAttemptId: "
           + applicationAttmeptIdStr, n);
     }
   }
   
+  public static ApplicationId toApplicationId(
+      String appIdStr) {
+    Iterator<String> it = _split(appIdStr).iterator();
+    if (!it.next().equals(APPLICATION_PREFIX)) {
+      throw new IllegalArgumentException("Invalid ApplicationId prefix: "
+          + appIdStr);
+    }
+    try {
+      return toApplicationId(it);
+    } catch (NumberFormatException n) {
+      throw new IllegalArgumentException("Invalid AppAttemptId: "
+          + appIdStr, n);
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java Wed Nov  2 05:34:31 2011
@@ -154,15 +154,24 @@ public final class StringHelper {
         if (part.startsWith("#") || isAbsUrl(part)) {
           sb.append(part);
         } else {
-          sb.append('/').append(pathPrefix).append('/').append(part);
+          uappend(sb, pathPrefix);
+          uappend(sb, part);
         }
       } else {
-        sb.append('/').append(part);
+        uappend(sb, part);
       }
     }
     return sb.toString();
   }
-
+  
+  private static void uappend(StringBuilder sb, String part) {
+    if((sb.length() <= 0 || sb.charAt(sb.length() - 1) != '/') 
+        && !part.startsWith("/")) {
+      sb.append('/');
+    }
+    sb.append(part);
+  }
+  
   public static String percent(double value) {
     return String.format("%.2f", value * 100);
   }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Params.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Params.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Params.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Params.java Wed Nov  2 05:34:31 2011
@@ -28,4 +28,4 @@ public interface Params {
   static final String TITLE_LINK = "title.href";
   static final String USER = "user";
   static final String ERROR_DETAILS = "error.details";
-}
+}
\ No newline at end of file

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/View.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/View.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/View.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/View.java Wed Nov  2 05:34:31 2011
@@ -32,6 +32,7 @@ import javax.servlet.http.HttpServletRes
 
 import static org.apache.hadoop.yarn.util.StringHelper.*;
 
+import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -175,8 +176,20 @@ public abstract class View implements Pa
     moreParams().put(key, value);
   }
 
+  public String root() {
+    String root = System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV);
+    if(root == null || root.isEmpty()) {
+      root = "/";
+    }
+    return root;
+  }
+  
   public String prefix() {
-    return context().rc.prefix;
+    if(context().rc.prefix == null) {
+      return root();
+    } else {
+      return ujoin(root(), context().rc.prefix);
+    }
   }
 
   public void setTitle(String title) {
@@ -193,6 +206,16 @@ public abstract class View implements Pa
    * @param parts components to join
    * @return an url string
    */
+  public String root_url(String... parts) {
+    return ujoin(root(), parts);
+  }
+
+  
+  /**
+   * Create an url from url components
+   * @param parts components to join
+   * @return an url string
+   */
   public String url(String... parts) {
     return ujoin(prefix(), parts);
   }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Wed Nov  2 05:34:31 2011
@@ -18,23 +18,29 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import static com.google.common.base.Preconditions.*;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.google.inject.servlet.GuiceFilter;
+import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.io.IOException;
 import java.net.ConnectException;
 import java.net.URL;
-import org.apache.commons.lang.StringUtils;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import javax.servlet.http.HttpServlet;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpServer;
-
+import org.apache.hadoop.yarn.security.AdminACLsManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceFilter;
+
 /**
  * Helpers to create an embedded webapp.
  *
@@ -59,6 +65,12 @@ public class WebApps {
   static final Logger LOG = LoggerFactory.getLogger(WebApps.class);
 
   public static class Builder<T> {
+    static class ServletStruct {
+      public Class<? extends HttpServlet> clazz;
+      public String name;
+      public String spec;
+    }
+    
     final String name;
     final Class<T> api;
     final T application;
@@ -67,6 +79,8 @@ public class WebApps {
     boolean findPort = false;
     Configuration conf;
     boolean devMode = false;
+    private final HashSet<ServletStruct> servlets = new HashSet<ServletStruct>();
+    private final HashMap<String, Object> attributes = new HashMap<String, Object>();
 
     Builder(String name, Class<T> api, T application) {
       this.name = name;
@@ -93,6 +107,21 @@ public class WebApps {
       return this;
     }
 
+    public Builder<T> withAttribute(String key, Object value) {
+      attributes.put(key, value);
+      return this;
+    }
+    
+    public Builder<T> withServlet(String name, String pathSpec, 
+        Class<? extends HttpServlet> servlet) {
+      ServletStruct struct = new ServletStruct();
+      struct.clazz = servlet;
+      struct.name = name;
+      struct.spec = pathSpec;
+      servlets.add(struct);
+      return this;
+    }
+    
     public Builder<T> with(Configuration conf) {
       this.conf = conf;
       return this;
@@ -151,13 +180,21 @@ public class WebApps {
         }
         HttpServer server =
             new HttpServer(name, bindAddress, port, findPort, conf, 
-            webapp.getServePathSpecs());
+            new AdminACLsManager(conf).getAdminAcl(), null, webapp.getServePathSpecs());
+        for(ServletStruct struct: servlets) {
+          server.addServlet(struct.name, struct.spec, struct.clazz);
+        }
+        for(Map.Entry<String, Object> entry : attributes.entrySet()) {
+          server.setAttribute(entry.getKey(), entry.getValue());
+        }
         server.addGlobalFilter("guice", GuiceFilter.class.getName(), null);
         webapp.setConf(conf);
         webapp.setHttpServer(server);
         server.start();
         LOG.info("Web app /"+ name +" started at "+ server.getPort());
-      } catch (Exception e) {
+      } catch (ClassNotFoundException e) {
+        throw new WebAppException("Error starting http server", e);
+      } catch (IOException e) {
         throw new WebAppException("Error starting http server", e);
       }
       Injector injector = Guice.createInjector(webapp, new AbstractModule() {

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ErrorPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ErrorPage.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ErrorPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ErrorPage.java Wed Nov  2 05:34:31 2011
@@ -34,7 +34,7 @@ public class ErrorPage extends HtmlPage 
     String title = "Sorry, got error "+ status();
     html.
       title(title).
-      link("/static/yarn.css").
+      link(root_url("static","yarn.css")).
       _(JQueryUI.class). // an embedded sub-view
       style("#msg { margin: 1em auto; width: 88%; }",
             "#msg h1 { padding: 0.2em 1.5em; font: bold 1.3em serif; }").

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java Wed Nov  2 05:34:31 2011
@@ -81,12 +81,12 @@ public class JQueryUI extends HtmlBlock 
     html.
       link(join("https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.9/themes/",
                 getTheme(), "/jquery-ui.css")).
-      link("/static/dt-1.7.5/css/jui-dt.css").
+      link(root_url("static/dt-1.7.5/css/jui-dt.css")).
       script("https://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js").
       script("https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.9/jquery-ui.min.js").
-      script("/static/dt-1.7.5/js/jquery.dataTables.min.js").
-      script("/static/yarn.dt.plugins.js").
-      script("/static/themeswitcher.js").
+      script(root_url("static/dt-1.7.5/js/jquery.dataTables.min.js")).
+      script(root_url("static/yarn.dt.plugins.js")).
+      script(root_url("static/themeswitcher.js")).
       style("#jsnotice { padding: 0.2em; text-align: center; }",
             ".ui-progressbar { height: 1em; min-width: 5em }"); // required
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnCssLayout.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnCssLayout.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnCssLayout.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnCssLayout.java Wed Nov  2 05:34:31 2011
@@ -35,7 +35,7 @@ public class TwoColumnCssLayout extends 
     preHead(html);
     html.
       title($("title")).
-      link("/static/yarn.css").
+      link(root_url("static","yarn.css")).
       style(".main { min-height: 100%; height: auto !important; height: 100%;",
             "  margin: 0 auto -4em; border: 0; }",
             ".footer, .push { height: 4em; clear: both; border: 0 }",

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java Wed Nov  2 05:34:31 2011
@@ -41,7 +41,7 @@ public class TwoColumnLayout extends Htm
     preHead(html);
     html.
       title($(TITLE)).
-      link("/static/yarn.css").
+      link(root_url("static","yarn.css")).
       style("#layout { height: 100%; }",
             "#layout thead td { height: 3em; }",
             "#layout #navcell { width: 11em; padding: 0 1em; }",

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.css
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.css?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.css (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.css Wed Nov  2 05:34:31 2011
@@ -2,8 +2,6 @@
 * { margin: 0; border: 0 }
 html, body { height: 100% }
 body { padding: 0; font: 90% sans-serif }
-a { text-decoration: none }
-a:hover { text-decoration: underline }
 .content { padding-right: 1em }
 .content h1, .content h2, .content h3 { margin: 0 0 0.3em; font-weight: normal }
 table { border-collapse: collapse; border-spacing: 0; width: 100% }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java Wed Nov  2 05:34:31 2011
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.util.Records;
 
 import com.google.common.collect.Iterators;
@@ -81,13 +82,19 @@ public class MockApps {
     final String queue = newQueue();
     final FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
     return new ApplicationReport() {
+      private ApplicationResourceUsageReport appUsageReport;
       @Override public ApplicationId getApplicationId() { return id; }
       @Override public String getUser() { return user; }
       @Override public String getName() { return name; }
       @Override public YarnApplicationState getYarnApplicationState() { return state; }
       @Override public String getQueue() { return queue; }
       @Override public String getTrackingUrl() { return ""; }
+      @Override public String getOriginalTrackingUrl() { return ""; }
       @Override public FinalApplicationStatus getFinalApplicationStatus() { return finishState; }
+      @Override
+      public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
+        return this.appUsageReport;
+      }
       public void setApplicationId(ApplicationId applicationId) {
         // TODO Auto-generated method stub
 
@@ -97,6 +104,11 @@ public class MockApps {
         // TODO Auto-generated method stub
 
       }
+      @Override public void setOriginalTrackingUrl(String url) { }
+      @Override
+      public void setApplicationResourceUsageReport(ApplicationResourceUsageReport appResources) {
+        this.appUsageReport = appResources;
+      }
       @Override
       public void setName(String name) {
         // TODO Auto-generated method stub

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java Wed Nov  2 05:34:31 2011
@@ -22,7 +22,7 @@ import java.net.InetSocketAddress;
 
 import junit.framework.Assert;
 
-import org.apache.avro.ipc.Server;
+import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.yarn.api.ClientRMProtocol;
@@ -162,7 +162,7 @@ public class TestRPC {
     }
     Assert.assertTrue(exception);
     
-    server.close();
+    server.stop();
     Assert.assertNotNull(status);
     Assert.assertEquals(ContainerState.RUNNING, status.getState());
   }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java Wed Nov  2 05:34:31 2011
@@ -18,12 +18,9 @@
 
 package org.apache.hadoop.yarn.conf;
 
-import java.net.InetSocketAddress;
 
 import junit.framework.Assert;
 
-import org.apache.avro.ipc.Server;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.junit.Test;
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/WebAppTests.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/WebAppTests.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/WebAppTests.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/WebAppTests.java Wed Nov  2 05:34:31 2011
@@ -34,7 +34,9 @@ import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Provides;
 
+import java.io.IOException;
 import java.io.PrintWriter;
+
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpServletRequest;
 
@@ -169,4 +171,16 @@ public class WebAppTests {
   public static <T> Injector testBlock(Class<? extends SubView> block) {
     return testBlock(block, null, null);
   }
+  
+  /**
+   * Convenience method to get the spy writer. 
+   * @param injector the injector used for the test.
+   * @return The Spy writer.
+   * @throws IOException
+   */
+  public static PrintWriter getPrintWriter(Injector injector)
+      throws IOException {
+    HttpServletResponse res = injector.getInstance(HttpServletResponse.class);
+    return res.getWriter();
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml Wed Nov  2 05:34:31 2011
@@ -16,15 +16,16 @@
   <parent>
     <artifactId>hadoop-yarn-server</artifactId>
     <groupId>org.apache.hadoop</groupId>
-    <version>${yarn.version}</version>
+    <version>0.24.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-yarn-server-common</artifactId>
+  <version>0.24.0-SNAPSHOT</version>
   <name>hadoop-yarn-server-common</name>
 
   <properties>
-    <install.file>${project.artifact.file}</install.file>
+    <!-- Needed for generating FindBugs warnings using parent pom -->
     <yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
   </properties>
 
@@ -56,6 +57,18 @@
               <goal>run</goal>
             </goals>
           </execution>
+          <execution>
+            <phase>pre-site</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <tasks>
+                <copy file="src/main/resources/yarn-default.xml" todir="src/site/resources"/>
+                <copy file="src/main/xsl/configuration.xsl" todir="src/site/resources"/>
+              </tasks>
+            </configuration>
+          </execution>
         </executions>
       </plugin>
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java Wed Nov  2 05:34:31 2011
@@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.api.record
 
 public interface HeartbeatResponse {
   int getResponseId();
-  boolean getReboot();
+  NodeAction getNodeAction();
   
   List<ContainerId> getContainersToCleanupList();
   ContainerId getContainerToCleanup(int index);
@@ -35,7 +35,7 @@ public interface HeartbeatResponse {
   int getApplicationsToCleanupCount();
   
   void setResponseId(int responseId);
-  void setReboot(boolean reboot);
+  void setNodeAction(NodeAction action);
   
   void addAllContainersToCleanup(List<ContainerId> containers);
   void addContainerToCleanup(ContainerId container);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java Wed Nov  2 05:34:31 2011
@@ -23,4 +23,8 @@ public interface RegistrationResponse {
   public abstract ByteBuffer getSecretKey();
   
   public abstract void setSecretKey(ByteBuffer secretKey);
+  
+  public abstract NodeAction getNodeAction();
+  
+  public abstract void setNodeAction(NodeAction nodeAction);
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java Wed Nov  2 05:34:31 2011
@@ -32,11 +32,12 @@ import org.apache.hadoop.yarn.proto.Yarn
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.HeartbeatResponseProto;
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.HeartbeatResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto;
 import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
+import org.apache.hadoop.yarn.server.api.records.NodeAction;
 
-
-    
-public class HeartbeatResponsePBImpl extends ProtoBase<HeartbeatResponseProto> implements HeartbeatResponse {
+public class HeartbeatResponsePBImpl extends
+    ProtoBase<HeartbeatResponseProto> implements HeartbeatResponse {
   HeartbeatResponseProto proto = HeartbeatResponseProto.getDefaultInstance();
   HeartbeatResponseProto.Builder builder = null;
   boolean viaProto = false;
@@ -100,16 +101,24 @@ public class HeartbeatResponsePBImpl ext
     builder.setResponseId((responseId));
   }
   @Override
-  public boolean getReboot() {
+  public NodeAction getNodeAction() {
     HeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
-    return (p.getReboot());
+    if(!p.hasNodeAction()) {
+      return null;
+    }
+    return (convertFromProtoFormat(p.getNodeAction()));
   }
 
   @Override
-  public void setReboot(boolean reboot) {
+  public void setNodeAction(NodeAction nodeAction) {
     maybeInitBuilder();
-    builder.setReboot((reboot));
+    if (nodeAction == null) {
+      builder.clearNodeAction();
+      return;
+    }
+    builder.setNodeAction(convertToProtoFormat(nodeAction));
   }
+
   @Override
   public List<ContainerId> getContainersToCleanupList() {
     initContainersToCleanup();
@@ -296,7 +305,12 @@ public class HeartbeatResponsePBImpl ext
   private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
     return ((ApplicationIdPBImpl)t).getProto();
   }
-
-
-
+  
+  private NodeAction convertFromProtoFormat(NodeActionProto p) {
+    return  NodeAction.valueOf(p.name());
+  }
+  
+  private NodeActionProto convertToProtoFormat(NodeAction t) {
+    return NodeActionProto.valueOf(t.name());
+  }
 }  

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java Wed Nov  2 05:34:31 2011
@@ -21,17 +21,15 @@ package org.apache.hadoop.yarn.server.ap
 
 import java.nio.ByteBuffer;
 
-import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.ProtoBase;
-import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto;
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.RegistrationResponseProto;
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.RegistrationResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.records.NodeAction;
 import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
 
-
-    
-public class RegistrationResponsePBImpl extends ProtoBase<RegistrationResponseProto> implements RegistrationResponse {
+public class RegistrationResponsePBImpl extends
+    ProtoBase<RegistrationResponseProto> implements RegistrationResponse {
   RegistrationResponseProto proto = RegistrationResponseProto.getDefaultInstance();
   RegistrationResponseProto.Builder builder = null;
   boolean viaProto = false;
@@ -98,4 +96,31 @@ public class RegistrationResponsePBImpl 
     this.secretKey = secretKey;
   }
 
+  @Override
+  public NodeAction getNodeAction() {
+    RegistrationResponseProtoOrBuilder p = viaProto ? proto : builder;
+    if(!p.hasNodeAction()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getNodeAction());
+  }
+
+  @Override
+  public void setNodeAction(NodeAction nodeAction) {
+    maybeInitBuilder();
+    if (nodeAction == null) {
+      builder.clearNodeAction();
+      return;
+    }
+    builder.setNodeAction(convertToProtoFormat(nodeAction));
+  }
+  
+  private NodeAction convertFromProtoFormat(NodeActionProto p) {
+    return  NodeAction.valueOf(p.name());
+  }
+  
+  private NodeActionProto convertToProtoFormat(NodeAction t) {
+    return NodeActionProto.valueOf(t.name());
+  }
+
 }  

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenSecretManager.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenSecretManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenSecretManager.java Wed Nov  2 05:34:31 2011
@@ -56,19 +56,19 @@ public class ContainerTokenSecretManager
   @Override
   public byte[] createPassword(ContainerTokenIdentifier identifier) {
     LOG.debug("Creating password for " + identifier.getContainerID()
-        + " to be run on NM " + identifier.getNmHostName() + " "
-        + this.secretkeys.get(identifier.getNmHostName()));
+        + " to be run on NM " + identifier.getNmHostAddress() + " "
+        + this.secretkeys.get(identifier.getNmHostAddress()));
     return createPassword(identifier.getBytes(),
-        this.secretkeys.get(identifier.getNmHostName()));
+        this.secretkeys.get(identifier.getNmHostAddress()));
   }
 
   @Override
   public byte[] retrievePassword(ContainerTokenIdentifier identifier)
       throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
     LOG.debug("Retrieving password for " + identifier.getContainerID()
-        + " to be run on NM " + identifier.getNmHostName());
+        + " to be run on NM " + identifier.getNmHostAddress());
     return createPassword(identifier.getBytes(),
-        this.secretkeys.get(identifier.getNmHostName()));
+        this.secretkeys.get(identifier.getNmHostAddress()));
   }
 
   @Override

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto Wed Nov  2 05:34:31 2011
@@ -23,6 +23,12 @@ option java_generate_equals_and_hash = t
 
 import "yarn_protos.proto";
 
+enum NodeActionProto {
+  NORMAL = 0;
+  REBOOT = 1;
+  SHUTDOWN = 2;
+}
+
 message NodeStatusProto {
   optional NodeIdProto node_id = 1;
   optional int32 response_id = 2;
@@ -32,11 +38,12 @@ message NodeStatusProto {
 
 message RegistrationResponseProto {
   optional bytes secret_key = 1;
+  optional NodeActionProto nodeAction = 2;
 }
 
 message HeartbeatResponseProto {
   optional int32 response_id = 1;
-  optional bool reboot = 2;
+  optional NodeActionProto nodeAction = 2;
   repeated ContainerIdProto containers_to_cleanup = 3;
   repeated ApplicationIdProto applications_to_cleanup = 4;
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml Wed Nov  2 05:34:31 2011
@@ -1,4 +1,27 @@
 <?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!-- Do not modify this file directly.  Instead, copy entries that you -->
+<!-- wish to modify from this file into yarn-site.xml and change them -->
+<!-- there.  If yarn-site.xml does not already exist, create it.      -->
+
 <configuration>
   
   <!-- IPC Configs -->
@@ -83,14 +106,14 @@
   </property>
 
   <property>
-    <description>Are RM acls enabled.</description>
-    <name>yarn.resourcemanager.acl.enable</name>
-    <value>false</value>
+    <description>Are acls enabled.</description>
+    <name>yarn.acl.enable</name>
+    <value>true</value>
   </property>
 
   <property>
-    <description>ACL of who can be admin of RM.</description>
-    <name>yarn.resourcemanager.admin.acl</name>
+    <description>ACL of who can be admin of the YARN cluster.</description>
+    <name>yarn.admin.acl</name>
     <value>*</value>
   </property>
 
@@ -191,7 +214,19 @@
   <property>
     <description>address of node manager IPC.</description>
     <name>yarn.nodemanager.address</name>
-    <value>0.0.0.0:45454</value>
+    <value>0.0.0.0:0</value>
+  </property>
+
+  <property>
+    <description>Environment variables that should be forwarded from the NodeManager's environment to the container's.</description>
+    <name>yarn.nodemanager.admin-env</name>
+    <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
+  </property>
+
+  <property>
+    <description>Environment variables that containers may override rather than use NodeManager's default.</description>
+    <name>yarn.nodemanager.env-whitelist</name>
+    <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME</value>
   </property>
 
   <property>
@@ -268,15 +303,48 @@
   </property>
 
   <property>
+    <description>Whether to enable log aggregation</description>
+    <name>yarn.nodemanager.log-aggregation-enable</name>
+    <value>false</value>
+  </property>
+  
+  <property>
+    <description>Time in seconds to retain user logs. Only applicable if
+    log aggregation is disabled
+    </description>
+    <name>yarn.nodemanager.log.retain-seconds</name>
+    <value>10800</value>
+  </property>
+
+  <property>
     <description>Where to aggregate logs to.</description>
     <name>yarn.nodemanager.remote-app-log-dir</name>
     <value>/tmp/logs</value>
   </property>
+  <property>
+    <description>The remote log dir will be created at 
+      {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}
+    </description>
+    <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
+    <value>logs</value>
+  </property>
+
+
+  <property>
+    <description>Amount of physical memory, in MB, that can be allocated 
+    for containers.</description>
+    <name>yarn.nodemanager.resource.memory-mb</name>
+    <value>8192</value>
+  </property>
 
   <property>
-    <description>Amount of memory in GB that can be allocated for containers.</description>
-    <name>yarn.nodemanager.resource.memory-gb</name>
-    <value>8</value>
+    <description>Ratio between virtual memory to physical memory when
+    setting memory limits for containers. Container allocations are
+    expressed in terms of physical memory, and virtual memory usage
+    is allowed to exceed this allocation by this ratio.
+    </description>
+    <name>yarn.nodemanager.vmem-pmem-ratio</name>
+    <value>2.1</value>
   </property>
 
   <property>
@@ -297,12 +365,6 @@
   </property>
 
   <property>
-    <description>Amount of physical ram to reserve for other applications, -1 disables.</description>
-    <name>yarn.nodemanager.reserved.memory-mb</name>
-    <value>-1</value>
-  </property>
-
-  <property>
     <description>Frequency of running node health script.</description>
     <name>yarn.nodemanager.health-checker.interval-ms</name>
     <value>600000</value>
@@ -349,6 +411,18 @@
     <!-- <value>mapreduce.shuffle</value> -->
   </property>
 
+  <property>
+    <description>No. of ms to wait between sending a SIGTERM and SIGKILL to a container</description>
+    <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
+    <value>250</value>
+  </property>
+
+  <property>
+    <description>Max time to wait for a process to come up when trying to cleanup a container</description>
+    <name>yarn.nodemanager.process-kill-wait.ms</name>
+    <value>2000</value>
+  </property>
+
   <!--Map Reduce configuration-->
   <property>
     <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
@@ -364,4 +438,27 @@
     <name>mapreduce.job.hdfs-servers</name>
     <value>${fs.default.name}</value>
   </property>
+
+  <!-- WebAppProxy Configuration-->
+  
+  <property>
+    <description>The kerberos principal for the proxy, if the proxy is not
+    running as part of the RM.</description>
+    <name>yarn.web-proxy.principal</name>
+    <value/>
+  </property>
+  
+  <property>
+    <description>Keytab for WebAppProxy, if the proxy is not running as part of 
+    the RM.</description>
+    <name>yarn.web-proxy.keytab</name>
+  </property>
+  
+  <property>
+    <description>The address for the web proxy as HOST:PORT, if this is not
+     given or if it matches yarn.resourcemanager.address then the proxy will
+     run as part of the RM</description>
+     <name>yarn.web-proxy.address</name>
+     <value/>
+  </property>
 </configuration>



Mime
View raw message