hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject svn commit: r1524865 [4/8] - in /hadoop/common/branches/HDFS-4949/hadoop-yarn-project: ./ hadoop-yarn/bin/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protoc...
Date Thu, 19 Sep 2013 23:42:29 GMT
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java Thu Sep 19 23:42:10 2013
@@ -23,6 +23,8 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshSuperUserGroupsConfigurationRequestProto;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
 
+import com.google.protobuf.TextFormat;
+
 @Private
 @Unstable
 public class RefreshSuperUserGroupsConfigurationRequestPBImpl 
@@ -64,6 +66,6 @@ extends RefreshSuperUserGroupsConfigurat
 
   @Override
   public String toString() {
-    return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
+    return TextFormat.shortDebugString(getProto());
   }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java Thu Sep 19 23:42:10 2013
@@ -23,6 +23,8 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshSuperUserGroupsConfigurationResponseProto;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse;
 
+import com.google.protobuf.TextFormat;
+
 @Private
 @Unstable
 public class RefreshSuperUserGroupsConfigurationResponsePBImpl extends RefreshSuperUserGroupsConfigurationResponse {
@@ -63,6 +65,6 @@ public class RefreshSuperUserGroupsConfi
 
   @Override
   public String toString() {
-    return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
+    return TextFormat.shortDebugString(getProto());
   }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshUserToGroupsMappingsRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshUserToGroupsMappingsRequestPBImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshUserToGroupsMappingsRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshUserToGroupsMappingsRequestPBImpl.java Thu Sep 19 23:42:10 2013
@@ -23,6 +23,8 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsRequestProto;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
 
+import com.google.protobuf.TextFormat;
+
 @Private
 @Unstable
 public class RefreshUserToGroupsMappingsRequestPBImpl 
@@ -64,6 +66,6 @@ extends RefreshUserToGroupsMappingsReque
 
   @Override
   public String toString() {
-    return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
+    return TextFormat.shortDebugString(getProto());
   }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshUserToGroupsMappingsResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshUserToGroupsMappingsResponsePBImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshUserToGroupsMappingsResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshUserToGroupsMappingsResponsePBImpl.java Thu Sep 19 23:42:10 2013
@@ -23,6 +23,8 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsResponseProto;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
 
+import com.google.protobuf.TextFormat;
+
 @Private
 @Unstable
 public class RefreshUserToGroupsMappingsResponsePBImpl extends RefreshUserToGroupsMappingsResponse {
@@ -63,6 +65,6 @@ public class RefreshUserToGroupsMappings
 
   @Override
   public String toString() {
-    return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
+    return TextFormat.shortDebugString(getProto());
   }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Thu Sep 19 23:42:10 2013
@@ -198,7 +198,8 @@ public class ConverterUtils {
     Iterator<String> it = _split(appIdStr).iterator();
     if (!it.next().equals(APPLICATION_PREFIX)) {
       throw new IllegalArgumentException("Invalid ApplicationId prefix: "
-          + appIdStr);
+          + appIdStr + ". The valid ApplicationId should start with prefix "
+          + APPLICATION_PREFIX);
     }
     try {
       return toApplicationId(it);

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Thu Sep 19 23:42:10 2013
@@ -33,6 +33,8 @@ import org.apache.commons.lang.StringUti
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.AdminACLsManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -65,7 +67,6 @@ import com.google.inject.servlet.GuiceFi
 @InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
 public class WebApps {
   static final Logger LOG = LoggerFactory.getLogger(WebApps.class);
-
   public static class Builder<T> {
     static class ServletStruct {
       public Class<? extends HttpServlet> clazz;
@@ -82,6 +83,8 @@ public class WebApps {
     boolean findPort = false;
     Configuration conf;
     boolean devMode = false;
+    private String spnegoPrincipalKey;
+    private String spnegoKeytabKey;
     private final HashSet<ServletStruct> servlets = new HashSet<ServletStruct>();
     private final HashMap<String, Object> attributes = new HashMap<String, Object>();
 
@@ -135,6 +138,16 @@ public class WebApps {
       this.conf = conf;
       return this;
     }
+    
+    public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
+      this.spnegoPrincipalKey = spnegoPrincipalKey;
+      return this;
+    }
+    
+    public Builder<T> withHttpSpnegoKeytabKey(String spnegoKeytabKey) {
+      this.spnegoKeytabKey = spnegoKeytabKey;
+      return this;
+    }
 
     public Builder<T> inDevMode() {
       devMode = true;
@@ -197,15 +210,41 @@ public class WebApps {
           }
         }
         HttpServer server =
-            new HttpServer(name, bindAddress, port, findPort, conf, 
-            new AdminACLsManager(conf).getAdminAcl(), null, webapp.getServePathSpecs());
+            new HttpServer(name, bindAddress, port, findPort, conf,
+                new AdminACLsManager(conf).getAdminAcl(), null,
+                webapp.getServePathSpecs()) {
+
+              {
+                if (UserGroupInformation.isSecurityEnabled()) {
+                  boolean initSpnego = true;
+                  if (spnegoPrincipalKey == null
+                      || conf.get(spnegoPrincipalKey, "").isEmpty()) {
+                    LOG.warn("Principal for spnego filter is not set");
+                    initSpnego = false;
+                  }
+                  if (spnegoKeytabKey == null
+                      || conf.get(spnegoKeytabKey, "").isEmpty()) {
+                    LOG.warn("Keytab for spnego filter is not set");
+                    initSpnego = false;
+                  }
+                  if (initSpnego) {
+                    LOG.info("Initializing spnego filter with principal key : "
+                        + spnegoPrincipalKey + " keytab key : "
+                        + spnegoKeytabKey);
+                    initSpnego(conf, spnegoPrincipalKey, spnegoKeytabKey);
+                  }
+                }
+              }
+            };
         for(ServletStruct struct: servlets) {
           server.addServlet(struct.name, struct.spec, struct.clazz);
         }
         for(Map.Entry<String, Object> entry : attributes.entrySet()) {
           server.setAttribute(entry.getKey(), entry.getValue());
         }
-        server.addGlobalFilter("guice", GuiceFilter.class.getName(), null);
+        server.defineFilter(server.getWebAppContext(), "guice",
+          GuiceFilter.class.getName(), null, new String[] { "/*" });
+
         webapp.setConf(conf);
         webapp.setHttpServer(server);
         server.start();

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Thu Sep 19 23:42:10 2013
@@ -141,6 +141,20 @@
   </property>
 
   <property>
+    <description>Maximum time to wait to establish connection to
+    ResourceManager.</description>
+    <name>yarn.resourcemanager.connect.max-wait.ms</name>
+    <value>900000</value>
+  </property>
+
+  <property>
+    <description>How often to try connecting to the
+    ResourceManager.</description>
+    <name>yarn.resourcemanager.connect.retry-interval.ms</name>
+    <value>30000</value>
+  </property>
+
+  <property>
     <description>The maximum number of application attempts. It's a global
     setting for all application masters. Each application master can specify
     its individual maximum number of application attempts via the API, but the
@@ -249,7 +263,7 @@
     RM state will be stored. This must be supplied when using
     org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
     as the value for yarn.resourcemanager.store.class</description>
-    <name>yarn.resourcemanager.fs.rm-state-store.uri</name>
+    <name>yarn.resourcemanager.fs.state-store.uri</name>
     <value>${hadoop.tmp.dir}/yarn/system/rmstore</value>
     <!--value>hdfs://localhost:9000/rmstore</value-->
   </property>

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java Thu Sep 19 23:42:10 2013
@@ -19,14 +19,30 @@
 package org.apache.hadoop.yarn.logaggregation;
 
 import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+
+import junit.framework.Assert;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Before;
 import org.junit.Test;
 
 public class TestLogDumper {
+  ByteArrayOutputStream sysOutStream;
+  private PrintStream sysOut;
+
+  @Before
+  public void setUp() {
+    sysOutStream = new ByteArrayOutputStream();
+    sysOut =  new PrintStream(sysOutStream);
+    System.setOut(sysOut);
+  }
+
   @Test
   public void testFailResultCodes() throws Exception {
     Configuration conf = new YarnConfiguration();
@@ -44,4 +60,30 @@ public class TestLogDumper {
         "nonexistentnode:1234", "nobody");
     assertTrue("Should return an error code", exitCode != 0);
   }
+
+  @Test
+  public void testHelpMessage() throws Exception {
+    Configuration conf = new YarnConfiguration();
+    LogDumper dumper = new LogDumper();
+    dumper.setConf(conf);
+
+    int exitCode = dumper.run(new String[]{});
+    assertTrue(exitCode == -1);
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintWriter pw = new PrintWriter(baos);
+    pw.println("Retrieve logs for completed YARN applications.");
+    pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
+    pw.println();
+    pw.println("general options are:");
+    pw.println(" -appOwner <Application Owner>   AppOwner (assumed to be current user if");
+    pw.println("                                 not specified)");
+    pw.println(" -containerId <Container ID>     ContainerId (must be specified if node");
+    pw.println("                                 address is specified)");
+    pw.println(" -nodeAddress <Node Address>     NodeAddress in the format nodename:port");
+    pw.println("                                 (must be specified if container id is");
+    pw.println("                                 specified)");
+    pw.close();
+    String appReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(appReportStr, sysOutStream.toString());
+  }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java Thu Sep 19 23:42:10 2013
@@ -19,11 +19,14 @@
 package org.apache.hadoop.yarn.util;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.BreakableService;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.service.Service;
@@ -314,6 +317,26 @@ public class TestCompositeService {
     composite.init(new Configuration());
     assertInState(STATE.INITED, child);
   }
+
+  @Test (timeout = 1000)
+  public void testAddIfService() {
+    CompositeService testService = new CompositeService("TestService") {
+      Service service;
+      @Override
+      public void serviceInit(Configuration conf) {
+        Integer notAService = new Integer(0);
+        assertFalse("Added an integer as a service",
+            addIfService(notAService));
+
+        service = new AbstractService("Service") {};
+        assertTrue("Unable to add a service", addIfService(service));
+      }
+    };
+
+    testService.init(new Configuration());
+    assertEquals("Incorrect number of services",
+        1, testService.getServices().size());
+  }
   
   public static class CompositeServiceAddingAChild extends CompositeService{
     Service child;

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java Thu Sep 19 23:42:10 2013
@@ -18,30 +18,47 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.yarn.MockApps;
-import org.apache.hadoop.yarn.webapp.Controller;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.hadoop.yarn.webapp.WebApps;
-import org.apache.hadoop.yarn.webapp.view.HtmlPage;
-import org.apache.hadoop.yarn.webapp.view.JQueryUI;
-import org.apache.hadoop.yarn.webapp.view.TextPage;
-
-import com.google.inject.Inject;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_TABLE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.ext.ContextResolver;
+import javax.ws.rs.ext.Provider;
+import javax.xml.bind.JAXBContext;
 
-import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
-
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.yarn.MockApps;
+import org.apache.hadoop.yarn.webapp.view.HtmlPage;
+import org.apache.hadoop.yarn.webapp.view.JQueryUI;
+import org.apache.hadoop.yarn.webapp.view.TextPage;
 import org.junit.Test;
-import static org.junit.Assert.*;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+
 public class TestWebApp {
   static final Logger LOG = LoggerFactory.getLogger(TestWebApp.class);
 
@@ -227,14 +244,19 @@ public class TestWebApp {
   }
 
   @Test public void testCustomRoutes() throws Exception {
-    WebApp app = WebApps.$for("test", this).start(new WebApp() {
-      @Override public void setup() {
-        route("/:foo", FooController.class);
-        route("/bar/foo", FooController.class, "bar");
-        route("/foo/:foo", DefaultController.class);
-        route("/foo/bar/:foo", DefaultController.class, "index");
-      }
-    });
+    WebApp app =
+        WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
+          @Override
+          public void setup() {
+            bind(MyTestJAXBContextResolver.class);
+            bind(MyTestWebService.class);
+
+            route("/:foo", FooController.class);
+            route("/bar/foo", FooController.class, "bar");
+            route("/foo/:foo", DefaultController.class);
+            route("/foo/bar/:foo", DefaultController.class, "index");
+          }
+        });
     String baseUrl = baseUrl(app);
     try {
       assertEquals("foo", getContent(baseUrl).trim());
@@ -245,6 +267,31 @@ public class TestWebApp {
       assertEquals("default1", getContent(baseUrl +"test/foo/1").trim());
       assertEquals("default2", getContent(baseUrl +"test/foo/bar/2").trim());
       assertEquals(404, getResponseCode(baseUrl +"test/goo"));
+      assertEquals(200, getResponseCode(baseUrl +"ws/v1/test"));
+      assertTrue(getContent(baseUrl +"ws/v1/test").contains("myInfo"));
+    } finally {
+      app.stop();
+    }
+  }
+
+  // This is to test the GuiceFilter should only be applied to webAppContext,
+  // not to staticContext  and logContext;
+  @Test public void testYARNWebAppContext() throws Exception {
+    // setting up the log context
+    System.setProperty("hadoop.log.dir", "/Not/Existing/dir");
+    WebApp app = WebApps.$for("test", this).start(new WebApp() {
+      @Override public void setup() {
+        route("/", FooController.class);
+      }
+    });
+    String baseUrl = baseUrl(app);
+    try {
+      // should not redirect to foo
+      assertFalse("foo".equals(getContent(baseUrl +"static").trim()));
+      // Not able to access a non-existing dir, should not redirect to foo.
+      assertEquals(404, getResponseCode(baseUrl +"logs"));
+      // should be able to redirect to foo.
+      assertEquals("foo", getContent(baseUrl).trim());
     } finally {
       app.stop();
     }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java Thu Sep 19 23:42:10 2013
@@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.proto.Yarn
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProtoOrBuilder;
 import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
 
+import com.google.protobuf.TextFormat;
+
 public class NodeHealthStatusPBImpl extends NodeHealthStatus {
 
   private NodeHealthStatusProto.Builder builder;
@@ -62,7 +64,7 @@ public class NodeHealthStatusPBImpl exte
 
   @Override
   public String toString() {
-    return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
+    return TextFormat.shortDebugString(getProto());
   }
 
   private void mergeLocalToProto() {

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java Thu Sep 19 23:42:10 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 
 /**
  * Context interface for sharing information across components in the
@@ -61,4 +62,8 @@ public interface Context {
   NodeHealthStatus getNodeHealthStatus();
 
   ContainerManagementProtocol getContainerManager();
+
+  LocalDirsHandlerService getLocalDirsHandler();
+
+  ApplicationACLsManager getApplicationACLsManager();
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Thu Sep 19 23:42:10 2013
@@ -123,7 +123,8 @@ public class NodeManager extends Composi
   protected NMContext createNMContext(
       NMContainerTokenSecretManager containerTokenSecretManager,
       NMTokenSecretManagerInNM nmTokenSecretManager) {
-    return new NMContext(containerTokenSecretManager, nmTokenSecretManager);
+    return new NMContext(containerTokenSecretManager, nmTokenSecretManager,
+        dirsHandler, aclsManager);
   }
 
   protected void doSecureLogin() throws IOException {
@@ -142,9 +143,6 @@ public class NodeManager extends Composi
     NMTokenSecretManagerInNM nmTokenSecretManager =
         new NMTokenSecretManagerInNM();
     
-    this.context =
-        createNMContext(containerTokenSecretManager, nmTokenSecretManager);
-
     this.aclsManager = new ApplicationACLsManager(conf);
 
     ContainerExecutor exec = ReflectionUtils.newInstance(
@@ -165,7 +163,9 @@ public class NodeManager extends Composi
     addService(nodeHealthChecker);
     dirsHandler = nodeHealthChecker.getDiskHandler();
 
-
+    this.context = createNMContext(containerTokenSecretManager,
+        nmTokenSecretManager);
+    
     nodeStatusUpdater =
         createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
 
@@ -319,14 +319,19 @@ public class NodeManager extends Composi
     private final NMContainerTokenSecretManager containerTokenSecretManager;
     private final NMTokenSecretManagerInNM nmTokenSecretManager;
     private ContainerManagementProtocol containerManager;
+    private final LocalDirsHandlerService dirsHandler;
+    private final ApplicationACLsManager aclsManager;
     private WebServer webServer;
     private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider
         .getRecordFactory(null).newRecordInstance(NodeHealthStatus.class);
-
+        
     public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
-        NMTokenSecretManagerInNM nmTokenSecretManager) {
+        NMTokenSecretManagerInNM nmTokenSecretManager,
+        LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager) {
       this.containerTokenSecretManager = containerTokenSecretManager;
       this.nmTokenSecretManager = nmTokenSecretManager;
+      this.dirsHandler = dirsHandler;
+      this.aclsManager = aclsManager;
       this.nodeHealthStatus.setIsNodeHealthy(true);
       this.nodeHealthStatus.setHealthReport("Healthy");
       this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
@@ -386,6 +391,16 @@ public class NodeManager extends Composi
     public void setNodeId(NodeId nodeId) {
       this.nodeId = nodeId;
     }
+
+    @Override
+    public LocalDirsHandlerService getLocalDirsHandler() {
+      return dirsHandler;
+    }
+    
+    @Override
+    public ApplicationACLsManager getApplicationACLsManager() {
+      return aclsManager;
+    }
   }
 
 

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java Thu Sep 19 23:42:10 2013
@@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.event.Even
 import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
 import org.apache.hadoop.yarn.server.api.AuxiliaryService;
 import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
 
 public class AuxServices extends AbstractService
     implements ServiceStateChangeListener, EventHandler<AuxServicesEvent> {
@@ -178,7 +180,21 @@ public class AuxServices extends Abstrac
           .getApplicationID()));
       }
       break;
-    default:
+    case CONTAINER_INIT:
+      for (AuxiliaryService serv : serviceMap.values()) {
+        serv.initializeContainer(new ContainerInitializationContext(
+            event.getUser(), event.getContainer().getContainerId(),
+            event.getContainer().getResource()));
+      }
+      break;
+    case CONTAINER_STOP:
+      for (AuxiliaryService serv : serviceMap.values()) {
+        serv.stopContainer(new ContainerTerminationContext(
+            event.getUser(), event.getContainer().getContainerId(),
+            event.getContainer().getResource()));
+      }
+      break;
+      default:
       throw new RuntimeException("Unknown type: " + event.getType());
     }
   }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java Thu Sep 19 23:42:10 2013
@@ -21,7 +21,10 @@ package org.apache.hadoop.yarn.server.no
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+    .Container;
 
 public class AuxServicesEvent extends AbstractEvent<AuxServicesEventType> {
 
@@ -29,18 +32,30 @@ public class AuxServicesEvent extends Ab
   private final String serviceId;
   private final ByteBuffer serviceData;
   private final ApplicationId appId;
+  private final Container container;
 
   public AuxServicesEvent(AuxServicesEventType eventType, ApplicationId appId) {
     this(eventType, null, appId, null, null);
   }
 
+  public AuxServicesEvent(AuxServicesEventType eventType, Container container) {
+    this(eventType, null, container.getContainerId().getApplicationAttemptId()
+        .getApplicationId(), null, null, container);
+  }
+
   public AuxServicesEvent(AuxServicesEventType eventType, String user,
       ApplicationId appId, String serviceId, ByteBuffer serviceData) {
+    this(eventType, user, appId, serviceId, serviceData, null);
+  }
+    public AuxServicesEvent(AuxServicesEventType eventType, String user,
+      ApplicationId appId, String serviceId, ByteBuffer serviceData,
+        Container container) {
     super(eventType);
     this.user = user;
     this.appId = appId;
     this.serviceId = serviceId;
     this.serviceData = serviceData;
+    this.container = container;
   }
 
   public String getServiceID() {
@@ -59,4 +74,8 @@ public class AuxServicesEvent extends Ab
     return appId;
   }
 
+  public Container getContainer() {
+    return container;
+  }
+
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java Thu Sep 19 23:42:10 2013
@@ -20,5 +20,7 @@ package org.apache.hadoop.yarn.server.no
 
 public enum AuxServicesEventType {
   APPLICATION_INIT,
-  APPLICATION_STOP
+  APPLICATION_STOP,
+  CONTAINER_INIT,
+  CONTAINER_STOP
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Thu Sep 19 23:42:10 2013
@@ -85,7 +85,6 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
 import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
-import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerInitEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -193,12 +192,6 @@ public class ContainerManagerImpl extend
     super.serviceInit(conf);
   }
 
-  private void addIfService(Object object) {
-    if (object instanceof Service) {
-      addService((Service) object);
-    }
-  }
-
   protected LogHandler createLogHandler(Configuration conf, Context context,
       DeletionService deletionService) {
     if (conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
@@ -223,7 +216,7 @@ public class ContainerManagerImpl extend
 
   protected ContainersLauncher createContainersLauncher(Context context,
       ContainerExecutor exec) {
-    return new ContainersLauncher(context, this.dispatcher, exec, dirsHandler);
+    return new ContainersLauncher(context, this.dispatcher, exec, dirsHandler, this);
   }
 
   @Override
@@ -417,7 +410,7 @@ public class ContainerManagerImpl extend
       }
     }
 
-    return StartContainersResponse.newInstance(auxiliaryServices.getMetaData(),
+    return StartContainersResponse.newInstance(getAuxServiceMetaData(),
       succeededContainers, failedContainers);
   }
 
@@ -475,8 +468,7 @@ public class ContainerManagerImpl extend
 
     // Create the application
     Application application =
-        new ApplicationImpl(dispatcher, this.aclsManager, user, applicationID,
-          credentials, context);
+        new ApplicationImpl(dispatcher, user, applicationID, credentials, context);
     if (null == context.getApplications().putIfAbsent(applicationID,
       application)) {
       LOG.info("Creating a new application reference for app " + applicationID);
@@ -767,4 +759,7 @@ public class ContainerManagerImpl extend
     return this.context;
   }
 
+  public Map<String, ByteBuffer> getAuxServiceMetaData() {
+    return this.auxiliaryServices.getMetaData();
+  }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Thu Sep 19 23:42:10 2013
@@ -73,14 +73,13 @@ public class ApplicationImpl implements 
   Map<ContainerId, Container> containers =
       new HashMap<ContainerId, Container>();
 
-  public ApplicationImpl(Dispatcher dispatcher,
-      ApplicationACLsManager aclsManager, String user, ApplicationId appId,
+  public ApplicationImpl(Dispatcher dispatcher, String user, ApplicationId appId,
       Credentials credentials, Context context) {
     this.dispatcher = dispatcher;
     this.user = user;
     this.appId = appId;
     this.credentials = credentials;
-    this.aclsManager = aclsManager;
+    this.aclsManager = context.getApplicationACLsManager();
     this.context = context;
     ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     readLock = lock.readLock();

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Thu Sep 19 23:42:10 2013
@@ -503,6 +503,9 @@ public class ContainerImpl implements Co
       final ContainerLaunchContext ctxt = container.launchContext;
       container.metrics.initingContainer();
 
+      container.dispatcher.getEventHandler().handle(new AuxServicesEvent
+          (AuxServicesEventType.CONTAINER_INIT, container));
+
       // Inform the AuxServices about the opaque serviceData
       Map<String,ByteBuffer> csd = ctxt.getServiceData();
       if (csd != null) {
@@ -820,8 +823,16 @@ public class ContainerImpl implements Co
   static class ContainerDoneTransition implements
       SingleArcTransition<ContainerImpl, ContainerEvent> {
     @Override
+    @SuppressWarnings("unchecked")
     public void transition(ContainerImpl container, ContainerEvent event) {
       container.finished();
+      //if the current state is NEW it means the CONTAINER_INIT was never 
+      // sent for the event, thus no need to send the CONTAINER_STOP
+      if (container.getCurrentState() 
+          != org.apache.hadoop.yarn.api.records.ContainerState.NEW) {
+        container.dispatcher.getEventHandler().handle(new AuxServicesEvent
+            (AuxServicesEventType.CONTAINER_STOP, container));
+      }
     }
   }
 

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Thu Sep 19 23:42:10 2013
@@ -26,6 +26,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintStream;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -60,6 +61,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
@@ -70,6 +72,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
 import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
 import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 
 public class ContainerLaunch implements Callable<Integer> {
@@ -88,6 +91,7 @@ public class ContainerLaunch implements 
   private final Container container;
   private final Configuration conf;
   private final Context context;
+  private final ContainerManagerImpl containerManager;
   
   private volatile AtomicBoolean shouldLaunchContainer = new AtomicBoolean(false);
   private volatile AtomicBoolean completed = new AtomicBoolean(false);
@@ -101,7 +105,8 @@ public class ContainerLaunch implements 
 
   public ContainerLaunch(Context context, Configuration configuration,
       Dispatcher dispatcher, ContainerExecutor exec, Application app,
-      Container container, LocalDirsHandlerService dirsHandler) {
+      Container container, LocalDirsHandlerService dirsHandler,
+      ContainerManagerImpl containerManager) {
     this.context = context;
     this.conf = configuration;
     this.app = app;
@@ -109,6 +114,7 @@ public class ContainerLaunch implements 
     this.container = container;
     this.dispatcher = dispatcher;
     this.dirsHandler = dirsHandler;
+    this.containerManager = containerManager;
     this.sleepDelayBeforeSigKill =
         conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
             YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
@@ -227,7 +233,6 @@ public class ContainerLaunch implements 
             ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME, 
             new Path(containerWorkDir, 
                 FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
-
         // Sanitize the container's environment
         sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
           localResources);
@@ -525,7 +530,8 @@ public class ContainerLaunch implements 
 
     @Override
     public void env(String key, String value) {
-      line("@set ", key, "=", value);
+      line("@set ", key, "=", value,
+          "\nif %errorlevel% neq 0 exit /b %errorlevel%");
     }
 
     @Override
@@ -588,20 +594,18 @@ public class ContainerLaunch implements 
     environment.put(Environment.LOG_DIRS.name(),
       StringUtils.join(",", containerLogDirs));
 
-    putEnvIfNotNull(environment, Environment.USER.name(), container.getUser());
-    
-    putEnvIfNotNull(environment, 
-        Environment.LOGNAME.name(),container.getUser());
+    environment.put(Environment.USER.name(), container.getUser());
     
-    putEnvIfNotNull(environment, 
-        Environment.HOME.name(),
+    environment.put(Environment.LOGNAME.name(), container.getUser());
+
+    environment.put(Environment.HOME.name(),
         conf.get(
             YarnConfiguration.NM_USER_HOME_DIR, 
             YarnConfiguration.DEFAULT_NM_USER_HOME_DIR
             )
         );
     
-    putEnvIfNotNull(environment, Environment.PWD.name(), pwd.toString());
+    environment.put(Environment.PWD.name(), pwd.toString());
     
     putEnvIfNotNull(environment, 
         Environment.HADOOP_CONF_DIR.name(), 
@@ -681,6 +685,12 @@ public class ContainerLaunch implements 
         environment.put(Environment.CLASSPATH.name(), classPathJar);
       }
     }
+    // put AuxiliaryService data to environment
+    for (Map.Entry<String, ByteBuffer> meta : containerManager
+        .getAuxServiceMetaData().entrySet()) {
+      AuxiliaryServiceHelper.setServiceDataIntoEnv(
+          meta.getKey(), meta.getValue(), environment);
+    }
   }
     
   static void writeLaunchEnv(OutputStream out,

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java Thu Sep 19 23:42:10 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
@@ -64,6 +66,7 @@ public class ContainersLauncher extends 
   private final Context context;
   private final ContainerExecutor exec;
   private final Dispatcher dispatcher;
+  private final ContainerManagerImpl containerManager;
 
   private LocalDirsHandlerService dirsHandler;
   @VisibleForTesting
@@ -88,12 +91,14 @@ public class ContainersLauncher extends 
 
 
   public ContainersLauncher(Context context, Dispatcher dispatcher,
-      ContainerExecutor exec, LocalDirsHandlerService dirsHandler) {
+      ContainerExecutor exec, LocalDirsHandlerService dirsHandler,
+      ContainerManagerImpl containerManager) {
     super("containers-launcher");
     this.exec = exec;
     this.context = context;
     this.dispatcher = dispatcher;
     this.dirsHandler = dirsHandler;
+    this.containerManager = containerManager;
   }
 
   @Override
@@ -127,7 +132,7 @@ public class ContainersLauncher extends 
 
         ContainerLaunch launch =
             new ContainerLaunch(context, getConfig(), dispatcher, exec, app,
-              event.getContainer(), dirsHandler);
+              event.getContainer(), dirsHandler, containerManager);
         running.put(containerId,
             new RunningContainer(containerLauncher.submit(launch), 
                 launch));
@@ -149,7 +154,8 @@ public class ContainersLauncher extends 
               dispatcher.getEventHandler().handle(
                   new ContainerExitEvent(containerId,
                       ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
-                      ExitCode.TERMINATED.getExitCode(),
+                      Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() : 
+                        ExitCode.TERMINATED.getExitCode(),
                       "Container terminated before launch."));
             }
           }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java Thu Sep 19 23:42:10 2013
@@ -28,36 +28,21 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.List;
 
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SecureIOUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
 import org.apache.hadoop.yarn.webapp.SubView;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.PRE;
 import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import org.mortbay.log.Log;
 
 import com.google.inject.Inject;
 
@@ -90,19 +75,11 @@ public class ContainerLogsPage extends N
 
   public static class ContainersLogsBlock extends HtmlBlock implements
       YarnWebParams {    
-    private final Configuration conf;
     private final Context nmContext;
-    private final ApplicationACLsManager aclsManager;
-    private final LocalDirsHandlerService dirsHandler;
 
     @Inject
-    public ContainersLogsBlock(Configuration conf, Context context,
-        ApplicationACLsManager aclsManager,
-        LocalDirsHandlerService dirsHandler) {
-      this.conf = conf;
+    public ContainersLogsBlock(Context context) {
       this.nmContext = context;
-      this.aclsManager = aclsManager;
-      this.dirsHandler = dirsHandler;
     }
 
     @Override
@@ -114,229 +91,123 @@ public class ContainerLogsPage extends N
         		" server. Log Server url may not be configured");
         //Intentional fallthrough.
       }
-      
+
       ContainerId containerId;
       try {
         containerId = ConverterUtils.toContainerId($(CONTAINER_ID));
-      } catch (IllegalArgumentException e) {
-        html.h1("Invalid containerId " + $(CONTAINER_ID));
+      } catch (IllegalArgumentException ex) {
+        html.h1("Invalid container ID: " + $(CONTAINER_ID));
         return;
       }
 
-      ApplicationId applicationId = containerId.getApplicationAttemptId()
-          .getApplicationId();
-      Application application = this.nmContext.getApplications().get(
-          applicationId);
-      Container container = this.nmContext.getContainers().get(containerId);
-
-      if (application == null) {
-        html.h1(
-            "Unknown container. Container either has not started or "
-                + "has already completed or "
-                + "doesn't belong to this node at all.");
-        return;
-      }
-      if (container == null) {
-        // Container may have alerady completed, but logs not aggregated yet.
-        printLogs(html, containerId, applicationId, application);
-        return;
-      }
-
-      if (EnumSet.of(ContainerState.NEW, ContainerState.LOCALIZING,
-          ContainerState.LOCALIZED).contains(container.getContainerState())) {
-        html.h1("Container is not yet running. Current state is "
-                + container.getContainerState());
-        return;
-      }
-
-      if (container.getContainerState() == ContainerState.LOCALIZATION_FAILED) {
-        html.h1("Container wasn't started. Localization failed.");
-        return;
-      }
-
-      if (EnumSet.of(ContainerState.RUNNING,
-          ContainerState.EXITED_WITH_FAILURE,
-          ContainerState.EXITED_WITH_SUCCESS).contains(
-          container.getContainerState())) {
-        printLogs(html, containerId, applicationId, application);
-        return;
-      }
-      if (EnumSet.of(ContainerState.KILLING,
-          ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
-          ContainerState.CONTAINER_RESOURCES_CLEANINGUP).contains(
-          container.getContainerState())) {
-        //Container may have generated some logs before being killed.
-        printLogs(html, containerId, applicationId, application);
-        return;
-      }
-      if (container.getContainerState().equals(ContainerState.DONE)) {
-        // Prev state unknown. Logs may be available.
-        printLogs(html, containerId, applicationId, application);
-        return;
-      } else {
-        html.h1("Container is no longer running...");
-        return;
+      try {
+        if ($(CONTAINER_LOG_TYPE).isEmpty()) {
+          List<File> logFiles = ContainerLogsUtils.getContainerLogDirs(containerId,
+              request().getRemoteUser(), nmContext);
+          printLogFileDirectory(html, logFiles);
+        } else {
+          File logFile = ContainerLogsUtils.getContainerLogFile(containerId,
+              $(CONTAINER_LOG_TYPE), request().getRemoteUser(), nmContext);
+          printLogFile(html, logFile);
+        }
+      } catch (YarnException ex) {
+        html.h1(ex.getMessage());
+      } catch (NotFoundException ex) {
+        html.h1(ex.getMessage());
       }
     }
-
-    private void printLogs(Block html, ContainerId containerId,
-        ApplicationId applicationId, Application application) {
-      // Check for the authorization.
-      String remoteUser = request().getRemoteUser();
-      UserGroupInformation callerUGI = null;
-
-      if (remoteUser != null) {
-        callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
-      }
-      if (callerUGI != null
-          && !this.aclsManager.checkAccess(callerUGI,
-              ApplicationAccessType.VIEW_APP, application.getUser(),
-              applicationId)) {
-        html.h1(
-            "User [" + remoteUser
-                + "] is not authorized to view the logs for application "
-                + applicationId);
+    
+    private void printLogFile(Block html, File logFile) {
+      long start =
+          $("start").isEmpty() ? -4 * 1024 : Long.parseLong($("start"));
+      start = start < 0 ? logFile.length() + start : start;
+      start = start < 0 ? 0 : start;
+      long end =
+          $("end").isEmpty() ? logFile.length() : Long.parseLong($("end"));
+      end = end < 0 ? logFile.length() + end : end;
+      end = end < 0 ? logFile.length() : end;
+      if (start > end) {
+        html.h1("Invalid start and end values. Start: [" + start + "]"
+            + ", end[" + end + "]");
         return;
-      }
+      } else {
+        FileInputStream logByteStream = null;
 
-      if (!$(CONTAINER_LOG_TYPE).isEmpty()) {
-        File logFile = null;
         try {
-          URI logPathURI = new URI(this.dirsHandler.getLogPathToRead(
-              ContainerLaunch.getRelativeContainerLogDir(
-                  applicationId.toString(), containerId.toString())
-                  + Path.SEPARATOR + $(CONTAINER_LOG_TYPE)).toString());
-          logFile = new File(logPathURI.getPath());
-        } catch (URISyntaxException e) {
-          html.h1("Cannot find this log on the local disk.");
-          return;
-        } catch (Exception e) {
-          html.h1("Cannot find this log on the local disk.");
+          logByteStream = ContainerLogsUtils.openLogFileForRead($(CONTAINER_ID),
+              logFile, nmContext);
+        } catch (IOException ex) {
+          html.h1(ex.getMessage());
           return;
         }
-        long start =
-            $("start").isEmpty() ? -4 * 1024 : Long.parseLong($("start"));
-        start = start < 0 ? logFile.length() + start : start;
-        start = start < 0 ? 0 : start;
-        long end =
-            $("end").isEmpty() ? logFile.length() : Long.parseLong($("end"));
-        end = end < 0 ? logFile.length() + end : end;
-        end = end < 0 ? logFile.length() : end;
-        if (start > end) {
-          html.h1("Invalid start and end values. Start: [" + start + "]"
-              + ", end[" + end + "]");
-          return;
-        } else {
-          FileInputStream logByteStream = null;
-          
-          try {
-            logByteStream =
-                SecureIOUtils.openForRead(logFile, application.getUser(), null);
-          }  catch (IOException e) {
-            LOG.error(
-              "Exception reading log file " + logFile.getAbsolutePath(), e);
-            if (e.getMessage().contains(
-              "did not match expected owner '" + application.getUser()
-                  + "'")) {
-              html.h1("Exception reading log file. Application submitted by '"
-                  + application.getUser()
-                  + "' doesn't own requested log file : "
-                  + logFile.getName());
-            } else {
-              html.h1("Exception reading log file. It might be because log "
-                  + "file was aggregated : " + logFile.getName());
-            }
-            return;
+        
+        try {
+          long toRead = end - start;
+          if (toRead < logFile.length()) {
+            html.p()._("Showing " + toRead + " bytes. Click ")
+                .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER), 
+                    logFile.getName(), "?start=0"), "here").
+                    _(" for full log")._();
           }
           
-          try {
-            long toRead = end - start;
-            if (toRead < logFile.length()) {
-              html.p()._("Showing " + toRead + " bytes. Click ")
-                  .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER), 
-                      logFile.getName(), "?start=0"), "here").
-                      _(" for full log")._();
-            }
-            // TODO Fix findBugs close warning along with IOUtils change
-            IOUtils.skipFully(logByteStream, start);
-            InputStreamReader reader = new InputStreamReader(logByteStream);
-            int bufferSize = 65536;
-            char[] cbuf = new char[bufferSize];
-
-            int len = 0;
-            int currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
-            PRE<Hamlet> pre = html.pre();
-
-            while ((len = reader.read(cbuf, 0, currentToRead)) > 0
-                && toRead > 0) {
-              pre._(new String(cbuf, 0, len));
-              toRead = toRead - len;
-              currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
-            }
+          IOUtils.skipFully(logByteStream, start);
+          InputStreamReader reader = new InputStreamReader(logByteStream);
+          int bufferSize = 65536;
+          char[] cbuf = new char[bufferSize];
+
+          int len = 0;
+          int currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
+          PRE<Hamlet> pre = html.pre();
+
+          while ((len = reader.read(cbuf, 0, currentToRead)) > 0
+              && toRead > 0) {
+            pre._(new String(cbuf, 0, len));
+            toRead = toRead - len;
+            currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
+          }
 
-            pre._();
-            reader.close();
+          pre._();
+          reader.close();
 
-          } catch (IOException e) {
-            LOG.error(
+        } catch (IOException e) {
+          LOG.error(
               "Exception reading log file " + logFile.getAbsolutePath(), e);
-            html.h1("Exception reading log file. It might be because log "
+          html.h1("Exception reading log file. It might be because log "
                 + "file was aggregated : " + logFile.getName());
-          } finally {
-            if (logByteStream != null) {
-              try {
-                logByteStream.close();
-              } catch (IOException e) {
-                // Ignore
-              }
+        } finally {
+          if (logByteStream != null) {
+            try {
+              logByteStream.close();
+            } catch (IOException e) {
+              // Ignore
             }
           }
         }
-      } else {
-        // Print out log types in lexical order
-        List<File> containerLogsDirs = getContainerLogDirs(containerId,
-            dirsHandler);
-        Collections.sort(containerLogsDirs);
-        boolean foundLogFile = false;
-        for (File containerLogsDir : containerLogsDirs) {
-          File[] logFiles = containerLogsDir.listFiles();
-          if (logFiles != null) {
-            Arrays.sort(logFiles);
-            for (File logFile : logFiles) {
-              foundLogFile = true;
-              html.p()
-                  .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
-                      logFile.getName(), "?start=-4096"),
-                      logFile.getName() + " : Total file length is "
-                          + logFile.length() + " bytes.")._();
-            }
-          }
-        }
-        if (!foundLogFile) {
-          html.h1("No logs available for container " + containerId.toString());
-          return;
-        }
       }
-      return;
     }
-
-    static List<File> getContainerLogDirs(ContainerId containerId,
-        LocalDirsHandlerService dirsHandler) {
-      List<String> logDirs = dirsHandler.getLogDirs();
-      List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
-      for (String logDir : logDirs) {
-        try {
-          logDir = new URI(logDir).getPath();
-        } catch (URISyntaxException e) {
-          Log.warn(e.getMessage());
+    
+    private void printLogFileDirectory(Block html, List<File> containerLogsDirs) {
+      // Print out log types in lexical order
+      Collections.sort(containerLogsDirs);
+      boolean foundLogFile = false;
+      for (File containerLogsDir : containerLogsDirs) {
+        File[] logFiles = containerLogsDir.listFiles();
+        if (logFiles != null) {
+          Arrays.sort(logFiles);
+          for (File logFile : logFiles) {
+            foundLogFile = true;
+            html.p()
+                .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
+                    logFile.getName(), "?start=-4096"),
+                    logFile.getName() + " : Total file length is "
+                        + logFile.length() + " bytes.")._();
+          }
         }
-        String appIdStr = ConverterUtils.toString(containerId
-            .getApplicationAttemptId().getApplicationId());
-        File appLogDir = new File(logDir, appIdStr);
-        String containerIdStr = ConverterUtils.toString(containerId);
-        containerLogDirs.add(new File(appLogDir, containerIdStr));
       }
-      return containerLogDirs;
+      if (!foundLogFile) {
+        html.h1("No logs available for container " + $(CONTAINER_ID));
+        return;
+      }
     }
   }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java Thu Sep 19 23:42:10 2013
@@ -17,19 +17,31 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.webapp;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.util.Map.Entry;
 
+import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import javax.ws.rs.core.StreamingOutput;
 import javax.ws.rs.core.UriInfo;
 
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -60,6 +72,9 @@ public class NMWebServices {
       .getRecordFactory(null);
 
   private @javax.ws.rs.core.Context 
+    HttpServletRequest request;
+  
+  private @javax.ws.rs.core.Context 
     HttpServletResponse response;
 
   @javax.ws.rs.core.Context
@@ -179,5 +194,66 @@ public class NMWebServices {
         .toString(), webapp.name());
 
   }
-
+  
+  /**
+   * Returns the contents of a container's log file in plain text. 
+   *
+   * Only works for containers that are still in the NodeManager's memory, so
+   * logs are no longer available after the corresponding application is no
+   * longer running.
+   * 
+   * @param containerIdStr
+   *    The container ID
+   * @param filename
+   *    The name of the log file
+   * @return
+   *    The contents of the container's log file
+   */
+  @GET
+  @Path("/containerlogs/{containerid}/{filename}")
+  @Produces({ MediaType.TEXT_PLAIN })
+  @Public
+  @Unstable
+  public Response getLogs(@PathParam("containerid") String containerIdStr,
+      @PathParam("filename") String filename) {
+    ContainerId containerId;
+    try {
+      containerId = ConverterUtils.toContainerId(containerIdStr);
+    } catch (IllegalArgumentException ex) {
+      return Response.status(Status.BAD_REQUEST).build();
+    }
+    
+    File logFile = null;
+    try {
+      logFile = ContainerLogsUtils.getContainerLogFile(
+          containerId, filename, request.getRemoteUser(), nmContext);
+    } catch (NotFoundException ex) {
+      return Response.status(Status.NOT_FOUND).entity(ex.getMessage()).build();
+    } catch (YarnException ex) {
+      return Response.serverError().entity(ex.getMessage()).build();
+    }
+    
+    try {
+      final FileInputStream fis = ContainerLogsUtils.openLogFileForRead(
+          containerIdStr, logFile, nmContext);
+      
+      StreamingOutput stream = new StreamingOutput() {
+        @Override
+        public void write(OutputStream os) throws IOException,
+            WebApplicationException {
+          int bufferSize = 65536;
+          byte[] buf = new byte[bufferSize];
+          int len;
+          while ((len = fis.read(buf, 0, bufferSize)) > 0) {
+            os.write(buf, 0, len);
+          }
+          os.flush();
+        }
+      };
+      
+      return Response.ok(stream).build();
+    } catch (IOException ex) {
+      return Response.serverError().entity(ex.getMessage()).build();
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java Thu Sep 19 23:42:10 2013
@@ -59,8 +59,15 @@ public class WebServer extends AbstractS
     LOG.info("Instantiating NMWebApp at " + bindAddress);
     try {
       this.webApp =
-          WebApps.$for("node", Context.class, this.nmContext, "ws")
-              .at(bindAddress).with(getConfig()).start(this.nmWebApp);
+          WebApps
+            .$for("node", Context.class, this.nmContext, "ws")
+            .at(bindAddress)
+            .with(getConfig())
+            .withHttpSpnegoPrincipalKey(
+              YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
+            .withHttpSpnegoKeytabKey(
+              YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+            .start(this.nmWebApp);
       this.port = this.webApp.httpServer().getPort();
     } catch (Exception e) {
       String msg = "NMWebapps failed to start.";

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Thu Sep 19 23:42:10 2013
@@ -145,7 +145,7 @@ public class DummyContainerManager exten
   protected ContainersLauncher createContainersLauncher(Context context,
       ContainerExecutor exec) {
     return new ContainersLauncher(context, super.dispatcher, exec,
-                                  super.dirsHandler) {
+                                  super.dirsHandler, this) {
       @Override
       public void handle(ContainersLauncherEvent event) {
         Container container = event.getContainer();

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Thu Sep 19 23:42:10 2013
@@ -79,7 +79,7 @@ public class TestEventFlow {
     YarnConfiguration conf = new YarnConfiguration();
     
     Context context = new NMContext(new NMContainerTokenSecretManager(conf),
-        new NMTokenSecretManagerInNM()) {
+        new NMTokenSecretManagerInNM(), null, null) {
       @Override
       public int getHttpPort() {
         return 1234;

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Thu Sep 19 23:42:10 2013
@@ -23,6 +23,7 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
@@ -163,7 +164,8 @@ public class TestNodeManagerShutdown {
     ContainerLaunchContext containerLaunchContext =
         recordFactory.newRecordInstance(ContainerLaunchContext.class);
 
-    NodeId nodeId = BuilderUtils.newNodeId("localhost", 12345);
+    NodeId nodeId = BuilderUtils.newNodeId(InetAddress.getByName("localhost")
+        .getCanonicalHostName(), 12345);
     
     URL localResourceUri =
         ConverterUtils.getYarnUrlFromPath(localFS



Mime
View raw message