hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1190174 [3/3] - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/or...
Date Fri, 28 Oct 2011 06:45:07 GMT
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java
Fri Oct 28 06:45:04 2011
@@ -23,4 +23,6 @@ public interface NMWebParams {
   String APPLICATION_ID = "nm.appId";
   String CONTAINER_ID = "nm.containerId";
   String CONTAINER_LOG_TYPE= "nm.containerLogType";
+  String ENTITY_STRING = "nm.entityString";
+  String APP_OWNER = "nm.appOwner";
 }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
Fri Oct 28 06:45:04 2011
@@ -99,7 +99,8 @@ public class WebServer extends AbstractS
           "application");
       route(pajoin("/container", CONTAINER_ID), NMController.class,
           "container");
-      route(pajoin("/containerlogs", CONTAINER_ID, CONTAINER_LOG_TYPE),
+      route(
+          pajoin("/containerlogs", CONTAINER_ID, APP_OWNER, CONTAINER_LOG_TYPE),
           NMController.class, "logs");
     }
 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
Fri Oct 28 06:45:04 2011
@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -56,8 +57,6 @@ public class DummyContainerManager exten
 
   private static final Log LOG = LogFactory
       .getLog(DummyContainerManager.class);
-
-  private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
   
   public DummyContainerManager(Context context, ContainerExecutor exec,
       DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
@@ -150,7 +149,8 @@ public class DummyContainerManager exten
   @Override
   protected LogAggregationService createLogAggregationService(Context context, 
       DeletionService deletionService) {
-    return new LogAggregationService(context, deletionService) {
+    return new LogAggregationService(new AsyncDispatcher(), context,
+        deletionService) {
       @Override
       public void handle(LogAggregatorEvent event) {
         switch (event.getType()) {

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
Fri Oct 28 06:45:04 2011
@@ -20,6 +20,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.event.DrainDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -351,6 +352,7 @@ public class TestApplication {
     final EventHandler<LogAggregatorEvent> logAggregationBus;
     final String user;
     final List<Container> containers;
+    final Context context;
 
     final ApplicationId appId;
     final Application app;
@@ -373,11 +375,13 @@ public class TestApplication {
       dispatcher.register(ContainerEventType.class, containerBus);
       dispatcher.register(LogAggregatorEventType.class, logAggregationBus);
 
+      context = mock(Context.class);
+      
       this.user = user;
       this.appId = BuilderUtils.newApplicationId(timestamp, id);
 
       app = new ApplicationImpl(dispatcher, new ApplicationACLsManager(
-          new Configuration()), this.user, appId, null);
+          new Configuration()), this.user, appId, null, context);
       containers = new ArrayList<Container>();
       for (int i = 0; i < numContainers; i++) {
         containers.add(createMockedContainer(this.appId, i));

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Fri Oct 28 06:45:04 2011
@@ -18,6 +18,12 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.times;
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+
 import java.io.DataInputStream;
 import java.io.EOFException;
 import java.io.File;
@@ -28,8 +34,10 @@ import java.io.Writer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import junit.framework.Assert;
 
@@ -41,6 +49,7 @@ import org.apache.hadoop.io.DataOutputBu
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -50,11 +59,15 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
 import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogKey;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogReader;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorAppFinishedEvent;
@@ -62,12 +75,15 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorContainerFinishedEvent;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.junit.Ignore;
 import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
 
-@Ignore
+//@Ignore
 public class TestLogAggregationService extends BaseContainerManagerTest {
 
+  private Map<ApplicationAccessType, String> acls = createAppAcls();
+  
   static {
     LOG = LogFactory.getLog(TestLogAggregationService.class);
   }
@@ -91,17 +107,25 @@ public class TestLogAggregationService e
   }
 
   @Test
+  @SuppressWarnings("unchecked")
   public void testLocalFileDeletionAfterUpload() throws IOException {
     this.delSrvc = new DeletionService(createContainerExecutor());
     this.delSrvc.init(conf);
     this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
     this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
         this.remoteRootLogDir.getAbsolutePath());
+    this.conf.setBoolean(YarnConfiguration.NM_LOG_AGGREGATION_ENABLED, true);
+    
+    DrainDispatcher dispatcher = createDispatcher();
+    EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);
+    dispatcher.register(ApplicationEventType.class, appEventHandler);
+    
     LogAggregationService logAggregationService =
-        new LogAggregationService(this.context, this.delSrvc);
+        new LogAggregationService(dispatcher, this.context, this.delSrvc);
     logAggregationService.init(this.conf);
     logAggregationService.start();
 
+    
     ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
 
     // AppLogDir should be created
@@ -111,13 +135,11 @@ public class TestLogAggregationService e
     logAggregationService
         .handle(new LogAggregatorAppStartedEvent(
             application1, this.user, null,
-            ContainerLogsRetentionPolicy.ALL_CONTAINERS));
+            ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls));
 
-    ApplicationAttemptId appAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    appAttemptId.setApplicationId(application1);
-    appAttemptId.setAttemptId(1);
-    ContainerId container11 =
-        BuilderUtils.newContainerId(recordFactory, application1, appAttemptId, 1);
+    ApplicationAttemptId appAttemptId =
+        BuilderUtils.newApplicationAttemptId(application1, 1);
+    ContainerId container11 = BuilderUtils.newContainerId(appAttemptId, 1);
     // Simulate log-file creation
     writeContainerLogs(app1LogDir, container11);
     logAggregationService.handle(
@@ -128,6 +150,7 @@ public class TestLogAggregationService e
 
     logAggregationService.stop();
 
+    
     String containerIdStr = ConverterUtils.toString(container11);
     File containerLogDir = new File(app1LogDir, containerIdStr);
     for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
@@ -136,17 +159,37 @@ public class TestLogAggregationService e
 
     Assert.assertFalse(app1LogDir.exists());
 
-    Assert.assertTrue(new File(logAggregationService
-        .getRemoteNodeLogFileForApp(application1).toUri().getPath()).exists());
+    Path logFilePath =
+        logAggregationService.getRemoteNodeLogFileForApp(application1,
+            this.user);
+    Assert.assertTrue("Log file [" + logFilePath + "] not found", new File(
+        logFilePath.toUri().getPath()).exists());
+    
+    dispatcher.await();
+    ArgumentCaptor<ApplicationEvent> eventCaptor =
+        ArgumentCaptor.forClass(ApplicationEvent.class);
+    verify(appEventHandler).handle(eventCaptor.capture());
+    assertEquals(ApplicationEventType.APPLICATION_LOG_AGGREGATION_FINISHED,
+        eventCaptor.getValue().getType());
+    assertEquals(appAttemptId.getApplicationId(), eventCaptor.getValue()
+        .getApplicationID());
+    
   }
 
   @Test
+  @SuppressWarnings("unchecked")
   public void testNoContainerOnNode() {
     this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
     this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
         this.remoteRootLogDir.getAbsolutePath());
+    this.conf.setBoolean(YarnConfiguration.NM_LOG_AGGREGATION_ENABLED, true);
+    
+    DrainDispatcher dispatcher = createDispatcher();
+    EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);
+    dispatcher.register(ApplicationEventType.class, appEventHandler);
+    
     LogAggregationService logAggregationService =
-        new LogAggregationService(this.context, this.delSrvc);
+        new LogAggregationService(dispatcher, this.context, this.delSrvc);
     logAggregationService.init(this.conf);
     logAggregationService.start();
 
@@ -159,27 +202,43 @@ public class TestLogAggregationService e
     logAggregationService
         .handle(new LogAggregatorAppStartedEvent(
             application1, this.user, null,
-            ContainerLogsRetentionPolicy.ALL_CONTAINERS));
+            ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls));
 
     logAggregationService.handle(new LogAggregatorAppFinishedEvent(
         application1));
 
     logAggregationService.stop();
 
-    Assert
-        .assertFalse(new File(logAggregationService
-            .getRemoteNodeLogFileForApp(application1).toUri().getPath())
-            .exists());
+    Assert.assertFalse(new File(logAggregationService
+        .getRemoteNodeLogFileForApp(application1, this.user).toUri().getPath())
+        .exists());
+    
+    dispatcher.await();
+    ArgumentCaptor<ApplicationEvent> eventCaptor =
+        ArgumentCaptor.forClass(ApplicationEvent.class);
+    verify(appEventHandler).handle(eventCaptor.capture());
+    assertEquals(ApplicationEventType.APPLICATION_LOG_AGGREGATION_FINISHED,
+        eventCaptor.getValue().getType());
+    verify(appEventHandler).handle(eventCaptor.capture());
+    assertEquals(application1, eventCaptor.getValue()
+        .getApplicationID());
   }
 
   @Test
+  @SuppressWarnings("unchecked")
   public void testMultipleAppsLogAggregation() throws IOException {
 
     this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
     this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
         this.remoteRootLogDir.getAbsolutePath());
+    this.conf.setBoolean(YarnConfiguration.NM_LOG_AGGREGATION_ENABLED, true);
+    
+    DrainDispatcher dispatcher = createDispatcher();
+    EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);
+    dispatcher.register(ApplicationEventType.class, appEventHandler);
+    
     LogAggregationService logAggregationService =
-        new LogAggregationService(this.context, this.delSrvc);
+        new LogAggregationService(dispatcher, this.context, this.delSrvc);
     logAggregationService.init(this.conf);
     logAggregationService.start();
 
@@ -192,81 +251,69 @@ public class TestLogAggregationService e
     logAggregationService
         .handle(new LogAggregatorAppStartedEvent(
             application1, this.user, null,
-            ContainerLogsRetentionPolicy.ALL_CONTAINERS));
+            ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls));
 
-    ApplicationAttemptId appAttemptId1 = 
-        recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    appAttemptId1.setApplicationId(application1);
-    ContainerId container11 =
-        BuilderUtils.newContainerId(recordFactory, application1, appAttemptId1, 1);
+    ApplicationAttemptId appAttemptId1 =
+        BuilderUtils.newApplicationAttemptId(application1, 1);
+    ContainerId container11 = BuilderUtils.newContainerId(appAttemptId1, 1);
+    
     // Simulate log-file creation
     writeContainerLogs(app1LogDir, container11);
     logAggregationService.handle(
         new LogAggregatorContainerFinishedEvent(container11, 0));
 
     ApplicationId application2 = BuilderUtils.newApplicationId(1234, 2);
-    ApplicationAttemptId appAttemptId2 = 
-        recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    appAttemptId1.setApplicationId(application2);
+    ApplicationAttemptId appAttemptId2 =
+        BuilderUtils.newApplicationAttemptId(application2, 1);
 
     File app2LogDir =
       new File(localLogDir, ConverterUtils.toString(application2));
     app2LogDir.mkdir();
     logAggregationService.handle(new LogAggregatorAppStartedEvent(
         application2, this.user, null,
-        ContainerLogsRetentionPolicy.APPLICATION_MASTER_ONLY));
+        ContainerLogsRetentionPolicy.APPLICATION_MASTER_ONLY, this.acls));
 
     
-    ContainerId container21 =
-        BuilderUtils.newContainerId(recordFactory, application2, 
-            appAttemptId2, 1);
+    ContainerId container21 = BuilderUtils.newContainerId(appAttemptId2, 1);
+    
     writeContainerLogs(app2LogDir, container21);
     logAggregationService.handle(
         new LogAggregatorContainerFinishedEvent(container21, 0));
 
-    ContainerId container12 =
-        BuilderUtils.newContainerId(recordFactory, application1, appAttemptId1, 
-            2);
+    ContainerId container12 = BuilderUtils.newContainerId(appAttemptId1, 2);
+
     writeContainerLogs(app1LogDir, container12);
     logAggregationService.handle(
         new LogAggregatorContainerFinishedEvent(container12, 0));
 
     ApplicationId application3 = BuilderUtils.newApplicationId(1234, 3);
-    ApplicationAttemptId appAttemptId3 = 
-        recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    appAttemptId1.setApplicationId(application3);
+    ApplicationAttemptId appAttemptId3 =
+        BuilderUtils.newApplicationAttemptId(application3, 1);
 
     File app3LogDir =
       new File(localLogDir, ConverterUtils.toString(application3));
     app3LogDir.mkdir();
-    logAggregationService.handle(new LogAggregatorAppStartedEvent(
-        application3, this.user, null,
-        ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY));
+    logAggregationService.handle(new LogAggregatorAppStartedEvent(application3,
+        this.user, null,
+        ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY, this.acls));
+        
 
-    ContainerId container31 =
-        BuilderUtils.newContainerId(recordFactory, application3, appAttemptId3, 
-            1);
+    ContainerId container31 = BuilderUtils.newContainerId(appAttemptId3, 1);
     writeContainerLogs(app3LogDir, container31);
     logAggregationService.handle(
         new LogAggregatorContainerFinishedEvent(container31, 0));
 
-    ContainerId container32 =
-        BuilderUtils.newContainerId(recordFactory, application3, appAttemptId3,
-            2);
+    ContainerId container32 = BuilderUtils.newContainerId(appAttemptId3, 2);
     writeContainerLogs(app3LogDir, container32);
     logAggregationService.handle(
         new LogAggregatorContainerFinishedEvent(container32, 1)); // Failed 
 
-    ContainerId container22 =
-        BuilderUtils.newContainerId(recordFactory, application2, appAttemptId2, 
-            2);
+    ContainerId container22 = BuilderUtils.newContainerId(appAttemptId2, 2);
     writeContainerLogs(app2LogDir, container22);
     logAggregationService.handle(
         new LogAggregatorContainerFinishedEvent(container22, 0));
 
-    ContainerId container33 =
-        BuilderUtils.newContainerId(recordFactory, application3, appAttemptId3, 
-            3);
+    ContainerId container33 = BuilderUtils.newContainerId(appAttemptId3, 3);
     writeContainerLogs(app3LogDir, container33);
     logAggregationService.handle(
         new LogAggregatorContainerFinishedEvent(container33, 0));
@@ -286,6 +333,22 @@ public class TestLogAggregationService e
         new ContainerId[] { container21 });
     verifyContainerLogs(logAggregationService, application3,
         new ContainerId[] { container31, container32 });
+    
+    dispatcher.await();
+    ArgumentCaptor<ApplicationEvent> eventCaptor =
+        ArgumentCaptor.forClass(ApplicationEvent.class);
+
+    verify(appEventHandler, times(3)).handle(eventCaptor.capture());
+    List<ApplicationEvent> capturedEvents = eventCaptor.getAllValues();
+    Set<ApplicationId> appIds = new HashSet<ApplicationId>();
+    for (ApplicationEvent cap : capturedEvents) {
+      assertEquals(ApplicationEventType.APPLICATION_LOG_AGGREGATION_FINISHED,
+          eventCaptor.getValue().getType());
+      appIds.add(cap.getApplicationID());
+    }
+    assertTrue(appIds.contains(application1));
+    assertTrue(appIds.contains(application2));
+    assertTrue(appIds.contains(application3));
   }
 
   private void writeContainerLogs(File appLogDir, ContainerId containerId)
@@ -306,7 +369,11 @@ public class TestLogAggregationService e
       ContainerId[] expectedContainerIds) throws IOException {
     AggregatedLogFormat.LogReader reader =
         new AggregatedLogFormat.LogReader(this.conf,
-            logAggregationService.getRemoteNodeLogFileForApp(appId));
+            logAggregationService.getRemoteNodeLogFileForApp(appId, this.user));
+    
+    Assert.assertEquals(this.user, reader.getApplicationOwner());
+    verifyAcls(reader.getApplicationAcls());
+    
     try {
       Map<String, Map<String, String>> logMap =
           new HashMap<String, Map<String, String>>();
@@ -380,8 +447,10 @@ public class TestLogAggregationService e
   public void testLogAggregationForRealContainerLaunch() throws IOException,
       InterruptedException {
 
+    this.conf.setBoolean(YarnConfiguration.NM_LOG_AGGREGATION_ENABLED, true);
     this.containerManager.start();
 
+
     File scriptFile = new File(tmpDir, "scriptFile.sh");
     PrintWriter fileWriter = new PrintWriter(scriptFile);
     fileWriter.write("\necho Hello World! Stdout! > "
@@ -400,13 +469,10 @@ public class TestLogAggregationService e
         recordFactory.newRecordInstance(ApplicationId.class);
     appId.setClusterTimestamp(0);
     appId.setId(0);
-    ApplicationAttemptId appAttemptId = 
-        recordFactory.newRecordInstance(ApplicationAttemptId.class);
-    appAttemptId.setApplicationId(appId);
-    appAttemptId.setAttemptId(1);
-    ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
-    cId.setId(0);
-    cId.setApplicationAttemptId(appAttemptId);
+    ApplicationAttemptId appAttemptId =
+        BuilderUtils.newApplicationAttemptId(appId, 1);
+    ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
+
     containerLaunchContext.setContainerId(cId);
 
     containerLaunchContext.setUser(this.user);
@@ -446,4 +512,27 @@ public class TestLogAggregationService e
         .asList(appId)));
     this.containerManager.stop();
   }
+
+  private void verifyAcls(Map<ApplicationAccessType, String> logAcls) {
+    Assert.assertEquals(this.acls.size(), logAcls.size());
+    for (ApplicationAccessType appAccessType : this.acls.keySet()) {
+      Assert.assertEquals(this.acls.get(appAccessType),
+          logAcls.get(appAccessType));
+    }
+  }
+
+  private DrainDispatcher createDispatcher() {
+    DrainDispatcher dispatcher = new DrainDispatcher();
+    dispatcher.init(this.conf);
+    dispatcher.start();
+    return dispatcher;
+  }
+  
+  private Map<ApplicationAccessType, String> createAppAcls() {
+    Map<ApplicationAccessType, String> appAcls =
+        new HashMap<ApplicationAccessType, String>();
+    appAcls.put(ApplicationAccessType.MODIFY_APP, "user group");
+    appAcls.put(ApplicationAccessType.VIEW_APP, "*");
+    return appAcls;
+  }
 }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
Fri Oct 28 06:45:04 2011
@@ -194,8 +194,13 @@ public class AMLauncher implements Runna
     // consumable by the AM.
     environment.put(ApplicationConstants.AM_CONTAINER_ID_ENV, container
         .getContainerId().toString());
-    environment.put(ApplicationConstants.NM_HTTP_ADDRESS_ENV, application
-        .getMasterContainer().getNodeHttpAddress());
+    environment.put(ApplicationConstants.NM_HOST_ENV, application
+        .getMasterContainer().getNodeId().getHost());
+    environment.put(ApplicationConstants.NM_PORT_ENV,
+        String.valueOf(application.getMasterContainer().getNodeId().getPort()));
+    String parts[] =
+        application.getMasterContainer().getNodeHttpAddress().split(":");
+    environment.put(ApplicationConstants.NM_HTTP_PORT_ENV, parts[1]);
     environment.put(
         ApplicationConstants.APP_SUBMIT_TIME_ENV,
         String.valueOf(rmContext.getRMApps()

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
Fri Oct 28 06:45:04 2011
@@ -54,8 +54,8 @@ public class MockNM {
     return nodeId;
   }
 
-  public String getHttpAddress() {
-    return nodeId.getHost() + ":" + String.valueOf(httpPort);
+  public int getHttpPort() {
+    return httpPort;
   }
 
   public void containerStatus(Container container) throws Exception {

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
Fri Oct 28 06:45:04 2011
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import java.io.IOException;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -60,7 +61,9 @@ public class TestApplicationMasterLaunch
     boolean cleanedup = false;
     String attemptIdAtContainerManager = null;
     String containerIdAtContainerManager = null;
-    String nmAddressAtContainerManager = null;
+    String nmHostAtContainerManager = null;
+    int nmPortAtContainerManager;
+    int nmHttpPortAtContainerManager;
     long submitTimeAtContainerManager;
 
     @Override
@@ -69,20 +72,22 @@ public class TestApplicationMasterLaunch
             throws YarnRemoteException {
       LOG.info("Container started by MyContainerManager: " + request);
       launched = true;
+      Map<String, String> env =
+          request.getContainerLaunchContext().getEnvironment();
       containerIdAtContainerManager =
-          request.getContainerLaunchContext().getEnvironment()
-              .get(ApplicationConstants.AM_CONTAINER_ID_ENV);
+          env.get(ApplicationConstants.AM_CONTAINER_ID_ENV);
       ContainerId containerId =
           ConverterUtils.toContainerId(containerIdAtContainerManager);
       attemptIdAtContainerManager =
           containerId.getApplicationAttemptId().toString();
-      nmAddressAtContainerManager =
-          request.getContainerLaunchContext().getEnvironment()
-              .get(ApplicationConstants.NM_HTTP_ADDRESS_ENV);
+      nmHostAtContainerManager = env.get(ApplicationConstants.NM_HOST_ENV);
+      nmPortAtContainerManager =
+          Integer.parseInt(env.get(ApplicationConstants.NM_PORT_ENV));
+      nmHttpPortAtContainerManager =
+          Integer.parseInt(env.get(ApplicationConstants.NM_HTTP_PORT_ENV));
       submitTimeAtContainerManager =
-          Long.parseLong(request.getContainerLaunchContext().getEnvironment()
-              .get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
-      
+          Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
+
       return null;
     }
 
@@ -168,8 +173,12 @@ public class TestApplicationMasterLaunch
     Assert.assertEquals(app.getRMAppAttempt(appAttemptId)
         .getSubmissionContext().getAMContainerSpec().getContainerId()
         .toString(), containerManager.containerIdAtContainerManager);
-    Assert.assertEquals(nm1.getHttpAddress(),
-        containerManager.nmAddressAtContainerManager);
+    Assert.assertEquals(nm1.getNodeId().getHost(),
+        containerManager.nmHostAtContainerManager);
+    Assert.assertEquals(nm1.getNodeId().getPort(),
+        containerManager.nmPortAtContainerManager);
+    Assert.assertEquals(nm1.getHttpPort(),
+        containerManager.nmHttpPortAtContainerManager);
 
     MockAM am = new MockAM(rm.getRMContext(), rm
         .getApplicationMasterService(), appAttemptId);

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
Fri Oct 28 06:45:04 2011
@@ -325,9 +325,9 @@ Hadoop MapReduce Next Generation - Writi
   
   * When the ApplicationMaster starts up, several parameters are made available
     to it via the environment. These include the ContainerId for the
-    ApplicationMaster container, the application submission time and the HTTP
-    address of the NodeManager running the container. Ref ApplicationConstants
-    for parameter names.
+    ApplicationMaster container, the application submission time and details
+    about the NodeManager host running the Application Master.
+    Ref ApplicationConstants for parameter names.
 
   * All interactions with the ResourceManager require an ApplicationAttemptId 
     (there can be multiple attempts per application in case of failures). The 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml Fri Oct 28 06:45:04 2011
@@ -266,6 +266,11 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.1</version>
+    </dependency>
+    <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-all</artifactId>
       <version>1.8.5</version>

Modified: hadoop/common/trunk/hadoop-mapreduce-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/pom.xml?rev=1190174&r1=1190173&r2=1190174&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/pom.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/pom.xml Fri Oct 28 06:45:04 2011
@@ -157,6 +157,11 @@
       <version>3.2.3.Final</version>
     </dependency>
     <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.1</version>
+    </dependency>
+    <dependency>
       <groupId>com.cenqua.clover</groupId>
       <artifactId>clover</artifactId>
       <version>3.0.2</version>



Mime
View raw message