Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 73ECE9961 for ; Wed, 30 Nov 2011 18:27:59 +0000 (UTC) Received: (qmail 32683 invoked by uid 500); 30 Nov 2011 18:27:59 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 32653 invoked by uid 500); 30 Nov 2011 18:27:59 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 32645 invoked by uid 99); 30 Nov 2011 18:27:59 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Nov 2011 18:27:59 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Nov 2011 18:27:50 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id B689C238899C; Wed, 30 Nov 2011 18:27:27 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1208644 [3/3] - in /hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapreduce-client/hadoop-mapreduce... Date: Wed, 30 Nov 2011 18:27:20 -0000 To: mapreduce-commits@hadoop.apache.org From: atm@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111130182727.B689C238899C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java Wed Nov 30 18:27:04 2011 @@ -35,6 +35,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -51,6 +52,7 @@ public class TestLinuxContainerExecutorW private LinuxContainerExecutor mockExec = null; private final File mockParamFile = new File("./params.txt"); + private LocalDirsHandlerService dirsHandler; private void deleteMockParamFile() { if(mockParamFile.exists()) { @@ -80,6 +82,8 @@ public class TestLinuxContainerExecutorW Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath); mockExec = new LinuxContainerExecutor(); + dirsHandler = new LocalDirsHandlerService(); + dirsHandler.init(conf); mockExec.setConf(conf); } @@ -114,10 +118,13 @@ public class TestLinuxContainerExecutorW mockExec.activateContainer(cId, pidFile); int ret = mockExec.launchContainer(container, scriptPath, tokensPath, - appSubmitter, appId, workDir); + appSubmitter, appId, workDir, dirsHandler.getLocalDirs(), + dirsHandler.getLogDirs()); assertEquals(0, ret); assertEquals(Arrays.asList(appSubmitter, cmd, appId, containerId, - workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString()), + workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), + StringUtils.join(",", dirsHandler.getLocalDirs()), + StringUtils.join(",", dirsHandler.getLogDirs())), readMockParams()); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Wed Nov 30 18:27:04 2011 @@ -29,7 +29,6 @@ import java.util.concurrent.ConcurrentMa import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.NodeHealthCheckerService; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; @@ -440,10 +439,11 @@ public class TestNodeStatusUpdater { ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater, ContainerTokenSecretManager containerTokenSecretManager, - ApplicationACLsManager aclsManager) { + ApplicationACLsManager aclsManager, + LocalDirsHandlerService diskhandler) { return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater, metrics, containerTokenSecretManager, - aclsManager) { + aclsManager, diskhandler) { @Override public void start() { // Simulating failure of starting RPC server Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Wed Nov 30 18:27:04 2011 @@ -45,7 +45,9 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.LocalRMInterface; +import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl; @@ -94,6 +96,8 @@ public abstract class BaseContainerManag protected ContainerExecutor exec; protected DeletionService delSrvc; protected String user = "nobody"; + protected NodeHealthCheckerService nodeHealthChecker; + protected LocalDirsHandlerService dirsHandler; protected NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl( context, new AsyncDispatcher(), null, metrics, this.containerTokenSecretManager) { @@ -147,9 +151,12 @@ public abstract class BaseContainerManag delSrvc.init(conf); exec = createContainerExecutor(); + nodeHealthChecker = new NodeHealthCheckerService(); + nodeHealthChecker.init(conf); + dirsHandler = nodeHealthChecker.getDiskHandler(); containerManager = new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater, metrics, this.containerTokenSecretManager, - new ApplicationACLsManager(conf)); + new ApplicationACLsManager(conf), dirsHandler); containerManager.init(conf); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Wed Nov 30 18:27:04 2011 @@ -383,11 +383,12 @@ public class TestContainerManager extend // Real del service delSrvc = new DeletionService(exec); delSrvc.init(conf); + ContainerTokenSecretManager containerTokenSecretManager = new ContainerTokenSecretManager(); containerManager = new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater, metrics, containerTokenSecretManager, - new ApplicationACLsManager(conf)); + new ApplicationACLsManager(conf), dirsHandler); containerManager.init(conf); containerManager.start(); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Wed Nov 30 18:27:04 2011 @@ -25,6 +25,7 @@ import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.AbstractMap.SimpleEntry; @@ -649,7 +650,8 @@ public class TestContainer { public void containerFailed(int exitCode) { c.handle(new ContainerExitEvent(cId, - ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, exitCode)); + ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, exitCode, + "Container completed with exit code " + exitCode)); drainDispatcherEvents(); } @@ -659,9 +661,10 @@ public class TestContainer { } public void containerKilledOnRequest() { + int exitCode = ExitCode.FORCE_KILLED.getExitCode(); c.handle(new ContainerExitEvent(cId, - ContainerEventType.CONTAINER_KILLED_ON_REQUEST, ExitCode.FORCE_KILLED - .getExitCode())); + ContainerEventType.CONTAINER_KILLED_ON_REQUEST, exitCode, + "Container completed with exit code " + exitCode)); drainDispatcherEvents(); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java Wed Nov 30 18:27:04 2011 @@ -59,6 +59,8 @@ import org.apache.hadoop.yarn.event.Drai import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAction; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse; @@ -109,19 +111,23 @@ public class TestResourceLocalizationSer doNothing().when(spylfs).mkdir( isA(Path.class), isA(FsPermission.class), anyBoolean()); + List localDirs = new ArrayList(); + String[] sDirs = new String[4]; + for (int i = 0; i < 4; ++i) { + localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); + sDirs[i] = localDirs.get(i).toString(); + } + conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); + LocalDirsHandlerService diskhandler = new LocalDirsHandlerService(); + diskhandler.init(conf); + ResourceLocalizationService locService = - spy(new ResourceLocalizationService(dispatcher, exec, delService)); + spy(new ResourceLocalizationService(dispatcher, exec, delService, + diskhandler)); doReturn(lfs) .when(locService).getLocalFileContext(isA(Configuration.class)); try { dispatcher.start(); - List localDirs = new ArrayList(); - String[] sDirs = new String[4]; - for (int i = 0; i < 4; ++i) { - localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); - sDirs[i] = localDirs.get(i).toString(); - } - conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); // initialize ResourceLocalizationService locService.init(conf); @@ -176,12 +182,16 @@ public class TestResourceLocalizationSer dispatcher.register(LocalizerEventType.class, localizerBus); ContainerExecutor exec = mock(ContainerExecutor.class); + LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); + dirsHandler.init(conf); + DeletionService delService = new DeletionService(exec); delService.init(null); delService.start(); ResourceLocalizationService rawService = - new ResourceLocalizationService(dispatcher, exec, delService); + new ResourceLocalizationService(dispatcher, exec, delService, + dirsHandler); ResourceLocalizationService spyService = spy(rawService); doReturn(ignore).when(spyService).createServer(); doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker( @@ -356,13 +366,17 @@ public class TestResourceLocalizationSer dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); + LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); + dirsHandler.init(conf); + DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(null); delService.start(); ResourceLocalizationService rawService = - new ResourceLocalizationService(dispatcher, exec, delService); + new ResourceLocalizationService(dispatcher, exec, delService, + dirsHandler); ResourceLocalizationService spyService = spy(rawService); doReturn(ignore).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); @@ -414,8 +428,9 @@ public class TestResourceLocalizationSer String appStr = ConverterUtils.toString(appId); String ctnrStr = c.getContainerID().toString(); ArgumentCaptor tokenPathCaptor = ArgumentCaptor.forClass(Path.class); - verify(exec).startLocalizer(tokenPathCaptor.capture(), isA(InetSocketAddress.class), - eq("user0"), eq(appStr), eq(ctnrStr), isA(List.class)); + verify(exec).startLocalizer(tokenPathCaptor.capture(), + isA(InetSocketAddress.class), eq("user0"), eq(appStr), eq(ctnrStr), + isA(List.class), isA(List.class)); Path localizationTokenPath = tokenPathCaptor.getValue(); // heartbeat from localizer Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Wed Nov 30 18:27:04 2011 @@ -122,7 +122,8 @@ public class TestLogAggregationService e dispatcher.register(ApplicationEventType.class, appEventHandler); LogAggregationService logAggregationService = - new LogAggregationService(dispatcher, this.context, this.delSrvc); + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler); logAggregationService.init(this.conf); logAggregationService.start(); @@ -189,7 +190,8 @@ public class TestLogAggregationService e dispatcher.register(ApplicationEventType.class, appEventHandler); LogAggregationService logAggregationService = - new LogAggregationService(dispatcher, this.context, this.delSrvc); + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler); logAggregationService.init(this.conf); logAggregationService.start(); @@ -237,7 +239,8 @@ public class TestLogAggregationService e dispatcher.register(ApplicationEventType.class, appEventHandler); LogAggregationService logAggregationService = - new LogAggregationService(dispatcher, this.context, this.delSrvc); + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler); logAggregationService.init(this.conf); logAggregationService.start(); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java Wed Nov 30 18:27:04 2011 @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.event.Drai import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent; @@ -74,13 +75,16 @@ public class TestNonAggregatingLogHandle EventHandler appEventHandler = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, appEventHandler); + LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); + dirsHandler.init(conf); + ApplicationId appId1 = BuilderUtils.newApplicationId(1234, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1); ContainerId container11 = BuilderUtils.newContainerId(appAttemptId1, 1); NonAggregatingLogHandler logHandler = - new NonAggregatingLogHandler(dispatcher, delService); + new NonAggregatingLogHandler(dispatcher, delService, dirsHandler); logHandler.init(conf); logHandler.start(); @@ -146,13 +150,17 @@ public class TestNonAggregatingLogHandle EventHandler appEventHandler = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, appEventHandler); + LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); + dirsHandler.init(conf); + ApplicationId appId1 = BuilderUtils.newApplicationId(1234, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1); ContainerId container11 = BuilderUtils.newContainerId(appAttemptId1, 1); NonAggregatingLogHandler logHandler = - new NonAggregatingLogHandlerWithMockExecutor(dispatcher, delService); + new NonAggregatingLogHandlerWithMockExecutor(dispatcher, delService, + dirsHandler); logHandler.init(conf); logHandler.start(); @@ -182,8 +190,8 @@ public class TestNonAggregatingLogHandle private ScheduledThreadPoolExecutor mockSched; public NonAggregatingLogHandlerWithMockExecutor(Dispatcher dispatcher, - DeletionService delService) { - super(dispatcher, delService); + DeletionService delService, LocalDirsHandlerService dirsHandler) { + super(dispatcher, delService, dirsHandler); } @Override Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Wed Nov 30 18:27:04 2011 @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.Writer; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -37,6 +38,8 @@ import org.apache.hadoop.yarn.event.Disp import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.ResourceView; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; @@ -47,6 +50,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -54,10 +58,19 @@ public class TestNMWebServer { private static final File testRootDir = new File("target", TestNMWebServer.class.getSimpleName()); + private static File testLogDir = new File("target", + TestNMWebServer.class.getSimpleName() + "LogDir"); @Before public void setup() { testRootDir.mkdirs(); + testLogDir.mkdir(); + } + + @After + public void tearDown() { + FileUtil.fullyDelete(testRootDir); + FileUtil.fullyDelete(testLogDir); } @Test @@ -74,9 +87,14 @@ public class TestNMWebServer { } }; Configuration conf = new Configuration(); - WebServer server = new WebServer(nmContext, resourceView, - new ApplicationACLsManager(conf)); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); + conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); + NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); + healthChecker.init(conf); + LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); + + WebServer server = new WebServer(nmContext, resourceView, + new ApplicationACLsManager(conf), dirsHandler); server.init(conf); server.start(); @@ -119,20 +137,20 @@ public class TestNMWebServer { containerId.getApplicationAttemptId().getApplicationId(); nmContext.getApplications().get(applicationId).getContainers() .put(containerId, container); - writeContainerLogs(conf, nmContext, containerId); + writeContainerLogs(nmContext, containerId, dirsHandler); } // TODO: Pull logs and test contents. // Thread.sleep(1000000); } - private void writeContainerLogs(Configuration conf, Context nmContext, - ContainerId containerId) + private void writeContainerLogs(Context nmContext, + ContainerId containerId, LocalDirsHandlerService dirsHandler) throws IOException { // ContainerLogDir should be created File containerLogDir = - ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs(conf, - containerId).get(0); + ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs(containerId, + dirsHandler).get(0); containerLogDir.mkdirs(); for (String fileType : new String[] { "stdout", "stderr", "syslog" }) { Writer writer = new FileWriter(new File(containerLogDir, fileType)); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Wed Nov 30 18:27:04 2011 @@ -23,7 +23,6 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.NodeHealthCheckerService; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; @@ -41,6 +40,7 @@ import org.apache.hadoop.yarn.server.api import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl; @@ -51,7 +51,6 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.service.CompositeService; -import org.apache.hadoop.yarn.service.Service.STATE; public class MiniYARNCluster extends CompositeService { @@ -69,13 +68,23 @@ public class MiniYARNCluster extends Com private File testWorkDir; - public MiniYARNCluster(String testName) { - //default number of nodeManagers = 1 - this(testName, 1); - } + // Number of nm-local-dirs per nodemanager + private int numLocalDirs; + // Number of nm-log-dirs per nodemanager + private int numLogDirs; + + /** + * @param testName name of the test + * @param noOfNodeManagers the number of node managers in the cluster + * @param numLocalDirs the number of nm-local-dirs per nodemanager + * @param numLogDirs the number of nm-log-dirs per nodemanager + */ + public MiniYARNCluster(String testName, int noOfNodeManagers, + int numLocalDirs, int numLogDirs) { - public MiniYARNCluster(String testName, int noOfNodeManagers) { super(testName); + this.numLocalDirs = numLocalDirs; + this.numLogDirs = numLogDirs; this.testWorkDir = new File("target", testName); try { FileContext.getLocalFSFileContext().delete( @@ -166,25 +175,39 @@ public class MiniYARNCluster extends Com super.init(config); } + /** + * Create local/log directories + * @param dirType type of directories i.e. local dirs or log dirs + * @param numDirs number of directories + * @return the created directories as a comma delimited String + */ + private String prepareDirs(String dirType, int numDirs) { + File []dirs = new File[numDirs]; + String dirsString = ""; + for (int i = 0; i < numDirs; i++) { + dirs[i]= new File(testWorkDir, MiniYARNCluster.this.getName() + + "-" + dirType + "Dir-nm-" + index + "_" + i); + dirs[i].mkdir(); + LOG.info("Created " + dirType + "Dir in " + dirs[i].getAbsolutePath()); + String delimiter = (i > 0) ? "," : ""; + dirsString = dirsString.concat(delimiter + dirs[i].getAbsolutePath()); + } + return dirsString; + } + public synchronized void start() { try { - File localDir = new File(testWorkDir, MiniYARNCluster.this.getName() - + "-localDir-nm-" + index); - localDir.mkdir(); - LOG.info("Created localDir in " + localDir.getAbsolutePath()); - getConfig().set(YarnConfiguration.NM_LOCAL_DIRS, - localDir.getAbsolutePath()); - File logDir = - new File(testWorkDir, MiniYARNCluster.this.getName() - + "-logDir-nm-" + index); + // create nm-local-dirs and configure them for the nodemanager + String localDirsString = prepareDirs("local", numLocalDirs); + getConfig().set(YarnConfiguration.NM_LOCAL_DIRS, localDirsString); + // create nm-log-dirs and configure them for the nodemanager + String logDirsString = prepareDirs("log", numLogDirs); + getConfig().set(YarnConfiguration.NM_LOG_DIRS, logDirsString); + File remoteLogDir = new File(testWorkDir, MiniYARNCluster.this.getName() + "-remoteLogDir-nm-" + index); - logDir.mkdir(); remoteLogDir.mkdir(); - LOG.info("Created logDir in " + logDir.getAbsolutePath()); - getConfig().set(YarnConfiguration.NM_LOG_DIRS, - logDir.getAbsolutePath()); getConfig().set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath()); // By default AM + 2 containers Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Wed Nov 30 18:27:04 2011 @@ -117,7 +117,7 @@ public class TestContainerManagerSecurit conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L); UserGroupInformation.setConfiguration(conf); yarnCluster = new MiniYARNCluster(TestContainerManagerSecurity.class - .getName()); + .getName(), 1, 1, 1); yarnCluster.init(conf); yarnCluster.start(); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm Wed Nov 30 18:27:04 2011 @@ -398,6 +398,15 @@ Hadoop MapReduce Next Generation - Clust | | | Timeout for health script execution. | *-------------------------+-------------------------+------------------------+ + The health checker script is not supposed to give ERROR if only some of the + local disks become bad. NodeManager has the ability to periodically check + the health of the local disks (specifically checks nodemanager-local-dirs + and nodemanager-log-dirs) and after reaching the threshold of number of + bad directories based on the value set for the config property + yarn.nodemanager.disk-health-checker.min-healthy-disks. The boot disk is + either raided or a failure in the boot disk is identified by the health + checker script. + * {Slaves file} Typically you choose one machine in the cluster to act as the NameNode and @@ -874,13 +883,6 @@ KVNO Timestamp Principal *-------------------------+-------------------------+------------------------+ || Parameter || Value || Notes | *-------------------------+-------------------------+------------------------+ -| <<>> | | -| | Comma-separated list of NodeManager local directories. | | -| | | Paths to NodeManager local directories. Should be same as the value | -| | | which was provided to key in <<>>. This is | -| | | required to validate paths passed to the setuid executable in order | -| | to prevent arbitrary paths being passed to it. | -*-------------------------+-------------------------+------------------------+ | <<>> | | | | | | Unix group of the NodeManager. The group owner of the | | | | binary should be this group. Should be same as the | @@ -888,14 +890,6 @@ KVNO Timestamp Principal | | | required for validating the secure access of the | | | | binary. | *-------------------------+-------------------------+------------------------+ -| <<>> | | -| | Comma-separated list of NodeManager log directories. | | -| | | Paths to NodeManager log directories. Should be same as the value | -| | | which was provided to key in <<>>. This is | -| | | required to set proper permissions on the log files so that they can | -| | | be written to by the user's containers and read by the NodeManager for | -| | | . | -*-------------------------+-------------------------+------------------------+ | <<>> | hfds,yarn,mapred,bin | Banned users. | *-------------------------+-------------------------+------------------------+ | <<>> | 1000 | Prevent other super-users. | Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/c++/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1208622 /hadoop/core/branches/branch-0.19/mapred/src/c++:713112 /hadoop/core/trunk/src/c++:776175-784663 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1208622 /hadoop/core/branches/branch-0.19/mapred/src/contrib:713112 /hadoop/core/trunk/src/contrib:784664-785643 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/block_forensics/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1208622 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112 /hadoop/core/branches/branch-0.19/mapred/src/contrib/block_forensics:713112 /hadoop/core/trunk/src/contrib/block_forensics:784664-785643 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build-contrib.xml ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1208622 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112 /hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1208622 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112 /hadoop/core/trunk/src/contrib/build.xml:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/data_join/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1208622 /hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112 /hadoop/core/trunk/src/contrib/data_join:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1208622 /hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112 /hadoop/core/branches/branch-0.19/mapred/src/contrib/eclipse-plugin:713112 /hadoop/core/trunk/src/contrib/eclipse-plugin:776175-785643 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/index/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1208622 /hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112 /hadoop/core/trunk/src/contrib/index:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/vaidya/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1208622 /hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112 /hadoop/core/trunk/src/contrib/vaidya:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1208622 /hadoop/core/branches/branch-0.19/mapred/src/examples:713112 /hadoop/core/trunk/src/examples:776175-784663 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1208622 /hadoop/core/branches/branch-0.19/mapred/src/java:713112 /hadoop/core/trunk/src/mapred:776175-785643 Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java Wed Nov 30 18:27:04 2011 @@ -2685,7 +2685,7 @@ public class JobInProgress { MapAttemptFinishedEvent mfe = new MapAttemptFinishedEvent( statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(), status.getMapFinishTime(), - status.getFinishTime(), trackerHostname, "", + status.getFinishTime(), trackerHostname, -1, "", status.getStateString(), new org.apache.hadoop.mapreduce.Counters(status.getCounters()), tip.getSplits(statusAttemptID).burst() @@ -2698,7 +2698,7 @@ public class JobInProgress { statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(), status.getShuffleFinishTime(), status.getSortFinishTime(), status.getFinishTime(), - trackerHostname, "", status.getStateString(), + trackerHostname, -1, "", status.getStateString(), new org.apache.hadoop.mapreduce.Counters(status.getCounters()), tip.getSplits(statusAttemptID).burst() ); @@ -3208,7 +3208,7 @@ public class JobInProgress { (taskid, taskType, taskStatus.getRunState().toString(), finishTime, - taskTrackerHostName, diagInfo, + taskTrackerHostName, -1, diagInfo, splits.burst()); jobHistory.logEvent(tue, taskid.getJobID()); Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1208622 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112 /hadoop/core/trunk/src/test/mapred:776175-785643 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1208622 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:817878-835934 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1152502-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1152502-1208622 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/hdfs:713112 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/hdfs:776175-785643 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:817878-835934 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1161333-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1161333-1208622 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/FileBench.java:713112 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:776175-785643 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1161333-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1161333-1208622 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:713112 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:776175-785643 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1159757-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1159757-1208622 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs-with-mr/org/apache/hadoop/ipc:713112 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/ipc:713112 /hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/ipc:776175-784663 Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java Wed Nov 30 18:27:04 2011 @@ -102,6 +102,8 @@ public class TestCombineOutputCollector public void testCustomCollect() throws Throwable { //mock creation TaskReporter mockTaskReporter = mock(TaskReporter.class); + + @SuppressWarnings("unchecked") Writer mockWriter = mock(Writer.class); Configuration conf = new Configuration(); @@ -122,6 +124,8 @@ public class TestCombineOutputCollector public void testDefaultCollect() throws Throwable { //mock creation TaskReporter mockTaskReporter = mock(TaskReporter.class); + + @SuppressWarnings("unchecked") Writer mockWriter = mock(Writer.class); Configuration conf = new Configuration(); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java Wed Nov 30 18:27:04 2011 @@ -83,7 +83,7 @@ public class TestJobHistoryEvents extend for (TaskType t : types) { TaskAttemptUnsuccessfulCompletionEvent tauce = new TaskAttemptUnsuccessfulCompletionEvent - (id, t, state, 0L, "", "", NULL_SPLITS_ARRAY); + (id, t, state, 0L, "", -1, "", NULL_SPLITS_ARRAY); assertEquals(expected, tauce.getEventType()); } } Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1161333-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1161333-1208622 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:713112 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:776175-785643 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1161333-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1161333-1208622 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:713112 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:776175-785643 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:817878-835934 Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java Wed Nov 30 18:27:04 2011 @@ -938,12 +938,12 @@ public class TestRumenJobTraces { (TaskAttemptID.forName("attempt_200904211745_0003_m_000004_1"), TaskType.valueOf("MAP"), "STATUS", 1234567890L, "/194\\.6\\.134\\.80/cluster50262\\.secondleveldomain\\.com", - "MACHINE_EXPLODED", splits)); + -1, "MACHINE_EXPLODED", splits)); subject.process(new TaskAttemptUnsuccessfulCompletionEvent (TaskAttemptID.forName("attempt_200904211745_0003_m_000004_2"), TaskType.valueOf("MAP"), "STATUS", 1234567890L, "/194\\.6\\.134\\.80/cluster50263\\.secondleveldomain\\.com", - "MACHINE_EXPLODED", splits)); + -1, "MACHINE_EXPLODED", splits)); subject.process(new TaskStartedEvent(TaskID .forName("task_200904211745_0003_m_000004"), 1234567890L, TaskType .valueOf("MAP"), Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java Wed Nov 30 18:27:04 2011 @@ -73,7 +73,7 @@ public class MapAttempt20LineHistoryEven that.originalTaskType, status, Long.parseLong(finishTime), Long.parseLong(finishTime), - hostName, null, state, maybeParseCounters(counters), + hostName, -1, null, state, maybeParseCounters(counters), null); } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java Wed Nov 30 18:27:04 2011 @@ -77,7 +77,7 @@ public class ReduceAttempt20LineHistoryE Long.parseLong(shuffleFinish), Long.parseLong(sortFinish), Long.parseLong(finishTime), - hostName, null, + hostName, -1, null, state, maybeParseCounters(counters), null); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java?rev=1208644&r1=1208643&r2=1208644&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java Wed Nov 30 18:27:04 2011 @@ -141,7 +141,7 @@ public abstract class TaskAttempt20LineE return new TaskAttemptUnsuccessfulCompletionEvent (taskAttemptID, that.originalTaskType, status, Long.parseLong(finishTime), - hostName, error, null); + hostName, -1, error, null); } return null; Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/webapps/job/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 30 18:27:04 2011 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1208001 +/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1208622 /hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112 /hadoop/core/trunk/src/webapps/job:776175-785643