Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3E962994C for ; Tue, 6 Mar 2012 23:47:20 +0000 (UTC) Received: (qmail 73609 invoked by uid 500); 6 Mar 2012 23:47:20 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 73564 invoked by uid 500); 6 Mar 2012 23:47:20 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 73556 invoked by uid 99); 6 Mar 2012 23:47:20 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Mar 2012 23:47:20 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Mar 2012 23:47:18 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 7D1192388978; Tue, 6 Mar 2012 23:46:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1297798 - in /hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/ hadoop-y... Date: Tue, 06 Mar 2012 23:46:58 -0000 To: mapreduce-commits@hadoop.apache.org From: bobby@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120306234658.7D1192388978@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: bobby Date: Tue Mar 6 23:46:57 2012 New Revision: 1297798 URL: http://svn.apache.org/viewvc?rev=1297798&view=rev Log: svn merge -c 1297796 from trunk to branch-0.23.2 FIXES: MAPREDUCE-3977. LogAggregationService leaks log aggregator objects (Jason Lowe via bobby) Modified: hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Modified: hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt?rev=1297798&r1=1297797&r2=1297798&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt Tue Mar 6 23:46:57 2012 @@ -154,6 +154,9 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3961. Map/ReduceSlotMillis computation incorrect (Siddharth Seth via bobby) + MAPREDUCE-3977. LogAggregationService leaks log aggregator objects + (Jason Lowe via bobby) + Release 0.23.1 - 2012-02-17 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1297798&r1=1297797&r2=1297798&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original) +++ hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Tue Mar 6 23:46:57 2012 @@ -28,6 +28,7 @@ import java.util.concurrent.Executors; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -139,12 +140,6 @@ public class LogAggregationService exten super.stop(); } - - - - - - private void verifyAndCreateRemoteLogDir(Configuration conf) { // Checking the existance of the TLD FileSystem remoteFS = null; @@ -289,7 +284,7 @@ public class LogAggregationService exten createAppDir(user, appId, userUgi); // New application - AppLogAggregator appLogAggregator = + final AppLogAggregator appLogAggregator = new AppLogAggregatorImpl(this.dispatcher, this.deletionService, getConfig(), appId, userUgi, dirsHandler, getRemoteNodeLogFileForApp(appId, user), logRetentionPolicy, @@ -303,7 +298,22 @@ public class LogAggregationService exten // aggregation. // Schedule the aggregator. - this.threadPool.execute(appLogAggregator); + Runnable aggregatorWrapper = new Runnable() { + public void run() { + try { + appLogAggregator.run(); + } finally { + appLogAggregators.remove(appId); + } + } + }; + this.threadPool.execute(aggregatorWrapper); + } + + // for testing only + @Private + int getNumAggregators() { + return this.appLogAggregators.size(); } private void stopContainer(ContainerId containerId, int exitCode) { Modified: hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1297798&r1=1297797&r2=1297798&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original) +++ hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Tue Mar 6 23:46:57 2012 @@ -565,4 +565,38 @@ public class TestLogAggregationService e logAggregationService.stop(); } + + @Test + @SuppressWarnings("unchecked") + public void testLogAggregatorCleanup() throws Exception { + DeletionService delSrvc = mock(DeletionService.class); + + // get the AppLogAggregationImpl thread to crash + LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class); + + DrainDispatcher dispatcher = createDispatcher(); + EventHandler appEventHandler = mock(EventHandler.class); + dispatcher.register(ApplicationEventType.class, appEventHandler); + + LogAggregationService logAggregationService = + new LogAggregationService(dispatcher, this.context, delSrvc, + mockedDirSvc); + logAggregationService.init(this.conf); + logAggregationService.start(); + + ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1); + logAggregationService.handle(new LogHandlerAppStartedEvent( + application1, this.user, null, + ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls)); + + logAggregationService.handle(new LogHandlerAppFinishedEvent(application1)); + dispatcher.await(); + int timeToWait = 20 * 1000; + while (timeToWait > 0 && logAggregationService.getNumAggregators() > 0) { + Thread.sleep(100); + timeToWait -= 100; + } + Assert.assertEquals("Log aggregator failed to cleanup!", 0, + logAggregationService.getNumAggregators()); + } }