hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject hadoop git commit: YARN-4773. Log aggregation performs extraneous filesystem operations when rolling log aggregation is disabled. Contributed by Jun Gong (cherry picked from commit 948b75807068c304ffe789e32f2b850c0d653e0a)
Date Mon, 28 Mar 2016 23:03:42 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 cb7953377 -> 35f9cfda6


YARN-4773. Log aggregation performs extraneous filesystem operations when rolling log aggregation
is disabled. Contributed by Jun Gong
(cherry picked from commit 948b75807068c304ffe789e32f2b850c0d653e0a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35f9cfda
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35f9cfda
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35f9cfda

Branch: refs/heads/branch-2.8
Commit: 35f9cfda61e989de437be4b8d86ae675ee7bdec2
Parents: cb79533
Author: Jason Lowe <jlowe@apache.org>
Authored: Mon Mar 28 23:00:56 2016 +0000
Committer: Jason Lowe <jlowe@apache.org>
Committed: Mon Mar 28 23:03:16 2016 +0000

----------------------------------------------------------------------
 .../logaggregation/AppLogAggregatorImpl.java          | 14 ++++++++++----
 .../logaggregation/TestLogAggregationService.java     | 12 +++++++-----
 2 files changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f9cfda/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index da7fc14..fed4a3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -124,11 +124,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
   private final long rollingMonitorInterval;
   private final boolean logAggregationInRolling;
   private final NodeId nodeId;
-  // This variable is only for testing
-  private final AtomicBoolean waiting = new AtomicBoolean(false);
 
-  // This variable is only for testing
+  // These variables are only for testing
+  private final AtomicBoolean waiting = new AtomicBoolean(false);
   private int logAggregationTimes = 0;
+  private int cleanupOldLogTimes = 0;
 
   private boolean renameTemporaryLogFileFailed = false;
 
@@ -365,8 +365,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
 
       // Before upload logs, make sure the number of existing logs
       // is smaller than the configured NM log aggregation retention size.
-      if (uploadedLogsInThisCycle) {
+      if (uploadedLogsInThisCycle && logAggregationInRolling) {
         cleanOldLogs();
+        cleanupOldLogTimes++;
       }
 
       if (writer != null) {
@@ -689,4 +690,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
   public int getLogAggregationTimes() {
     return this.logAggregationTimes;
   }
+
+  @VisibleForTesting
+  int getCleanupOldLogTimes() {
+    return this.cleanupOldLogTimes;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f9cfda/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index 6239ded..bdb4d1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -2133,7 +2133,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest
{
         Records.newRecord(LogAggregationContext.class);
     logAggregationContext.setLogAggregationPolicyClassName(
         FailedOrKilledContainerLogAggregationPolicy.class.getName());
-    verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2);
+    verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2, 0);
   }
 
   @Test (timeout = 20000)
@@ -2147,13 +2147,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest
{
         AMOnlyLogAggregationPolicy.class.getName());
     contextWithAMOnly.setRolledLogsIncludePattern("sys*");
     contextWithAMOnly.setRolledLogsExcludePattern("std_final");
-    verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4);
+    verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4, 1);
   }
 
   private void verifySkipUnnecessaryNNOperations(
       LogAggregationContext logAggregationContext,
-      int expectedLogAggregationTimes, int expectedAggregationReportNum)
-      throws Exception {
+      int expectedLogAggregationTimes, int expectedAggregationReportNum,
+      int expectedCleanupOldLogsTimes) throws Exception {
     LogAggregationService logAggregationService = new LogAggregationService(
         dispatcher, this.context, this.delSrvc, super.dirsHandler);
     logAggregationService.init(this.conf);
@@ -2164,7 +2164,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest
{
         null, this.acls, logAggregationContext));
 
     // Container finishes
-    String[] logFiles = new String[] { "stdout" };
+    String[] logFiles = new String[] { "sysout" };
     finishContainer(appId, logAggregationService,
         ContainerType.APPLICATION_MASTER, 1, 0, logFiles);
     AppLogAggregatorImpl aggregator =
@@ -2184,6 +2184,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest
{
         aggregator.getLogAggregationTimes());
     assertEquals(expectedAggregationReportNum,
         this.context.getLogAggregationStatusForApps().size());
+    assertEquals(expectedCleanupOldLogsTimes,
+        aggregator.getCleanupOldLogTimes());
   }
 
   private int numOfLogsAvailable(LogAggregationService logAggregationService,


Mime
View raw message