hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjs...@apache.org
Subject git commit: YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong.
Date Mon, 13 Oct 2014 17:56:46 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 4fb157655 -> 6da637ec1


YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong.

(cherry picked from commit 4aed2d8e91c7dccc78fbaffc409d3076c3316289)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6da637ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6da637ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6da637ec

Branch: refs/heads/branch-2.6
Commit: 6da637ec1282fcb9b54bbd95ace842c0e00ecbb6
Parents: 4fb1576
Author: Zhijie Shen <zjshen@apache.org>
Authored: Mon Oct 13 10:53:37 2014 -0700
Committer: Zhijie Shen <zjshen@apache.org>
Committed: Mon Oct 13 10:56:22 2014 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +++
 .../yarn/api/records/LogAggregationContext.java | 28 +-------------------
 .../hadoop/yarn/conf/YarnConfiguration.java     |  7 +++++
 .../src/main/proto/yarn_protos.proto            |  1 -
 .../impl/pb/LogAggregationContextPBImpl.java    | 15 -----------
 .../src/main/resources/yarn-default.xml         | 10 +++++++
 .../logaggregation/AppLogAggregatorImpl.java    |  8 +++---
 .../TestContainerManagerRecovery.java           |  5 +---
 .../TestLogAggregationService.java              |  4 ++-
 .../capacity/TestContainerAllocation.java       |  4 +--
 10 files changed, 31 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 13a09c0..2ebae00 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -567,6 +567,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2667. Fix the release audit warning caused by hadoop-yarn-registry
     (Yi Liu via jlowe)
 
+    YARN-2651. Spun off LogRollingInterval from LogAggregationContext. (Xuan Gong
+    via zjshen)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
index 9a0a157..46c1809 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
@@ -37,13 +37,6 @@ import org.apache.hadoop.yarn.util.Records;
  *     which match the defined exclude pattern and those log files
  *     will not be uploaded. If the log file name matches both the
  *     include and the exclude pattern, this file will be excluded eventually</li>
- *     <li>rollingIntervalSeconds. The default value is -1. By default,
- *     the logAggregationService only uploads container logs when
- *     the application is finished. This configure defines
- *     how often the logAggregationSerivce uploads container logs in seconds.
- *     By setting this configure, the logAggregationSerivce can upload container
- *     logs periodically when the application is running.
- *     </li>
  *   </ul>
  * </p>
  *
@@ -57,11 +50,10 @@ public abstract class LogAggregationContext {
   @Public
   @Unstable
   public static LogAggregationContext newInstance(String includePattern,
-      String excludePattern, long rollingIntervalSeconds) {
+      String excludePattern) {
     LogAggregationContext context = Records.newRecord(LogAggregationContext.class);
     context.setIncludePattern(includePattern);
     context.setExcludePattern(excludePattern);
-    context.setRollingIntervalSeconds(rollingIntervalSeconds);
     return context;
   }
 
@@ -100,22 +92,4 @@ public abstract class LogAggregationContext {
   @Public
   @Unstable
   public abstract void setExcludePattern(String excludePattern);
-
-  /**
-   * Get rollingIntervalSeconds
-   *
-   * @return the rollingIntervalSeconds
-   */
-  @Public
-  @Unstable
-  public abstract long getRollingIntervalSeconds();
-
-  /**
-   * Set rollingIntervalSeconds
-   *
-   * @param rollingIntervalSeconds
-   */
-  @Public
-  @Unstable
-  public abstract void setRollingIntervalSeconds(long rollingIntervalSeconds);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b1b0dc0..d60cbc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -712,6 +712,13 @@ public class YarnConfiguration extends Configuration {
   public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60;
 
   /**
+   * Define how often NMs wake up and upload log files
+   */
+  public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS =
+      NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds";
+  public static final long
+      DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1;
+  /**
    * Number of threads used in log cleanup. Only applicable if Log aggregation
    * is disabled
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 8db451d..136192d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -302,7 +302,6 @@ message ApplicationSubmissionContextProto {
 message LogAggregationContextProto {
  optional string include_pattern = 1 [default = ".*"];
  optional string exclude_pattern = 2 [default = ""];
- optional int64 rolling_interval_seconds = 3 [default = -1];
 }
 
 enum ApplicationAccessTypeProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java
index 4406ef9..dc7a21d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java
@@ -116,19 +116,4 @@ public class LogAggregationContextPBImpl extends LogAggregationContext{
     }
     builder.setExcludePattern(excludePattern);
   }
-
-  @Override
-  public long getRollingIntervalSeconds() {
-    LogAggregationContextProtoOrBuilder p = viaProto ? proto : builder;
-    if (! p.hasRollingIntervalSeconds()) {
-      return -1;
-    }
-    return p.getRollingIntervalSeconds();
-  }
-
-  @Override
-  public void setRollingIntervalSeconds(long rollingIntervalSeconds) {
-    maybeInitBuilder();
-    builder.setRollingIntervalSeconds(rollingIntervalSeconds);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 75efc27..5cb1988 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1468,4 +1468,14 @@
     <value>Client</value>
   </property>
 
+  <property>
+    <description>Defines how often NMs wake up to upload log files.
+    The default value is -1. By default, the logs will be uploaded when
+    the application is finished. By setting this configure, logs can be uploaded
+    periodically when the application is running. The minimum rolling-interval-seconds
+    can be set is 3600.
+    </description>
+    <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
+    <value>-1</value>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 63f7c66..98e3caf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -148,9 +148,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
     } else {
       this.retentionSize = configuredRentionSize;
     }
-    long configuredRollingMonitorInterval =
-        this.logAggregationContext == null ? -1 : this.logAggregationContext
-          .getRollingIntervalSeconds();
+    long configuredRollingMonitorInterval = conf.getLong(
+      YarnConfiguration
+        .NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
+      YarnConfiguration
+        .DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS);
     boolean debug_mode =
         conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED,
           DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 2c69843..7850a1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -130,8 +130,7 @@ public class TestContainerManagerRecovery {
         containerTokens, acls);
     // create the logAggregationContext
     LogAggregationContext logAggregationContext =
-        LogAggregationContext.newInstance("includePattern", "excludePattern",
-          1000);
+        LogAggregationContext.newInstance("includePattern", "excludePattern");
     StartContainersResponse startResponse = startContainer(context, cm, cid,
         clc, logAggregationContext);
     assertTrue(startResponse.getFailedRequests().isEmpty());
@@ -168,8 +167,6 @@ public class TestContainerManagerRecovery {
     LogAggregationContext recovered =
         ((ApplicationImpl) app).getLogAggregationContext();
     assertNotNull(recovered);
-    assertEquals(logAggregationContext.getRollingIntervalSeconds(),
-      recovered.getRollingIntervalSeconds());
     assertEquals(logAggregationContext.getIncludePattern(),
       recovered.getIncludePattern());
     assertEquals(logAggregationContext.getExcludePattern(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index 2c0f349..ab86a18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -1235,10 +1235,12 @@ public class TestLogAggregationService extends BaseContainerManagerTest
{
       throws Exception {
     LogAggregationContext logAggregationContextWithInterval =
         Records.newRecord(LogAggregationContext.class);
-    logAggregationContextWithInterval.setRollingIntervalSeconds(5000);
     this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
     this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
       this.remoteRootLogDir.getAbsolutePath());
+    this.conf.setLong(
+      YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
+      3600);
     if (retentionSizeLimitation) {
       // set the retention size as 1. The number of logs for one application
       // in one NM should be 1.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 85ef381..365c754 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -212,16 +212,14 @@ public class TestContainerAllocation {
       .assertNull(getLogAggregationContextFromContainerToken(rm1, nm1, null));
 
     // create a not-null LogAggregationContext
-    final int interval = 2000;
     LogAggregationContext logAggregationContext =
         LogAggregationContext.newInstance(
-          "includePattern", "excludePattern", interval);
+          "includePattern", "excludePattern");
     LogAggregationContext returned =
         getLogAggregationContextFromContainerToken(rm1, nm2,
           logAggregationContext);
     Assert.assertEquals("includePattern", returned.getIncludePattern());
     Assert.assertEquals("excludePattern", returned.getExcludePattern());
-    Assert.assertEquals(interval, returned.getRollingIntervalSeconds());
     rm1.stop();
   }
 


Mime
View raw message