hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [06/50] [abbrv] hadoop git commit: YARN-6931. Make the aggregation interval in AppLevelTimelineCollector configurable. (Abhishek Modi via Haibo Chen)
Date Thu, 14 Jun 2018 16:50:02 GMT
YARN-6931. Make the aggregation interval in AppLevelTimelineCollector configurable. (Abhishek
Modi via Haibo Chen)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24a89825
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24a89825
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24a89825

Branch: refs/heads/HDDS-48
Commit: 24a89825f0cbc92b0a462152bc765e2195edd5a1
Parents: e9ea902
Author: Haibo Chen <haibochen@apache.org>
Authored: Tue Jun 12 10:03:07 2018 -0700
Committer: Haibo Chen <haibochen@apache.org>
Committed: Tue Jun 12 10:03:34 2018 -0700

----------------------------------------------------------------------
 .../apache/hadoop/yarn/conf/YarnConfiguration.java    |  9 +++++++++
 .../src/main/resources/yarn-default.xml               |  9 +++++++++
 .../collector/AppLevelTimelineCollectorWithAgg.java   | 14 +++++++++-----
 3 files changed, 27 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/24a89825/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f7f82f8..5292a25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2665,6 +2665,15 @@ public class YarnConfiguration extends Configuration {
       TIMELINE_SERVICE_PREFIX + "read.authentication.enabled";
 
   /**
+   * The name for setting that controls how often in-memory app level
+   * aggregation is kicked off in timeline collector.
+   */
+  public static final String TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS =
+      TIMELINE_SERVICE_PREFIX + "app-aggregation-interval-secs";
+
+  public static final int
+      DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS = 15;
+  /**
    * The default setting for authentication checks for reading timeline
    * service v2 data.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24a89825/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b0ffc48..2cc842f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2547,6 +2547,15 @@
 
   <property>
     <description>
+      The setting that controls how often in-memory app level
+      aggregation is kicked off in timeline collector.
+    </description>
+    <name>yarn.timeline-service.app-aggregation-interval-secs</name>
+    <value>15</value>
+  </property>
+
+  <property>
+    <description>
     The default hdfs location for flowrun coprocessor jar.
     </description>
     <name>yarn.timeline-service.hbase.coprocessor.jar.hdfs.location

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24a89825/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java
index d7f47c8..aa041a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -50,7 +51,7 @@ public class AppLevelTimelineCollectorWithAgg
       LoggerFactory.getLogger(TimelineCollector.class);
 
   private final static int AGGREGATION_EXECUTOR_NUM_THREADS = 1;
-  private final static int AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS = 15;
+  private int aggregationExecutorIntervalSecs;
   private static Set<String> entityTypesSkipAggregation
       = initializeSkipSet();
 
@@ -71,6 +72,11 @@ public class AppLevelTimelineCollectorWithAgg
 
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
+    aggregationExecutorIntervalSecs = conf.getInt(
+        YarnConfiguration.TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS,
+        YarnConfiguration.
+            DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS
+    );
     super.serviceInit(conf);
   }
 
@@ -84,10 +90,8 @@ public class AppLevelTimelineCollectorWithAgg
             .build());
     appAggregator = new AppLevelAggregator();
     appAggregationExecutor.scheduleAtFixedRate(appAggregator,
-        AppLevelTimelineCollectorWithAgg.
-          AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS,
-        AppLevelTimelineCollectorWithAgg.
-          AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS,
+        aggregationExecutorIntervalSecs,
+        aggregationExecutorIntervalSecs,
         TimeUnit.SECONDS);
     super.serviceStart();
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message