eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jinh...@apache.org
Subject incubator-eagle git commit: [EAGLE-644] optimize some default configurations for JPM
Date Thu, 20 Oct 2016 03:10:27 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 44259aeca -> c3bdf46ca


[EAGLE-644] optimize some default configurations for JPM

Author: wujinhu <wujinhu920@126.com>

Closes #535 from wujinhu/EAGLE-642.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/c3bdf46c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/c3bdf46c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/c3bdf46c

Branch: refs/heads/master
Commit: c3bdf46ca290cd9b36a09d79943d680af2ece5f4
Parents: 44259ae
Author: wujinhu <wujinhu920@126.com>
Authored: Thu Oct 20 11:10:19 2016 +0800
Committer: wujinhu <wujinhu920@126.com>
Committed: Thu Oct 20 11:10:19 2016 +0800

----------------------------------------------------------------------
 ...he.eagle.jpm.aggregation.AggregationApplicationProvider.xml | 2 +-
 ...he.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml | 2 +-
 .../org/apache/eagle/jpm/mr/running/parser/MRJobParser.java    | 6 ++----
 ...he.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml | 6 +++---
 4 files changed, 7 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c3bdf46c/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
index 2b328cc..aa35d11 100644
--- a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
@@ -25,7 +25,7 @@
             <name>workers</name>
             <displayName>Worker Number</displayName>
             <description>the number of storm workers will be used</description>
-            <value>4</value>
+            <value>2</value>
         </property>
         <property>
             <name>stormConfig.mrHistoryAggregationSpoutTasks</name>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c3bdf46c/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
index 77a3709..ee09d50 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
@@ -26,7 +26,7 @@
             <name>workers</name>
             <displayName>Worker Number</displayName>
             <description>the number of storm workers will be used</description>
-            <value>4</value>
+            <value>2</value>
         </property>
         <property>
             <name>stormConfig.mrHistoryJobSpoutTasks</name>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c3bdf46c/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
index 797bf21..edcb9f7 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
@@ -74,11 +74,11 @@ public class MRJobParser implements Runnable {
     private MRRunningJobManager runningJobManager;
     private ParserStatus parserStatus;
     private ResourceFetcher rmResourceFetcher;
-    private boolean first;
     private Set<String> finishedTaskIds;
     private List<String> configKeys;
     private MRRunningJobConfig.EndpointConfig endpointConfig;
     private static final int TOP_BOTTOM_TASKS_BY_ELASPED_TIME = 10;
+    private static final int FLUSH_TASKS_EVERY_TIME = 5;
 
     static {
         OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
@@ -106,7 +106,6 @@ public class MRJobParser implements Runnable {
         this.runningJobManager = runningJobManager;
         this.parserStatus  = ParserStatus.FINISHED;
         this.rmResourceFetcher = rmResourceFetcher;
-        this.first = true;
         this.finishedTaskIds = new HashSet<>();
         this.configKeys = configKeys;
     }
@@ -150,11 +149,10 @@ public class MRJobParser implements Runnable {
         List<Function<String, Boolean>> functions = new ArrayList<>();
         functions.add(fetchJobConfig);
         functions.add(fetchJobCounters);
-        if (!this.first) { //do not fetch these info below for the first time
+        if ((int)(Math.random() * 10) % FLUSH_TASKS_EVERY_TIME == 0) {
             functions.add(fetchTasks);
         }
 
-        this.first = false;
         for (String jobId : mrJobEntityMap.keySet()) {
             for (Function<String, Boolean> function : functions) {
                 int i = 0;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c3bdf46c/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
index 4063b3a..d57629a 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
@@ -25,7 +25,7 @@
             <name>workers</name>
             <displayName>Worker Number</displayName>
             <description>the number of storm workers will be used</description>
-            <value>4</value>
+            <value>2</value>
         </property>
         <property>
             <name>stormConfig.mrRunningJobFetchSpoutTasks</name>
@@ -37,7 +37,7 @@
             <name>stormConfig.mrRunningJobParseBoltTasks</name>
             <displayName>Parse Task Number</displayName>
             <description>number of tasks to parse map reduce running jobs got from
resource manager</description>
-            <value>5</value>
+            <value>6</value>
         </property>
 
         <property>
@@ -58,7 +58,7 @@
             <name>endpointConfig.parseJobThreadPoolSize</name>
             <displayName>Parse Job ThreadPool Size in Each Parse Task</displayName>
             <description>parse job thread pool size in each parse task</description>
-            <value>5</value>
+            <value>2</value>
         </property>
         <property>
             <name>endpointConfig.rmUrls</name>


Mime
View raw message