eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jinh...@apache.org
Subject incubator-eagle git commit: [EAGLE-679] Fix missing CoprocessorService and MR Running feeder task number limi…
Date Tue, 25 Oct 2016 04:07:59 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 6b87fb3a7 -> 15f39705d


[EAGLE-679] Fix missing CoprocessorService and MR Running feeder task number limi…

Author: wujinhu <wujinhu920@126.com>

Closes #558 from wujinhu/EAGLE-679.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/15f39705
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/15f39705
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/15f39705

Branch: refs/heads/master
Commit: 15f39705d0e4c86803d1c552810a3adf2f9b50d6
Parents: 6b87fb3
Author: wujinhu <wujinhu920@126.com>
Authored: Tue Oct 25 12:07:48 2016 +0800
Committer: wujinhu <wujinhu920@126.com>
Committed: Tue Oct 25 12:07:48 2016 +0800

----------------------------------------------------------------------
 .../apache/eagle/jpm/mr/running/parser/MRJobParser.java  | 11 +++++++++++
 eagle-topology-assembly/pom.xml                          |  4 ----
 2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/15f39705/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
index 0abf9a0..945e8ce 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
@@ -78,6 +78,7 @@ public class MRJobParser implements Runnable {
     private List<String> configKeys;
     private static final int TOP_BOTTOM_TASKS_BY_ELAPSED_TIME = 10;
     private static final int FLUSH_TASKS_EVERY_TIME = 5;
+    private static final int MAX_TASKS_PERMIT = 5000;
 
     static {
         OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
@@ -428,6 +429,16 @@ public class MRJobParser implements Runnable {
     }
 
     private Function<String, Boolean> fetchTasks = jobId -> {
+        try {
+            JobExecutionAPIEntity entity = this.mrJobEntityMap.get(jobId);
+            int taskNumber = entity.getNumTotalMaps() + entity.getNumTotalReduces();
+            if (taskNumber > MAX_TASKS_PERMIT) {
+                LOG.info("too many tasks {}, ignore tasks", taskNumber);
+                return true;
+            }
+        } catch (Exception e) {
+            return true;
+        }
         String taskURL = app.getTrackingUrl() + Constants.MR_JOBS_URL + "/" + jobId + "/"
+ Constants.MR_TASKS_URL + "?" + Constants.ANONYMOUS_PARAMETER;
         InputStream is = null;
         List<MRTask> tasks = null;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/15f39705/eagle-topology-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/pom.xml b/eagle-topology-assembly/pom.xml
index a9dddb0..f5aee5f 100644
--- a/eagle-topology-assembly/pom.xml
+++ b/eagle-topology-assembly/pom.xml
@@ -42,10 +42,6 @@
             <version>${project.version}</version>
             <exclusions>
                 <exclusion>
-                    <groupId>org.apache.hbase</groupId>
-                    <artifactId>hbase-server</artifactId>
-                </exclusion>
-                <exclusion>
                     <groupId>javax.servlet</groupId>
                     <artifactId>servlet-api</artifactId>
                 </exclusion>


Mime
View raw message