eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject incubator-eagle git commit: [EAGLE-494] add fields to mr history job entity
Date Wed, 24 Aug 2016 07:35:30 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/develop 97c5f1ec9 -> b52405f12


[EAGLE-494] add fields to mr history job entity

Author: wujinhu <wujinhu920@126.com>

Closes #377 from wujinhu/EAGLE-494.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/b52405f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/b52405f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/b52405f1

Branch: refs/heads/develop
Commit: b52405f128dee0354c1791bd4b7995a730c9050c
Parents: 97c5f1e
Author: wujinhu <wujinhu920@126.com>
Authored: Wed Aug 24 15:35:17 2016 +0800
Committer: Hao Chen <hao@apache.org>
Committed: Wed Aug 24 15:35:17 2016 +0800

----------------------------------------------------------------------
 .../mr/historyentity/JobExecutionAPIEntity.java | 44 ++++++++++++++++++++
 .../mr/history/parser/JHFEventReaderBase.java   | 26 ++++++++++++
 2 files changed, 70 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b52405f1/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
index db8f0d6..d9093ff 100644
--- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
+++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
@@ -73,6 +73,14 @@ public class JobExecutionAPIEntity extends JobBaseAPIEntity {
     private long lastReduceDuration;
     @Column("s")
     private long durationTime;
+    @Column("u")
+    private double avgMapTaskDuration;
+    @Column("v")
+    private double avgReduceTaskDuration;
+    @Column("w")
+    private double maxMapTaskDuration;
+    @Column("x")
+    private double maxReduceTaskDuration;
 
     public long getDurationTime() {
         return durationTime;
@@ -227,4 +235,40 @@ public class JobExecutionAPIEntity extends JobBaseAPIEntity {
         this.lastReduceDuration = lastReduceDuration;
         valueChanged("lastReduceDuration");
     }
+
+    public double getAvgMapTaskDuration() {
+        return avgMapTaskDuration;
+    }
+
+    public void setAvgMapTaskDuration(double avgMapTaskDuration) {
+        this.avgMapTaskDuration = avgMapTaskDuration;
+        valueChanged("avgMapTaskDuration");
+    }
+
+    public double getAvgReduceTaskDuration() {
+        return avgReduceTaskDuration;
+    }
+
+    public void setAvgReduceTaskDuration(double avgReduceTaskDuration) {
+        this.avgReduceTaskDuration = avgReduceTaskDuration;
+        valueChanged("avgReduceTaskDuration");
+    }
+
+    public double getMaxMapTaskDuration() {
+        return maxMapTaskDuration;
+    }
+
+    public void setMaxMapTaskDuration(double maxMapTaskDuration) {
+        this.maxMapTaskDuration = maxMapTaskDuration;
+        valueChanged("maxMapTaskDuration");
+    }
+
+    public double getMaxReduceTaskDuration() {
+        return maxReduceTaskDuration;
+    }
+
+    public void setMaxReduceTaskDuration(double maxReduceTaskDuration) {
+        this.maxReduceTaskDuration = maxReduceTaskDuration;
+        valueChanged("maxReduceTaskDuration");
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b52405f1/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
index af3df85..9992690 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
@@ -71,6 +71,9 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher
impl
 
     protected final Configuration configuration;
 
+    private long sumMapTaskDuration;
+    private long sumReduceTaskDuration;
+
     public Constants.JobType fetchJobType(Configuration config) {
         if (config.get(Constants.JobConfiguration.CASCADING_JOB) != null) { return Constants.JobType.CASCADING;
}
         if (config.get(Constants.JobConfiguration.HIVE_JOB) != null) { return Constants.JobType.HIVE;
}
@@ -113,6 +116,8 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher
impl
         if (this.configuration != null && this.m_jobType == null) {
             this.setJobType(fetchJobType(this.configuration).toString());
         }
+        this.sumMapTaskDuration = 0l;
+        this.sumReduceTaskDuration = 0l;
     }
 
     public void register(HistoryJobEntityLifecycleListener lifecycleListener){
@@ -261,6 +266,13 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher
impl
                    m_jobExecutionEntity.setRackLocalMapsPercentage(m_jobExecutionEntity.getRackLocalMaps()
* 1.0 / m_jobExecutionEntity.getTotalLaunchedMaps());
                }
            }
+           m_jobExecutionEntity.setAvgMapTaskDuration(this.sumMapTaskDuration * 1.0 / m_numTotalMaps);
+           if (m_numTotalReduces == 0) {
+               m_jobExecutionEntity.setMaxReduceTaskDuration(0);
+               m_jobExecutionEntity.setAvgReduceTaskDuration(0);
+           } else {
+               m_jobExecutionEntity.setAvgReduceTaskDuration(this.sumReduceTaskDuration *
1.0 / m_numTotalReduces);
+           }
            entityCreated(m_jobExecutionEntity);
        }
     }
@@ -333,6 +345,20 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher
impl
             if (taskType.equals(Constants.TaskType.REDUCE.toString()) && duration
> m_jobExecutionEntity.getLastReduceDuration()) {
                 m_jobExecutionEntity.setLastReduceDuration(duration);
             }
+
+            if (taskType.equals(Constants.TaskType.MAP.toString()) && entity.getDuration()
> m_jobExecutionEntity.getMaxMapTaskDuration()) {
+                m_jobExecutionEntity.setMaxMapTaskDuration(entity.getDuration());
+            }
+            if (taskType.equals(Constants.TaskType.REDUCE.toString()) && entity.getDuration()
> m_jobExecutionEntity.getMaxReduceTaskDuration()) {
+                m_jobExecutionEntity.setMaxReduceTaskDuration(entity.getDuration());
+            }
+
+            if (taskType.equals(Constants.TaskType.MAP.toString())) {
+                this.sumMapTaskDuration += entity.getDuration();
+            } else {
+                this.sumReduceTaskDuration += entity.getDuration();
+            }
+
             entityCreated(entity);
             //_taskStartTime.remove(taskID); // clean this taskID
         } else if ((recType == RecordTypes.MapAttempt || recType == RecordTypes.ReduceAttempt)
&& startTime != null) { // task attempt start


Mime
View raw message