hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sj...@apache.org
Subject [41/50] [abbrv] hadoop git commit: MAPREDUCE-6720. Inconsistent values of counters across tasks and job reported to timeline service. Contributed by Varun Saxena
Date Sun, 10 Jul 2016 15:51:32 GMT
MAPREDUCE-6720. Inconsistent values of counters across tasks and job reported to timeline service.
Contributed by Varun Saxena


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bf87eed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bf87eed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bf87eed

Branch: refs/heads/YARN-2928
Commit: 8bf87eede2b1735993abc0f2cc49c971f9f8e222
Parents: 6d94303
Author: Naganarasimha <naganarasimha_gr@apache.org>
Authored: Wed Jun 22 23:32:19 2016 +0530
Committer: Sangjin Lee <sjlee@apache.org>
Committed: Sun Jul 10 08:46:04 2016 -0700

----------------------------------------------------------------------
 .../mapreduce/jobhistory/JobHistoryEventHandler.java    |  4 ++++
 .../hadoop/mapreduce/jobhistory/JobFinishedEvent.java   | 12 ++++++------
 .../hadoop/mapreduce/util/JobHistoryEventUtils.java     |  7 ++++++-
 3 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf87eed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index a09c17b..9b59676 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1125,6 +1125,10 @@ public class JobHistoryEventHandler extends AbstractService
     org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
         createBaseEntity(event, timestamp, entityType, setCreatedTime);
     entity.setId(taskId);
+    if (event.getEventType() == EventType.TASK_STARTED) {
+      entity.addInfo("TASK_TYPE",
+          ((TaskStartedEvent)event).getTaskType().toString());
+    }
     entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString());
     return entity;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf87eed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
index 76d6897..ea21f60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
@@ -158,12 +158,12 @@ public class JobFinishedEvent  implements HistoryEvent {
 
   @Override
   public Set<TimelineMetric> getTimelineMetrics() {
-    Set<TimelineMetric> jobMetrics = JobHistoryEventUtils
-        .countersToTimelineMetric(getMapCounters(), finishTime);
-    jobMetrics.addAll(JobHistoryEventUtils
-        .countersToTimelineMetric(getReduceCounters(), finishTime));
-    jobMetrics.addAll(JobHistoryEventUtils
-        .countersToTimelineMetric(getTotalCounters(), finishTime));
+    Set<TimelineMetric> jobMetrics = JobHistoryEventUtils.
+        countersToTimelineMetric(getTotalCounters(), finishTime);
+    jobMetrics.addAll(JobHistoryEventUtils.
+        countersToTimelineMetric(getMapCounters(), finishTime, "MAP:"));
+    jobMetrics.addAll(JobHistoryEventUtils.
+        countersToTimelineMetric(getReduceCounters(), finishTime, "REDUCE:"));
     return jobMetrics;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf87eed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
index 35d066c..89abf0e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
@@ -61,11 +61,16 @@ public final class JobHistoryEventUtils {
 
   public static Set<TimelineMetric> countersToTimelineMetric(Counters counters,
       long timestamp) {
+    return countersToTimelineMetric(counters, timestamp, "");
+  }
+
+  public static Set<TimelineMetric> countersToTimelineMetric(Counters counters,
+      long timestamp, String groupNamePrefix) {
     Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
     for (CounterGroup g : counters) {
       String groupName = g.getName();
       for (Counter c : g) {
-        String name = groupName + ":" + c.getName();
+        String name = groupNamePrefix + groupName + ":" + c.getName();
         TimelineMetric metric = new TimelineMetric();
         metric.setId(name);
         metric.addValue(timestamp, c.getValue());


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message