hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [42/50] [abbrv] hadoop git commit: MAPREDUCE-7072. mapred job -history prints duplicate counter in human output (wilfreds via rkanter)
Date Wed, 02 May 2018 19:22:17 GMT
MAPREDUCE-7072. mapred job -history prints duplicate counter in human output (wilfreds via
rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b47275fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b47275fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b47275fe

Branch: refs/heads/YARN-8200
Commit: b47275fe28f51d14adf777822b45c0812e71a2a6
Parents: 600f4d4
Author: Robert Kanter <rkanter@apache.org>
Authored: Fri Apr 27 09:57:31 2018 -0700
Committer: Robert Kanter <rkanter@apache.org>
Committed: Fri Apr 27 09:57:31 2018 -0700

----------------------------------------------------------------------
 .../HumanReadableHistoryViewerPrinter.java      |  3 +-
 .../jobhistory/JSONHistoryViewerPrinter.java    |  3 +-
 .../jobhistory/TestHistoryViewerPrinter.java    | 76 ++++++++++++++++++++
 3 files changed, 80 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b47275fe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
index d3da9f4..fdf3c47 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
@@ -148,7 +148,8 @@ class HumanReadableHistoryViewerPrinter implements HistoryViewerPrinter
{
           "Total Value"));
       buff.append("\n------------------------------------------" +
           "---------------------------------------------");
-      for (String groupName : totalCounters.getGroupNames()) {
+      for (CounterGroup counterGroup : totalCounters) {
+        String groupName = counterGroup.getName();
         CounterGroup totalGroup = totalCounters.getGroup(groupName);
         CounterGroup mapGroup = mapCounters.getGroup(groupName);
         CounterGroup reduceGroup = reduceCounters.getGroup(groupName);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b47275fe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
index 456dcf7..850fe2f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
@@ -104,7 +104,8 @@ class JSONHistoryViewerPrinter implements HistoryViewerPrinter {
     // Killed jobs might not have counters
     if (totalCounters != null) {
       JSONObject jGroups = new JSONObject();
-      for (String groupName : totalCounters.getGroupNames()) {
+      for (CounterGroup counterGroup : totalCounters) {
+        String groupName = counterGroup.getName();
         CounterGroup totalGroup = totalCounters.getGroup(groupName);
         CounterGroup mapGroup = mapCounters.getGroup(groupName);
         CounterGroup reduceGroup = reduceCounters.getGroup(groupName);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b47275fe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
index 358e9b2..74a2605 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
@@ -859,6 +859,46 @@ public class TestHistoryViewerPrinter {
         "}\n", outStr, JSONCompareMode.NON_EXTENSIBLE);
   }
 
+  @Test
+  public void testHumanDupePrinter() throws Exception {
+    JobHistoryParser.JobInfo job = createJobInfo2();
+    // Counters are only part of the overview so printAll can be false or true
+    // this does not affect the test, task counters are not printed
+    HumanReadableHistoryViewerPrinter printer =
+        new HumanReadableHistoryViewerPrinter(job, false, "http://",
+            TimeZone.getTimeZone("GMT"));
+    String outStr = run(printer);
+    // We are not interested in anything but the duplicate counter
+    int count1 = outStr.indexOf(
+        "|Map-Reduce Framework          |Map input records             |");
+    Assert.assertNotEquals("First counter occurrence not found", -1, count1);
+    int count2 = outStr.indexOf(
+        "|Map-Reduce Framework          |Map input records             |",
+        count1 + 1);
+    Assert.assertEquals("Duplicate counter found at: " + count1 +
+        " and " + count2, -1, count2);
+  }
+
+  @Test
+  public void testJSONDupePrinter() throws Exception {
+    JobHistoryParser.JobInfo job = createJobInfo2();
+    // Counters are part of the overview and task info
+    // Tasks only have bogus counters in the test if that is changed printAll
+    // must then be kept as false for this test to pass
+    JSONHistoryViewerPrinter printer =
+        new JSONHistoryViewerPrinter(job, false, "http://");
+    String outStr = run(printer);
+    // We are not interested in anything but the duplicate counter
+    int count1 = outStr.indexOf(
+        "\"counterName\":\"MAP_INPUT_RECORDS\"");
+    Assert.assertNotEquals("First counter occurrence not found", -1, count1);
+    int count2 = outStr.indexOf(
+        "\"counterName\":\"MAP_INPUT_RECORDS\"",
+        count1 + 1);
+    Assert.assertEquals("Duplicate counter found at: " + count1 +
+        " and " + count2, -1, count2);
+  }
+
   private String run(HistoryViewerPrinter printer) throws Exception {
     ByteArrayOutputStream boas = new ByteArrayOutputStream();
     PrintStream out = new PrintStream(boas, true);
@@ -901,6 +941,32 @@ public class TestHistoryViewerPrinter {
     addTaskInfo(job, TaskType.JOB_CLEANUP, 9, TaskStatus.State.SUCCEEDED);
     return job;
   }
+  private static JobHistoryParser.JobInfo createJobInfo2() {
+    JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo();
+    job.submitTime = 1317928501754L;
+    job.finishTime = job.submitTime + 15000;
+    job.jobid = JobID.forName("job_1317928501754_0001");
+    job.username = "test";
+    job.jobname = "Dupe counter output";
+    job.jobQueueName = "root.test";
+    job.jobConfPath = "/tmp/job.xml";
+    job.launchTime = job.submitTime + 1000;
+    job.totalMaps = 1;
+    job.totalReduces = 0;
+    job.finishedMaps = 1;
+    job.finishedReduces = 0;
+    job.failedMaps = 0;
+    job.failedReduces = 0;
+    job.jobStatus = JobStatus.State.SUCCEEDED.name();
+    job.totalCounters = createDeprecatedCounters();
+    job.mapCounters = createDeprecatedCounters();
+    job.reduceCounters = createDeprecatedCounters();
+    job.tasksMap = new HashMap<>();
+    addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED);
+    addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.SUCCEEDED);
+    addTaskInfo(job, TaskType.JOB_CLEANUP, 3, TaskStatus.State.SUCCEEDED);
+    return job;
+  }
 
   private static Counters createCounters() {
     Counters counters = new Counters();
@@ -910,6 +976,16 @@ public class TestHistoryViewerPrinter {
     return counters;
   }
 
+  private static Counters createDeprecatedCounters() {
+    Counters counters = new Counters();
+    // Deprecated counter: make sure it is only printed once
+    counters.findCounter("org.apache.hadoop.mapred.Task$Counter",
+        "MAP_INPUT_RECORDS").setValue(1);
+    counters.findCounter("File System Counters",
+        "FILE: Number of bytes read").setValue(1);
+    return counters;
+  }
+
   private static void addTaskInfo(JobHistoryParser.JobInfo job,
       TaskType type, int id, TaskStatus.State status) {
     JobHistoryParser.TaskInfo task = new JobHistoryParser.TaskInfo();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message