hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject hadoop git commit: MAPREDUCE-5568. Fixed CompletedJob in JHS to show progress percentage correctly in case the number of mappers or reducers is zero. Contributed by MinJi Kim
Date Tue, 25 Nov 2014 21:03:20 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 5805a81ef -> 78f7cdbfd


MAPREDUCE-5568. Fixed CompletedJob in JHS to show progress percentage correctly in case the
number of mappers or reducers is zero. Contributed by MinJi Kim


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78f7cdbf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78f7cdbf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78f7cdbf

Branch: refs/heads/trunk
Commit: 78f7cdbfd6e2b9fac51c369c748ae93d12ef065a
Parents: 5805a81
Author: Jian He <jianhe@apache.org>
Authored: Tue Nov 25 13:02:55 2014 -0800
Committer: Jian He <jianhe@apache.org>
Committed: Tue Nov 25 13:02:55 2014 -0800

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  4 +++
 .../hadoop/mapreduce/v2/hs/CompletedJob.java    | 14 +++++++--
 .../mapreduce/v2/hs/TestJobHistoryEntities.java | 30 ++++++++++++++++----
 ...33-2-0-SUCCEEDED-default-1416424779349.jhist | 29 +++++++++++++++++++
 4 files changed, 70 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/78f7cdbf/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 4724839..912f1a9 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -253,6 +253,10 @@ Release 2.7.0 - UNRELEASED
 
     MAPREDUCE-6049. AM JVM does not exit if MRClientService graceful shutdown
     fails (Rohith via devaraj)
+
+    MAPREDUCE-5568. Fixed CompletedJob in JHS to show progress percentage
+    correctly in case the number of mappers or reducers is zero. (MinJi Kim via
+    jianhe)
  
 Release 2.6.0 - 2014-11-18
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/78f7cdbf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
index 9e38316..66c44c8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
@@ -141,8 +141,18 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
     report.setFinishTime(jobInfo.getFinishTime());
     report.setJobName(jobInfo.getJobname());
     report.setUser(jobInfo.getUsername());
-    report.setMapProgress((float) getCompletedMaps() / getTotalMaps());
-    report.setReduceProgress((float) getCompletedReduces() / getTotalReduces());
+
+    if ( getTotalMaps() == 0 ) {
+      report.setMapProgress(1.0f);
+    } else {
+      report.setMapProgress((float) getCompletedMaps() / getTotalMaps());
+    }
+    if ( getTotalReduces() == 0 ) {
+      report.setReduceProgress(1.0f);
+    } else {
+      report.setReduceProgress((float) getCompletedReduces() / getTotalReduces());
+    }
+
     report.setJobFile(getConfFile().toString());
     String historyUrl = "N/A";
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/78f7cdbf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
index 2d9e09a..41b1fd6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
@@ -55,14 +55,19 @@ public class TestJobHistoryEntities {
 
   private final String historyFileName =
       "job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";
+  private final String historyFileNameZeroReduceTasks =
+    "job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist";
   private final String confFileName = "job_1329348432655_0001_conf.xml";
   private final Configuration conf = new Configuration();
   private final JobACLsManager jobAclsManager = new JobACLsManager(conf);
   private boolean loadTasks;
   private JobId jobId = MRBuilderUtils.newJobId(1329348432655l, 1, 1);
-  Path fulleHistoryPath =
+  Path fullHistoryPath =
     new Path(this.getClass().getClassLoader().getResource(historyFileName)
         .getFile());
+  Path fullHistoryPathZeroReduces =
+    new Path(this.getClass().getClassLoader().getResource(historyFileNameZeroReduceTasks)
+        .getFile());
   Path fullConfPath =
     new Path(this.getClass().getClassLoader().getResource(confFileName)
         .getFile());
@@ -87,7 +92,7 @@ public class TestJobHistoryEntities {
     when(info.getConfFile()).thenReturn(fullConfPath);
     //Re-initialize to verify the delayed load.
     completedJob =
-      new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
+      new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
           info, jobAclsManager);
     //Verify tasks loaded based on loadTask parameter.
     assertEquals(loadTasks, completedJob.tasksLoaded.get());
@@ -106,12 +111,27 @@ public class TestJobHistoryEntities {
     assertEquals(JobState.SUCCEEDED, jobReport.getJobState());
   }
   
+  @Test (timeout=100000)
+  public void testCopmletedJobReportWithZeroTasks() throws Exception {
+    HistoryFileInfo info = mock(HistoryFileInfo.class);
+    when(info.getConfFile()).thenReturn(fullConfPath);
+    completedJob =
+      new CompletedJob(conf, jobId, fullHistoryPathZeroReduces, loadTasks, "user",
+          info, jobAclsManager);
+    JobReport jobReport = completedJob.getReport();
+    // Make sure that the number reduces (completed and total) are equal to zero.
+    assertEquals(0, completedJob.getTotalReduces());
+    assertEquals(0, completedJob.getCompletedReduces());
+    // Verify that the reduce progress is 1.0 (not NaN)
+    assertEquals(1.0, jobReport.getReduceProgress(), 0.001);
+  }
+
   @Test (timeout=10000)
   public void testCompletedTask() throws Exception {
     HistoryFileInfo info = mock(HistoryFileInfo.class);
     when(info.getConfFile()).thenReturn(fullConfPath);
     completedJob =
-      new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
+      new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
           info, jobAclsManager);
     TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
     TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
@@ -140,7 +160,7 @@ public class TestJobHistoryEntities {
     HistoryFileInfo info = mock(HistoryFileInfo.class);
     when(info.getConfFile()).thenReturn(fullConfPath);
     completedJob =
-      new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
+      new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
           info, jobAclsManager);
     TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
     TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
@@ -179,7 +199,7 @@ public class TestJobHistoryEntities {
     HistoryFileInfo info = mock(HistoryFileInfo.class);
     when(info.getConfFile()).thenReturn(fullConfPath);
     completedJob =
-      new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
+      new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
           info, jobAclsManager);
     TaskCompletionEvent[] events= completedJob.getMapAttemptCompletionEvents(0,1000);
     assertEquals(10, completedJob.getMapAttemptCompletionEvents(0,10).length);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/78f7cdbf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist
new file mode 100644
index 0000000..1a02fc8
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist
@@ -0,0 +1,29 @@
+Avro-Json
+{"type":"record","name":"Event","namespace":"org.apache.hadoop.mapreduce.jobhistory","fields":[{"name":"type","type":{"type":"enum","name":"EventType","symbols":["JOB_SUBMITTED","JOB_INITED","JOB_FINISHED","JOB_PRIORITY_CHANGED","JOB_STATUS_CHANGED","JOB_QUEUE_CHANGED","JOB_FAILED","JOB_KILLED","JOB_ERROR","JOB_INFO_CHANGED","TASK_STARTED","TASK_FINISHED","TASK_FAILED","TASK_UPDATED","NORMALIZED_RESOURCE","MAP_ATTEMPT_STARTED","MAP_ATTEMPT_FINISHED","MAP_ATTEMPT_FAILED","MAP_ATTEMPT_KILLED","REDUCE_ATTEMPT_STARTED","REDUCE_ATTEMPT_FINISHED","REDUCE_ATTEMPT_FAILED","REDUCE_ATTEMPT_KILLED","SETUP_ATTEMPT_STARTED","SETUP_ATTEMPT_FINISHED","SETUP_ATTEMPT_FAILED","SETUP_ATTEMPT_KILLED","CLEANUP_ATTEMPT_STARTED","CLEANUP_ATTEMPT_FINISHED","CLEANUP_ATTEMPT_FAILED","CLEANUP_ATTEMPT_KILLED","AM_STARTED"]}},{"name":"event","type":[{"type":"record","name":"JobFinished","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"f
 inishedReduces","type":"int"},{"name":"failedMaps","type":"int"},{"name":"failedReduces","type":"int"},{"name":"totalCounters","type":{"type":"record","name":"JhCounters","fields":[{"name":"name","type":"string"},{"name":"groups","type":{"type":"array","items":{"type":"record","name":"JhCounterGroup","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"counts","type":{"type":"array","items":{"type":"record","name":"JhCounter","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"value","type":"long"}]}}}]}}}]}},{"name":"mapCounters","type":"JhCounters"},{"name":"reduceCounters","type":"JhCounters"}]},{"type":"record","name":"JobInfoChange","fields":[{"name":"jobid","type":"string"},{"name":"submitTime","type":"long"},{"name":"launchTime","type":"long"}]},{"type":"record","name":"JobInited","fields":[{"name":"jobid","type":"string"},{"name":"launchTime","type":"long"},{"name":"totalMaps","type":"int"},{"name":"t
 otalReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"uberized","type":"boolean"}]},{"type":"record","name":"AMStarted","fields":[{"name":"applicationAttemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"containerId","type":"string"},{"name":"nodeManagerHost","type":"string"},{"name":"nodeManagerPort","type":"int"},{"name":"nodeManagerHttpPort","type":"int"}]},{"type":"record","name":"JobPriorityChange","fields":[{"name":"jobid","type":"string"},{"name":"priority","type":"string"}]},{"type":"record","name":"JobQueueChange","fields":[{"name":"jobid","type":"string"},{"name":"jobQueueName","type":"string"}]},{"type":"record","name":"JobStatusChanged","fields":[{"name":"jobid","type":"string"},{"name":"jobStatus","type":"string"}]},{"type":"record","name":"JobSubmitted","fields":[{"name":"jobid","type":"string"},{"name":"jobName","type":"string"},{"name":"userName","type":"string"},{"name":"submitTime","type":"long"},{"name":"jobConfPath","type":"st
 ring"},{"name":"acls","type":{"type":"map","values":"string"}},{"name":"jobQueueName","type":"string"},{"name":"workflowId","type":["null","string"],"default":null},{"name":"workflowName","type":["null","string"],"default":null},{"name":"workflowNodeName","type":["null","string"],"default":null},{"name":"workflowAdjacencies","type":["null","string"],"default":null},{"name":"workflowTags","type":["null","string"],"default":null}]},{"type":"record","name":"JobUnsuccessfulCompletion","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"diagnostics","type":["null","string"],"default":null}]},{"type":"record","name":"MapAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"mapFinishTime","type":"long"},{"name":"finis
 hTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"ReduceAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"shuffleFinishTime","type":"long"},{"name":"sortFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}}
 ,{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"finishTime","type":"long"},{"name":"rackname","type":"string"},{"name":"hostname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"}]},{"type":"record","name":"TaskAttemptStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"trackerName","type":"string"},{"name":"httpPort","type":"int"},{"name":"shufflePort","type":"int"},{"name":"containerId","type":"string"},{"name":"locality","type":["null","string"],"default":null},{"name":"avataar","type"
 :["null","string"],"default":null}]},{"type":"record","name":"TaskAttemptUnsuccessfulCompletion","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"status","type":"string"},{"name":"error","type":"string"},{"name":"counters","type":["null","JhCounters"],"default":null},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskFailed","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"error","type":"string"},{"name":"failedDueToAttempt","type":["null","string"]},{"name":"status","type":"stri
 ng"},{"name":"counters","type":["null","JhCounters"],"default":null}]},{"type":"record","name":"TaskFinished","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"status","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"successfulAttemptId","type":["null","string"],"default":null}]},{"type":"record","name":"TaskStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"startTime","type":"long"},{"name":"splitLocations","type":"string"}]},{"type":"record","name":"TaskUpdated","fields":[{"name":"taskid","type":"string"},{"name":"finishTime","type":"long"}]}]}]}
+{"type":"AM_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.AMStarted":{"applicationAttemptId":"appattempt_1416424547277_0002_000001","startTime":1416424776369,"containerId":"container_1416424547277_0002_01_000001","nodeManagerHost":"mfs137.qa.lab","nodeManagerPort":35535,"nodeManagerHttpPort":8042}}}
+
+{"type":"JOB_SUBMITTED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobSubmitted":{"jobid":"job_1416424547277_0002","jobName":"TeraGen","userName":"root","submitTime":1416424775281,"jobConfPath":"hdfs://localhost:9000/tmp/hadoop-yarn/staging/root/.staging/job_1416424547277_0002/job.xml","acls":{},"jobQueueName":"default","workflowId":{"string":""},"workflowName":{"string":""},"workflowNodeName":{"string":""},"workflowAdjacencies":{"string":""},"workflowTags":{"string":""}}}}
+
+{"type":"JOB_QUEUE_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobQueueChange":{"jobid":"job_1416424547277_0002","jobQueueName":"default"}}}
+
+{"type":"JOB_INITED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInited":{"jobid":"job_1416424547277_0002","launchTime":1416424779349,"totalMaps":2,"totalReduces":0,"jobStatus":"INITED","uberized":false}}}
+
+{"type":"JOB_INFO_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInfoChange":{"jobid":"job_1416424547277_0002","submitTime":1416424775281,"launchTime":1416424779349}}}
+
+{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1416424547277_0002_m_000000","taskType":"MAP","startTime":1416424779381,"splitLocations":""}}}
+
+{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1416424547277_0002_m_000001","taskType":"MAP","startTime":1416424779383,"splitLocations":""}}}
+
+{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1416424547277_0002_m_000000","taskType":"MAP","attemptId":"attempt_1416424547277_0002_m_000000_0","startTime":1416424781561,"trackerName":"mfs137.qa.lab","httpPort":8042,"shufflePort":13562,"containerId":"container_1416424547277_0002_01_000002","locality":{"string":"OFF_SWITCH"},"avataar":{"string":"VIRGIN"}}}}
+
+{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1416424547277_0002_m_000001","taskType":"MAP","attemptId":"attempt_1416424547277_0002_m_000001_0","startTime":1416424782404,"trackerName":"mfs137.qa.lab","httpPort":8042,"shufflePort":13562,"containerId":"container_1416424547277_0002_01_000003","locality":{"string":"OFF_SWITCH"},"avataar":{"string":"VIRGIN"}}}}
+
+{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1416424547277_0002_m_000000","attemptId":"attempt_1416424547277_0002_m_000000_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1416424784304,"finishTime":1416424784542,"hostname":"mfs137.qa.lab","port":35535,"rackname":"/default-rack","state":"map","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes
read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":96586},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number
of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number
of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDF
 S: Number of bytes read","value":79},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number
of bytes written","value":5000},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read
operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large
read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write
operations","value":2}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":50},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map
output records","value":50},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":79},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged
Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":36},{
 "name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":550},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes) snapshot","value":142049280},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes) snapshot","value":1169448960},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":152043520}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":118655541199}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":5000}]}]},"clockSplits":[2772,18,1
 8,18,18,18,18,18,18,18,18,18],"cpuUsages":[45,46,46,46,46,46,45,46,46,46,46,46],"vMemKbytes":[47585,142755,237925,333094,428265,523434,618605,713774,808945,904115,999284,1094455],"physMemKbytes":[5780,17340,28900,40459,52020,63579,75140,86699,98260,109820,121379,132940]}}}
+
+{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1416424547277_0002_m_000000","taskType":"MAP","finishTime":1416424784542,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes
read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":96586},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number
of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number
of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes
read","value":79},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":5000},{"name":"HDFS_READ_OPS","displayName":"HDFS:
Number of
  read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of
large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write operations","value":2}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":50},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map
output records","value":50},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":79},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged
Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":36},{"name":"CPU_MILLISECONDS","displayName":"CPU
time spent (ms)","value":550},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory
(bytes) snapshot","value":142049280},{"nam
 e":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":1169448960},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":152043520}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":118655541199}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":5000}]}]},"successfulAttemptId":{"string":"attempt_1416424547277_0002_m_000000_0"}}}}
+
+{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1416424547277_0002_m_000001","attemptId":"attempt_1416424547277_0002_m_000001_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1416424785165,"finishTime":1416424785379,"hostname":"mfs137.qa.lab","port":35535,"rackname":"/default-rack","state":"map","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes
read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":96586},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number
of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number
of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDF
 S: Number of bytes read","value":79},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number
of bytes written","value":5000},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read
operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large
read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write
operations","value":2}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":50},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map
output records","value":50},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":79},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged
Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":34},{
 "name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":510},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes) snapshot","value":142311424},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes) snapshot","value":1169448960},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":152043520}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":114863641618}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":5000}]}]},"clockSplits":[2781,17,1
 7,17,17,18,17,17,17,17,17,18],"cpuUsages":[42,43,42,43,42,43,42,43,42,43,42,43],"vMemKbytes":[47585,142755,237925,333094,428265,523434,618605,713774,808945,904115,999284,1094455],"physMemKbytes":[5790,17371,28953,40534,52115,63696,75278,86859,98441,110022,121603,133185]}}}
+
+{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1416424547277_0002_m_000001","taskType":"MAP","finishTime":1416424785379,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes
read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":96586},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number
of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number
of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes
read","value":79},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":5000},{"name":"HDFS_READ_OPS","displayName":"HDFS:
Number of
  read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of
large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write operations","value":2}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":50},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map
output records","value":50},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":79},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged
Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":34},{"name":"CPU_MILLISECONDS","displayName":"CPU
time spent (ms)","value":510},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory
(bytes) snapshot","value":142311424},{"nam
 e":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":1169448960},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":152043520}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":114863641618}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":5000}]}]},"successfulAttemptId":{"string":"attempt_1416424547277_0002_m_000001_0"}}}}
+
+{"type":"JOB_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobFinished":{"jobid":"job_1416424547277_0002","finishTime":1416424785433,"finishedMaps":2,"finishedReduces":0,"failedMaps":0,"failedReduces":0,"totalCounters":{"name":"TOTAL_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes
read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":193172},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number
of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number
of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes
read","value":158},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":10000},{"name":"HDFS_READ_OP
 S","displayName":"HDFS: Number of read operations","value":8},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS:
Number of write operations","value":4}]},{"name":"org.apache.hadoop.mapreduce.JobCounter","displayName":"Job
Counters ","counts":[{"name":"TOTAL_LAUNCHED_MAPS","displayName":"Launched map tasks","value":2},{"name":"OTHER_LOCAL_MAPS","displayName":"Other
local map tasks","value":2},{"name":"SLOTS_MILLIS_MAPS","displayName":"Total time spent by
all maps in occupied slots (ms)","value":5956},{"name":"MILLIS_MAPS","displayName":"Total
time spent by all map tasks (ms)","value":5956},{"name":"VCORES_MILLIS_MAPS","displayName":"Total
vcore-seconds taken by all map tasks","value":5956},{"name":"MB_MILLIS_MAPS","displayName":"Total
megabyte-seconds taken by all map tasks","value":6098944}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name
 ":"MAP_INPUT_RECORDS","displayName":"Map input records","value":100},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map
output records","value":100},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":158},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged
Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":70},{"name":"CPU_MILLISECONDS","displayName":"CPU
time spent (ms)","value":1060},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory
(bytes) snapshot","value":284360704},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes) snapshot","value":2338897920},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":304087040}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort
 .TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":233519182817}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":10000}]}]},"mapCounters":{"name":"MAP_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes
read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":193172},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number
of large read operations","value":0},{"name":"FILE_
 WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS:
Number of bytes read","value":158},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number
of bytes written","value":10000},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read
operations","value":8},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large
read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write
operations","value":4}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":100},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map
output records","value":100},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":158},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","
 displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time
elapsed (ms)","value":70},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":1060},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes) snapshot","value":284360704},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes) snapshot","value":2338897920},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":304087040}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":233519182817}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output For
 mat Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":10000}]}]},"reduceCounters":{"name":"REDUCE_COUNTERS","groups":[]}}}}


Mime
View raw message