eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject eagle git commit: [MINOR] fix small bugs in MR job performance monitoring
Date Mon, 23 Jan 2017 06:40:01 GMT
Repository: eagle
Updated Branches:
  refs/heads/master eae6e8f11 -> 9381c2d17


[MINOR] fix small bugs in MR job performance monitoring

Author: Zhao, Qingwen <qingwzhao@apache.org>

Closes #789 from qingwen220/minor.


Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/9381c2d1
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/9381c2d1
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/9381c2d1

Branch: refs/heads/master
Commit: 9381c2d17e3e10a02e9f0f359a05d0728ee17752
Parents: eae6e8f
Author: Zhao, Qingwen <qingwzhao@apache.org>
Authored: Mon Jan 23 14:39:52 2017 +0800
Committer: Zhao, Qingwen <qingwzhao@apache.org>
Committed: Mon Jan 23 14:39:52 2017 +0800

----------------------------------------------------------------------
 .../apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java   | 2 +-
 .../jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java     | 4 ++++
 .../jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java      | 3 +--
 .../java/org/apache/eagle/jpm/analyzer/publisher/Result.java     | 4 +++-
 4 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
index e32a37c..57e1765 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
@@ -48,7 +48,7 @@ public class MRJobPerformanceAnalyzer<T extends AnalyzerEntity> implements
JobAn
         evaluators.add(new JobSuggestionEvaluator(config));
 
         publishers.add(new EagleStorePublisher(config));
-        publishers.add(new EmailPublisher(config));
+        //publishers.add(new EmailPublisher(config));
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java
b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java
index 103de7a..4007747 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java
@@ -20,6 +20,7 @@ package org.apache.eagle.jpm.analyzer.mr.suggestion;
 import org.apache.eagle.jpm.analyzer.Processor;
 import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity;
 import org.apache.eagle.jpm.analyzer.publisher.Result;
+import org.apache.eagle.jpm.util.Constants;
 import org.apache.eagle.jpm.util.jobcounter.JobCounters;
 
 import java.util.ArrayList;
@@ -45,9 +46,11 @@ public class MapReduceGCTimeProcessor implements Processor<MapReduceAnalyzerEnti
             long mapGCTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS);
             long mapCPUTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS);
 
+            String message = "%s GC_MILLISECONDS is: %s, and CPU_MILLISECONDS is: %s. ";
             if (mapGCTime > mapCPUTime * 0.1) {
                 setting = String.format("-D%s", MAP_JAVA_OPTS);
                 optSettings.add(setting);
+                sb.append(String.format(message, Constants.TaskType.MAP, mapGCTime, mapCPUTime));
                 sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory
via ").append(setting);
                 sb.append(", or optimize your mapper class.\n");
             }
@@ -58,6 +61,7 @@ public class MapReduceGCTimeProcessor implements Processor<MapReduceAnalyzerEnti
                 if (reduceGCTime > reduceCPUTime * 0.1) {
                     setting = String.format("-D%s", REDUCE_JAVA_OPTS);
                     optSettings.add(setting);
+                    sb.append(String.format(message, Constants.TaskType.REDUCE, reduceGCTime,
reduceCPUTime));
                     sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory
for reduce via ").append(setting);
                     sb.append(", or optimize your reducer class.\n");
                 }

http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java
b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java
index 835b382..96be2d5 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java
@@ -82,7 +82,7 @@ public class MapReduceSpillProcessor implements Processor<MapReduceAnalyzerEntit
                     long heapSize = getMaxHeapSize(context.getJobconf().get(MAP_JAVA_OPTS));
                     if (heapSize < 3 * minMapSpillMemMB) {
                         long expectedHeapSizeMB = (minMapSpillMemMB * 3 + 1024) / 1024 *
1024;
-                        setting = String.format(" -D%s=-Xmx%sM", MAP_JAVA_OPTS, expectedHeapSizeMB);
+                        setting = String.format("-D%s=-Xmx%sM", MAP_JAVA_OPTS, expectedHeapSizeMB);
                         sb.append(" ").append(setting);
                         optSettings.add(setting);
                     }
@@ -90,7 +90,6 @@ public class MapReduceSpillProcessor implements Processor<MapReduceAnalyzerEntit
                 sb.append(" to avoid spilled records.\n");
             }
 
-
             long reduceInputRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS);
             spillRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS);
             if (reduceInputRecords < spillRecords) {

http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java
b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java
index a9f5132..7d7442b 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java
@@ -63,9 +63,11 @@ public class Result {
     }
 
     private void normalizeResult(ProcessorResult processorResult) {
+        String settingList = "";
         if (processorResult.getSettings() != null && !processorResult.getSettings().isEmpty())
{
-            processorResult.setSettingList(StringUtils.join(processorResult.getSettings(),
"\n"));
+            settingList = StringUtils.join(processorResult.getSettings(), "\n");
         }
+        processorResult.setSettingList(settingList);
     }
 
     /**


Mime
View raw message