eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject incubator-eagle git commit: [EAGLE-834] Update Daily Job Summery Report
Date Fri, 16 Dec 2016 07:04:07 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 229d7b907 -> a89275bfc


[EAGLE-834] Update Daily Job Summery Report

https://issues.apache.org/jira/browse/EAGLE-834

* update mail subject & title
* adjust mail template and enrich the summary data
* fix a query bug in the mr job list API (GET /mrJobs)

Author: Zhao, Qingwen <qingwzhao@apache.org>

Closes #749 from qingwen220/EAGLE-834.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a89275bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a89275bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a89275bf

Branch: refs/heads/master
Commit: a89275bfc86c4db63688e28ff12196ee97a78207
Parents: 229d7b9
Author: Zhao, Qingwen <qingwzhao@apache.org>
Authored: Fri Dec 16 15:04:00 2016 +0800
Committer: Zhao, Qingwen <qingwzhao@apache.org>
Committed: Fri Dec 16 15:04:00 2016 +0800

----------------------------------------------------------------------
 .../mr/history/MRHistoryJobDailyReporter.java   | 169 +++++++++----------
 .../src/main/resources/JobReportTemplate.vm     | 118 +++++++------
 .../history/MRHistoryJobDailyReporterTest.java  |  43 ++---
 .../service/jpm/MRJobExecutionResource.java     |  26 +--
 4 files changed, 177 insertions(+), 179 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a89275bf/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
index 6b38244..0dc6c5f 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
@@ -39,19 +39,20 @@ import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.eagle.common.config.EagleConfigConstants.EAGLE_TIME_ZONE;
 import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_HOST;
 import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_PORT;
 
 public class MRHistoryJobDailyReporter extends AbstractScheduledService {
     private static final Logger LOG = LoggerFactory.getLogger(MRHistoryJobDailyReporter.class);
-    private static final String TIMEZONE_PATH = "service.timezone";
+
     private static final String DAILY_SENT_HOUROFDAY = "application.dailyJobReport.reportHourTime";
     private static final String DAILY_SENT_PERIOD = "application.dailyJobReport.reportPeriodInHour";
     private static final String NUM_TOP_USERS  = "application.dailyJobReport.numTopUsers";
     private static final String JOB_OVERTIME_LIMIT_HOUR  = "application.dailyJobReport.jobOvertimeLimitInHour";
 
     public static final String SERVICE_PATH = "application.dailyJobReport";
-    public static final String APP_TYPE = "MR_HISTORY_JOB_APP";
+    protected static final String APP_TYPE = "MR_HISTORY_JOB_APP";
 
     // alert context keys
     protected static final String NUM_TOP_USERS_KEY = "numTopUsers";
@@ -64,6 +65,12 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService
{
     protected static final String FINISHED_JOB_USERS_KEY = "finishedJobUsers";
     protected static final String EAGLE_JOB_LINK_KEY = "eagleJobLink";
 
+    // queries
+    private static final String STATUS_QUERY = "%s[@site=\"%s\" and @endTime<=%s]<@currentState>{count}.{count
desc}";
+    private static final String FAILED_JOBS_QUERY = "%s[@site=\"%s\" and @currentState=\"FAILED\"
and @endTime<=%s]<@user>{count}.{count desc}";
+    private static final String SUCCEEDED_JOB_QUERY = "%s[@site=\"%s\" and @currentState=\"SUCCEEDED\"
and @durationTime>%s and @endTime<=%s]<@user>{count}.{count desc}";
+    private static final String FINISHED_JOB_QUERY = "%s[@site=\"%s\" and @endTime<=%s]<@user>{count}.{count
desc}";
+
     private Config config;
     private IEagleServiceClient client;
     private ApplicationEntityService applicationResource;
@@ -77,13 +84,13 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService
{
     private int jobOvertimeLimit = 6;
 
     // scheduler
-    private int initialDelayMin = 10;
+    private int initialDelayMin = 5;
     private int periodInMin = 60;
     private TimeZone timeZone;
 
     @Inject
     public MRHistoryJobDailyReporter(Config config, ApplicationEntityService applicationEntityService)
{
-        this.timeZone = TimeZone.getTimeZone(config.getString(TIMEZONE_PATH));
+        this.timeZone = TimeZone.getTimeZone(config.getString(EAGLE_TIME_ZONE));
 
         if (config.hasPath(SERVICE_PATH) && config.hasPath(AlertEmailConstants.EAGLE_APPLICATION_EMAIL_SERVICE))
{
             this.emailService = new ApplicationEmailService(config, SERVICE_PATH);
@@ -139,8 +146,10 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService
{
                 for (String site : sites) {
                     int reportHour = currentHour / dailySentPeriod * dailySentPeriod;
                     calendar.set(Calendar.HOUR_OF_DAY, reportHour);
-                    String subject = String.format("%s %s", site.toUpperCase(), config.getString(SERVICE_PATH
+ "." + AlertEmailConstants.SUBJECT));
-                    Map<String, Object> alertData = buildAlertData(site, calendar.getTimeInMillis());
+                    long endTime = calendar.getTimeInMillis() / DateTimeUtil.ONEHOUR * DateTimeUtil.ONEHOUR;
+                    long startTime = endTime - DateTimeUtil.ONEHOUR * dailySentPeriod;
+                    String subject = buildAlertSubject(site, startTime, endTime);
+                    Map<String, Object> alertData = buildAlertData(site, startTime,
endTime);
                     sendByEmailWithSubject(alertData, subject);
                 }
             } catch (Exception ex) {
@@ -159,27 +168,30 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService
{
         emailService.onAlert(alertContext, alertData);
     }
 
-    private Map<String, Object> buildAlertData(String site, long endTime) {
+    protected String buildAlertSubject(String site, long startTime, long endTime) {
+        String subjectFormat = "[%s] Job Report by %s";
+        String date = DateTimeUtil.format(endTime, "yyyyMMdd HH:mm");
+        //String startHour = DateTimeUtil.format(startTime, "HH:mm");
+        //String endHour = DateTimeUtil.format(endTime, "kk:mm");
+        return String.format(subjectFormat, site.toUpperCase(), date);
+    }
+
+    private Map<String, Object> buildAlertData(String site, long startTime, long endTime)
{
         StopWatch watch = new StopWatch();
         Map<String, Object> data = new HashMap<>();
         this.client = new EagleServiceClientImpl(config);
-        long startTime = endTime - DateTimeUtil.ONEHOUR * dailySentPeriod;
-        LOG.info("Going to report job summery info for site {} from {} to {}", site,
-            DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime),
-            DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime));
+        String startTimeStr = DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime);
+        String endTimeStr = DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime);
+        LOG.info("Going to report job summery info for site {} from {} to {}", site, startTimeStr,
endTimeStr);
         try {
             watch.start();
             data.putAll(buildJobSummery(site, startTime, endTime));
-            data.putAll(buildFailedJobInfo(site, startTime, endTime));
-            data.putAll(buildSucceededJobInfo(site, startTime, endTime));
-            data.putAll(buildFinishedJobInfo(site, startTime, endTime));
             data.put(NUM_TOP_USERS_KEY, numTopUsers);
             data.put(JOB_OVERTIME_LIMIT_KEY, jobOvertimeLimit);
-            data.put(ALERT_TITLE_KEY, String.format("[%s] Daily Job Report", site.toUpperCase()));
-            data.put(REPORT_RANGE_KEY, String.format("%s ~ %s %s",
-                DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime),
-                DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime),
-                DateTimeUtil.CURRENT_TIME_ZONE.getID()));
+            data.put(ALERT_TITLE_KEY, String.format("[%s] Job Report for 12 Hours", site.toUpperCase()));
+            data.put(REPORT_RANGE_KEY, String.format("%s ~ %s %s", startTimeStr, endTimeStr,
DateTimeUtil.CURRENT_TIME_ZONE.getID()));
+            data.put(EAGLE_JOB_LINK_KEY, String.format("http://%s:%d/#/site/%s/jpm/list?startTime=%s&endTime=%s",
+                    config.getString(SERVICE_HOST), config.getInt(SERVICE_PORT), site, startTimeStr,
endTimeStr));
             watch.stop();
             LOG.info("Fetching DailyJobReport tasks {} seconds", watch.getTime() / DateTimeUtil.ONESECOND);
         } finally {
@@ -192,6 +204,48 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService
{
         return data;
     }
 
+    private Map<String, Object> buildJobSummery(String site, long startTime, long endTime)
{
+        Map<String, Object> data = new HashMap<>();
+
+        String query = String.format(STATUS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME,
site, endTime);
+        Map<String, Long> jobSummery = queryGroupByMetrics(query, startTime, endTime,
Integer.MAX_VALUE);
+        if (jobSummery == null || jobSummery.isEmpty()) {
+            LOG.warn("Result set is empty for query={}", query);
+            return data;
+        }
+        Long totalJobs = jobSummery.values().stream().reduce((a, b) -> a + b).get();
+        String finishedJobQuery = String.format(FINISHED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME,
site, endTime);
+        String failedJobQuery = String.format(FAILED_JOBS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME,
site, endTime);
+        String succeededJobQuery = String.format(SUCCEEDED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME,
site, jobOvertimeLimit * DateTimeUtil.ONEHOUR, endTime);
+        data.put(SUMMARY_INFO_KEY, processResult(jobSummery, totalJobs));
+        data.put(FAILED_JOB_USERS_KEY, buildJobSummery(failedJobQuery, startTime, endTime,
jobSummery.get(Constants.JobState.FAILED.toString())));
+        data.put(SUCCEEDED_JOB_USERS_KEY, buildJobSummery(succeededJobQuery, startTime, endTime,
jobSummery.get(Constants.JobState.SUCCEEDED.toString())));
+        data.put(FINISHED_JOB_USERS_KEY, buildJobSummery(finishedJobQuery, startTime, endTime,
totalJobs));
+
+        return data;
+    }
+
+    private List<JobSummaryInfo> buildJobSummery(String query,  long startTime, long
endTime, long totalJobs) {
+        Map<String, Long> jobUsers = queryGroupByMetrics(query, startTime, endTime,
numTopUsers);
+        if (jobUsers == null || jobUsers.isEmpty()) {
+            LOG.warn("Result set is empty for query={}", query);
+            return null;
+        }
+        return processResult(jobUsers, totalJobs);
+    }
+
+    private List<JobSummaryInfo> processResult(Map<String, Long> parsedResult,
long totalJobs) {
+        List<JobSummaryInfo> summaryInfoList = new ArrayList<>();
+        for (Map.Entry<String, Long> entry : parsedResult.entrySet()) {
+            JobSummaryInfo summaryInfo = new JobSummaryInfo();
+            summaryInfo.key = entry.getKey();
+            summaryInfo.numOfJobs = entry.getValue();
+            summaryInfo.ratio = Double.parseDouble(String.format("%.2f", summaryInfo.numOfJobs
* 100d / totalJobs));
+            summaryInfoList.add(summaryInfo);
+        }
+        return summaryInfoList;
+    }
+
     private Map<String, Long> parseQueryResult(List<Map<List<String>, List<Double>>>
result, int limit) {
         Map<String, Long> stateCount = new LinkedHashMap<>();
         for (Map<List<String>, List<Double>> map : result) {
@@ -213,6 +267,7 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService
{
                 .startTime(startTime)
                 .endTime(endTime).send();
             if (!response.isSuccess()) {
+                LOG.error(response.getException());
                 return null;
             }
             List<Map<List<String>, List<Double>>> result = response.getObj();
@@ -223,86 +278,18 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService
{
         }
     }
 
-    private Map<String, Object> buildJobSummery(String site, long startTime, long endTime)
{
-        Map<String, Object> data = new HashMap<>();
-        List<JobSummeryInfo> statusCount = new ArrayList<>();
-        String query = String.format("%s[@site=\"%s\" and @endTime<=%s]<@currentState>{count}",
-            Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
-        Map<String, Long> jobSummery = queryGroupByMetrics(query, startTime, endTime,
Integer.MAX_VALUE);
-        if (jobSummery == null || jobSummery.isEmpty()) {
-            return data;
-        }
-        Optional<Long> totalJobs = jobSummery.values().stream().reduce((a, b) ->
a + b);
-        for (Map.Entry<String, Long> entry : jobSummery.entrySet()) {
-            JobSummeryInfo summeryInfo = new JobSummeryInfo();
-            summeryInfo.status = entry.getKey();
-            summeryInfo.numOfJobs = entry.getValue();
-            summeryInfo.ratio = Double.parseDouble(String.format("%.2f", entry.getValue()
* 1d / totalJobs.get()));
-            statusCount.add(summeryInfo);
-        }
-        data.put(SUMMARY_INFO_KEY, statusCount);
-        return data;
-    }
-
-    private Map<String, Object> buildFailedJobInfo(String site, long startTime, long
endTime) {
-        Map<String, Object> data = new HashMap<>();
-        String query = String.format("%s[@site=\"%s\" and @currentState=\"FAILED\" and @endTime<=%s]<@user>{count}.{count
desc}",
-            Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
-
-        Map<String, Long> failedJobUsers = queryGroupByMetrics(query, startTime, endTime,
numTopUsers);
-        if (failedJobUsers == null || failedJobUsers.isEmpty()) {
-            LOG.warn("Result set is empty for query={}", query);
-            return data;
-        }
-        data.put(FAILED_JOB_USERS_KEY, failedJobUsers);
-        data.put(EAGLE_JOB_LINK_KEY, String.format("http://%s:%d/#/site/%s/jpm/list?startTime=%s&endTime=%s",
-            config.getString(SERVICE_HOST),
-            config.getInt(SERVICE_PORT),
-            site,
-            DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime),
-            DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime)));
-        return data;
-    }
-
-    private Map<String, Object> buildSucceededJobInfo(String site, long startTime,
long endTime) {
-        Map<String, Object> data = new HashMap<>();
-        long overtimeLimit = jobOvertimeLimit * DateTimeUtil.ONEHOUR;
-        String query = String.format("%s[@site=\"%s\" and @currentState=\"SUCCEEDED\" and
@durationTime>%s and @endTime<=%s]<@user>{count}.{count desc}",
-            Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, overtimeLimit, endTime);
-        Map<String, Long> succeededJobUsers = queryGroupByMetrics(query, startTime,
endTime, numTopUsers);
-        if (succeededJobUsers == null || succeededJobUsers.isEmpty()) {
-            LOG.warn("Result set is empty for query={}", query);
-            return data;
-        }
-        data.put(SUCCEEDED_JOB_USERS_KEY, succeededJobUsers);
-        return data;
-    }
-
-    private Map<String, Object> buildFinishedJobInfo(String site, long startTime, long
endTime) {
-        Map<String, Object> data = new HashMap<>();
-        String query = String.format("%s[@site=\"%s\" and @endTime<=%s]<@user>{count}.{count
desc}",
-            Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
-        Map<String, Long> jobUsers = queryGroupByMetrics(query, startTime, endTime,
numTopUsers);
-        if (jobUsers == null || jobUsers.isEmpty()) {
-            LOG.warn("Result set is empty for query={}", query);
-            return data;
-        }
-        data.put(FINISHED_JOB_USERS_KEY, jobUsers);
-        return data;
-    }
-
     @Override
     protected Scheduler scheduler() {
         return Scheduler.newFixedRateSchedule(initialDelayMin, periodInMin, TimeUnit.MINUTES);
     }
 
-    public static class JobSummeryInfo {
-        public String status;
+    public static class JobSummaryInfo {
+        public String key;
         public long numOfJobs;
         public double ratio;
 
-        public String getStatus() {
-            return status;
+        public String getKey() {
+            return key;
         }
 
         public long getNumOfJobs() {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a89275bf/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
index c3afa58..c878d6e 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
@@ -139,7 +139,7 @@
                                 </tr>
                                 <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;">
                                     <td class="content-block"
-                                        style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
+                                        style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 0px;"
                                         valign="top">
                                         <h2 class="aligncenter" style="font-family: 'Helvetica
Neue',Helvetica,Arial,'Lucida Grande',sans-serif; box-sizing: border-box; font-size: 20px;
color: #000; line-height: 1.2em; font-weight: 400; text-align: left; margin: 0 0 0;" align="center">
                                         Summary
@@ -176,12 +176,12 @@
                                                             <td style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align:
top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding:
5px 0;"
                                                                 valign="top">
                                                                 #set($statusColor = "#337ab7")
-                                                                #if($item.status == "SUCCEEDED")
+                                                                #if($item.key == "SUCCEEDED")
                                                                     #set($statusColor = "green")
-                                                                #elseif($item.status == "FAILED"
|| $item.status == "KILLED")
+                                                                #elseif($item.key == "FAILED"
|| $item.key == "KILLED")
                                                                     #set($statusColor = "red")
                                                                 #end
-                                                                <span style="color: $statusColor">$item.status</span>
+                                                                <span style="color: $statusColor">$item.key</span>
                                                             </td>
                                                             <td style="text-align: center;font-family:
'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align:
top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding:
5px 0;"
                                                                 valign="top">
@@ -189,8 +189,9 @@
                                                             </td>
                                                             <td style="text-align: right;font-family:
'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align:
top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding:
5px 0;"
                                                                 valign="top">
-                                                                #set ($ratio_percentage =
$item.ratio * 100)
-                                                                $ratio_percentage %
+##                                                                #set ($ratio_percentage
= $item.ratio * 100)
+##                                                                $ratio_percentage %
+                                                                  $item.ratio %
                                                             </td>
                                                         </tr>
                                                         #end
@@ -203,11 +204,11 @@
 
                                 <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;">
                                     <td class="content-block"
-                                        style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
+                                        style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 0px;"
                                         valign="top">
 ##                                        Top $alert["numTopUsers"] Users (Order by Number
of Failed Jobs)
                                         <h2 class="aligncenter" style="font-family: 'Helvetica
Neue',Helvetica,Arial,'Lucida Grande',sans-serif; box-sizing: border-box; font-size: 18px;
color: #000; line-height: 1.2em; font-weight: 400; text-align: left; margin: 0 0 0;" align="center">
-                                            Top $alert["numTopUsers"] Failed Job Users
+                                            Top $alert["numTopUsers"] Users by Failed Jobs
                                         </h2>
                                     </td>
                                 </tr>
@@ -236,19 +237,21 @@
                                                                 Ratio
                                                             </th>
                                                         </tr>
-                                                        #foreach($userItem in $alert["failedJobUsers"].entrySet())
-                                                            <tr style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
-                                                                <td style="font-family:
'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align:
top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding:
5px 0;"
-                                                                    valign="top">
-                                                                    $userItem.key
-                                                                </td>
-                                                                <td style="text-align:
center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style:
solid; margin: 0; padding: 5px 0;"
-                                                                    valign="top">$userItem.value
-                                                                </td>
-                                                                <td style="text-align:
right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style:
solid; margin: 0; padding: 5px 0;"
-                                                                    valign="top">$TODO_RATIO
-                                                                </td>
-                                                            </tr>
+                                                        #foreach($item in $alert["failedJobUsers"])
+															<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; margin: 0;">
+																<td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color:
#eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+																	valign="top">
+                                                                    $item.key
+																</td>
+																<td style="text-align: center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color:
#eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+																	valign="top">
+                                                                    $item.numOfJobs
+																</td>
+																<td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color:
#eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+																	valign="top">
+                                                                    $item.ratio %
+																</td>
+															</tr>
                                                         #end
                                                     </table>
                                                 </td>
@@ -261,15 +264,15 @@
                                     <td class="content-block"
                                         style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
                                         valign="top">
-                                       <a href="$alert["eagleJobLink"]" style="font-family:
'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color:
#999; text-decoration: underline; margin: 0;">
-                                           View Jobs on Eagle
+                                       <a href="$alert["eagleJobLink"]" style="font-family:
'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color:
#337ab7; text-decoration: underline; margin: 0;">
+                                           View Failed Jobs on Eagle
                                        </a>
                                     </td>
                                 </tr>
                                 <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;">
-                                    <td class="content-block" style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align:
top; margin: 0; padding: 0 0 20px;" valign="top">
+                                    <td class="content-block" style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align:
top; margin: 0; padding: 0 0 0px;" valign="top">
                                         <h2 class="aligncenter" style="font-family: 'Helvetica
Neue',Helvetica,Arial,'Lucida Grande',sans-serif; box-sizing: border-box; font-size: 18px;
color: #000; line-height: 1.2em; font-weight: 400; text-align: left; margin: 0 0 0;" align="center">
-                                            Top $alert["numTopUsers"] Finished* Job Users
+                                            Top $alert["numTopUsers"] Users by Finished*
Job
                                         </h2>
                                     </td>
                                 </tr>
@@ -298,20 +301,21 @@
                                                                 Ratio
                                                             </th>
                                                         </tr>
-                                                        #foreach($userItem in $alert["finishedJobUsers"].entrySet())
-                                                            <tr style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
-                                                                <td style="font-family:
'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align:
top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding:
5px 0;"
-                                                                    valign="top">
-                                                                    $userItem.key
-                                                                </td>
-                                                                <td style="text-align:center;
font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style:
solid; margin: 0; padding: 5px 0;"
-                                                                    valign="top">
-                                                                    $userItem.value
-                                                                </td>
-                                                                <td style="text-align:
right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style:
solid; margin: 0; padding: 5px 0;"
-                                                                    valign="top">$TODO_RATIO
-                                                                </td>
-                                                            </tr>
+                                                        #foreach($item in $alert["finishedJobUsers"])
+															<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; margin: 0;">
+																<td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color:
#eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+																	valign="top">
+                                                                    $item.key
+																</td>
+																<td style="text-align: center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color:
#eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+																	valign="top">
+                                                                    $item.numOfJobs
+																</td>
+																<td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color:
#eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+																	valign="top">
+                                                                    $item.ratio %
+																</td>
+															</tr>
                                                         #end
                                                     </table>
                                                 </td>
@@ -322,10 +326,10 @@
 
                                 <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;">
                                     <td class="content-block"
-                                        style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
+                                        style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 0px;"
                                         valign="top">
                                         <h2 class="aligncenter" style="font-family: 'Helvetica
Neue',Helvetica,Arial,'Lucida Grande',sans-serif; box-sizing: border-box; font-size: 18px;
color: #000; line-height: 1.2em; font-weight: 400; text-align: left; margin: 0 0 0;" align="center">
-                                        Top $alert["numTopUsers"] Succeeded Long Running*
Job Users
+                                        Top $alert["numTopUsers"] Users by Long Running*
Job
                                         </h2>
                                     </td>
                                 </tr>
@@ -355,19 +359,21 @@
                                                                 Ratio
                                                             </th>
                                                         </tr>
-                                                        #foreach($userItem in $alert["succeededJobUsers"].entrySet())
-                                                            <tr style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
-                                                                <td style="font-family:
'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align:
top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding:
5px 0;"
-                                                                    valign="top">
-                                                                    $userItem.key
-                                                                </td>
-                                                                <td style="text-align:
center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style:
solid; margin: 0; padding: 5px 0;"
-                                                                    valign="top">$userItem.value
-                                                                </td>
-                                                                <td style="text-align:
right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style:
solid; margin: 0; padding: 5px 0;"
-                                                                    valign="top">$TODO_RATIO
-                                                                </td>
-                                                            </tr>
+                                                        #foreach($item in $alert["succeededJobUsers"])
+															<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; margin: 0;">
+																<td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color:
#eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+																	valign="top">
+                                                                    $item.key
+																</td>
+																<td style="text-align: center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color:
#eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+																	valign="top">
+                                                                    $item.numOfJobs
+																</td>
+																<td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color:
#eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+																	valign="top">
+                                                                    $item.ratio %
+																</td>
+															</tr>
                                                         #end
                                                     </table>
                                                 </td>
@@ -380,8 +386,8 @@
                                         style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
                                         valign="top">
                                         * Notes:<br/>
-                                        1) Finished jobs include those <i>SUCCEEDED</i>,
<i>FAILED</i> and <i>KILLED</i>.<br/>
-                                        2) Long running jobs mean those duration over $alert["jobOvertimeLimit"]
hours.
+                                        1) Finished jobs include those <i>SUCCEEDED</i>,
<i>FAILED</i>, <i>KILLED</i>, etc.<br/>
+                                        2) Long running jobs mean those succeeded jobs with
duration over $alert["jobOvertimeLimit"] hours.
                                     </td>
                                 </tr>
                             </table>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a89275bf/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
index d80ae37..3b297ae 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
@@ -52,6 +52,7 @@ public class MRHistoryJobDailyReporterTest {
             server.stop();
         }
     }
+
     @Test
     public void test() throws Exception {
         MRHistoryJobDailyReporter reporter = new MRHistoryJobDailyReporter(config, null);
@@ -65,33 +66,26 @@ public class MRHistoryJobDailyReporterTest {
 
     private Map<String, Object> mockAlertData() {
         Map<String, Object> alertData = new HashMap<>();
-        List<MRHistoryJobDailyReporter.JobSummeryInfo> summeryInfos = new ArrayList<>();
-        MRHistoryJobDailyReporter.JobSummeryInfo summeryInfo1 = new MRHistoryJobDailyReporter.JobSummeryInfo();
-        summeryInfo1.status = "FAILED";
-        summeryInfo1.numOfJobs = 10;
-        summeryInfo1.ratio = 0.1;
-        MRHistoryJobDailyReporter.JobSummeryInfo summeryInfo2 = new MRHistoryJobDailyReporter.JobSummeryInfo();
-        summeryInfo2.status = "SUCCEEDED";
-        summeryInfo2.numOfJobs = 90;
-        summeryInfo2.ratio = 0.9;
-        summeryInfos.add(summeryInfo1);
-        summeryInfos.add(summeryInfo2);
+        List<JobSummaryInfo> summeryInfos = new ArrayList<>();
+        summeryInfos.add(buildJobSummaryInfo("FAILED", 8, 8));
+        summeryInfos.add(buildJobSummaryInfo("SUCCEEDED", 90, 89.9));
+        summeryInfos.add(buildJobSummaryInfo("KILLED", 2, 2));
         alertData.put(SUMMARY_INFO_KEY, summeryInfos);
 
-        Map<String,Double> failedJobUsers = new TreeMap<>();
-        failedJobUsers.put("alice", 100d);
-        failedJobUsers.put("bob", 97d);
+        List<JobSummaryInfo> failedJobUsers = new ArrayList<>();
+        failedJobUsers.add(buildJobSummaryInfo("alice", 100L, 98d));
+        failedJobUsers.add(buildJobSummaryInfo("bob", 97L, 2));
         alertData.put(FAILED_JOB_USERS_KEY, failedJobUsers);
 
-        Map<String,Double> succeededJobUsers = new TreeMap<>();
-        succeededJobUsers.put("alice1", 100d);
-        succeededJobUsers.put("bob1", 97d);
+        List<JobSummaryInfo> succeededJobUsers = new ArrayList<>();
+        succeededJobUsers.add(buildJobSummaryInfo("alice1", 100L, 98));
+        succeededJobUsers.add(buildJobSummaryInfo("bob1", 97, 2));
         alertData.put(SUCCEEDED_JOB_USERS_KEY, succeededJobUsers);
 
 
-        Map<String,Double> finishedJobUsers = new TreeMap<>();
-        finishedJobUsers.put("alice2", 100d);
-        finishedJobUsers.put("bob2", 97d);
+        List<JobSummaryInfo> finishedJobUsers = new ArrayList<>();
+        finishedJobUsers.add(buildJobSummaryInfo("alice2", 100L, 98));
+        finishedJobUsers.add(buildJobSummaryInfo("bob2", 97, 2));
         alertData.put(FINISHED_JOB_USERS_KEY, finishedJobUsers);
 
         alertData.put(ALERT_TITLE_KEY, "[TEST_CLUSTER] Daily Job Report");
@@ -104,4 +98,13 @@ public class MRHistoryJobDailyReporterTest {
         alertData.put(EAGLE_JOB_LINK_KEY, "http://localhost:9090/#/site/sandbox/jpm/statistics");
         return alertData;
     }
+
+    private JobSummaryInfo buildJobSummaryInfo(String key, long number, double ratio) {
+        JobSummaryInfo jobSummaryInfo = new JobSummaryInfo();
+        jobSummaryInfo.numOfJobs = number;
+        jobSummaryInfo.ratio = ratio;
+        jobSummaryInfo.key = key;
+        return jobSummaryInfo;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a89275bf/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java
b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java
index 42b6584..28e6bb3 100644
--- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java
+++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java
@@ -59,30 +59,32 @@ public class MRJobExecutionResource {
                                                     @QueryParam("filterIfMissing") boolean
filterIfMissing,
                                                     @QueryParam("parallel") int parallel,
                                                     @QueryParam("metricName") String metricName,
-                                                    @QueryParam("verbose") Boolean verbose)
{
+                                                    @QueryParam("verbose") Boolean verbose)
throws ParseException {
         GenericServiceAPIResponseEntity response = new GenericServiceAPIResponseEntity();
 
         List<TaggedLogAPIEntity> jobs = new ArrayList<>();
-        List<TaggedLogAPIEntity> finishedJobs = new ArrayList<>();
+        List<JobExecutionAPIEntity> finishedJobs = new ArrayList<>();
         Set<String> jobIds = new HashSet<>();
         final Map<String, Object> meta = new HashMap<>();
         StopWatch stopWatch = new StopWatch();
 
         stopWatch.start();
         String jobQuery = String.format(query, Constants.JPA_JOB_EXECUTION_SERVICE_NAME);
-        GenericServiceAPIResponseEntity<TaggedLogAPIEntity> res =
-            resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg,
timeSeries, intervalmin,
-                top, filterIfMissing, parallel, metricName, verbose);
+        GenericServiceAPIResponseEntity<JobExecutionAPIEntity> res =
+            resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg,
timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose);
         if (res.isSuccess() && res.getObj() != null) {
-            for (TaggedLogAPIEntity o : res.getObj()) {
-                finishedJobs.add(o);
-                jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString()));
+            long maxFinishedTime = DateTimeUtil.humanDateToSeconds(endTime) * DateTimeUtil.ONESECOND;
+            for (JobExecutionAPIEntity o : res.getObj()) {
+                if (o.getEndTime() <= maxFinishedTime) {
+                    finishedJobs.add(o);
+                    jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString()));
+                }
             }
             jobQuery = String.format(query, Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME);
-            res = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg,
timeSeries, intervalmin,
-                top, filterIfMissing, parallel, metricName, verbose);
-            if (res.isSuccess() && res.getObj() != null) {
-                for (TaggedLogAPIEntity o : res.getObj()) {
+            GenericServiceAPIResponseEntity<org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity>
runningRes =
+                    resource.search(jobQuery, startTime, endTime, pageSize, startRowkey,
treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose);
+            if (runningRes.isSuccess() && runningRes.getObj() != null) {
+                for (org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity o : runningRes.getObj())
{
                     String key = o.getTags().get(MRJobTagName.JOB_ID.toString());
                     if (!ResourceUtils.isDuplicate(jobIds, key)) {
                         jobs.add(o);



Mime
View raw message