eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jinh...@apache.org
Subject incubator-eagle git commit: [MINOR] health check optimize
Date Wed, 23 Nov 2016 12:09:39 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 8d7f81e1c -> 929a21da7


[MINOR] health check optimize

Author: wujinhu <wujinhu920@126.com>

Closes #677 from wujinhu/healthCheckOptimize.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/929a21da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/929a21da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/929a21da

Branch: refs/heads/master
Commit: 929a21da7182986dc62db953639e27cc983f6f23
Parents: 8d7f81e
Author: wujinhu <wujinhu920@126.com>
Authored: Wed Nov 23 20:09:32 2016 +0800
Committer: wujinhu <wujinhu920@126.com>
Committed: Wed Nov 23 20:09:32 2016 +0800

----------------------------------------------------------------------
 .../ApplicationHealthCheckPublisher.java        |  4 +-
 .../impl/ApplicationHealthCheckBase.java        | 12 ++++
 .../ApplicationHealthCheckEmailPublisher.java   | 20 ++++--
 .../impl/ApplicationHealthCheckServiceImpl.java | 68 +++++++++++++++-----
 .../eagle/app/spi/ApplicationProvider.java      |  1 -
 .../src/main/resources/HealthCheckTemplate.vm   | 18 +++++-
 ...adoopQueueRunningApplicationHealthCheck.java | 13 ++--
 .../MRHistoryJobApplicationHealthCheck.java     | 10 +--
 .../JobEntityCreationEagleServiceListener.java  |  5 +-
 .../SparkHistoryJobApplicationHealthCheck.java  | 12 ++--
 .../src/main/resources/application.conf         |  3 +-
 .../TopologyCheckApplicationHealthCheck.java    | 10 +--
 12 files changed, 127 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
index 9469521..4a8d2e7 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
@@ -19,6 +19,8 @@ package org.apache.eagle.app.service;
 
 import com.codahale.metrics.health.HealthCheck;
 
+import java.util.Map;
+
 public interface ApplicationHealthCheckPublisher {
-    void onUnHealthApplication(String appId, HealthCheck.Result result);
+    void onUnHealthApplication(Map<String, HealthCheck.Result> results);
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
index 104e76d..75b7c2d 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
@@ -23,6 +23,9 @@ import com.typesafe.config.Config;
 import org.apache.eagle.metadata.model.ApplicationEntity;
 import org.apache.eagle.metadata.service.ApplicationEntityService;
 
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
 public abstract class ApplicationHealthCheckBase extends HealthCheck {
     private static final String APP_ID_PATH = "appId";
     protected static final long DEFAULT_MAX_DELAY_TIME = 2 * 60 * 60 * 1000L;
@@ -41,4 +44,13 @@ public abstract class ApplicationHealthCheckBase extends HealthCheck {
         ApplicationEntity applicationEntity = applicationEntityService.getByUUIDOrAppId(null,
config.getString(APP_ID_PATH));
         return applicationEntity.getStatus();
     }
+
+    protected String printMessages(String ... messages) {
+        StringWriter sw = new StringWriter();
+        PrintWriter pw = new PrintWriter(sw, true);
+        for (int i = 0; i < messages.length; i++) {
+            pw.println(messages[i]);
+        }
+        return sw.getBuffer().toString();
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
index 033d326..f3d543a 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
@@ -39,6 +39,9 @@ public class ApplicationHealthCheckEmailPublisher implements ApplicationHealthCh
     private static final String CONF_MAIL_CC = "mail.smtp.cc";
     private static final String CONF_MAIL_TEMPLATE = "mail.smtp.template";
     private static final String UNHEALTHY_CONTEXT = "unHealthyContext";
+    private static final Integer HEALTH_CHECK_PORT = 9091;
+    private static final String SERVICE_HOST = "host";
+    private static final String SERVICE_PORT = "port";
 
     private Config config;
 
@@ -47,7 +50,7 @@ public class ApplicationHealthCheckEmailPublisher implements ApplicationHealthCh
     }
 
     @Override
-    public void onUnHealthApplication(String appId, HealthCheck.Result result) {
+    public void onUnHealthApplication(Map<String, HealthCheck.Result> results) {
         Properties properties = parseMailClientConfig();
         if (properties == null) {
             return;
@@ -65,16 +68,21 @@ public class ApplicationHealthCheckEmailPublisher implements ApplicationHealthCh
                 }
 
                 final VelocityContext context = new VelocityContext();
+                Map<String, String> appMsgs = new HashMap<>();
+                for (String appId : results.keySet()) {
+                    appMsgs.put(appId, results.get(appId).getMessage());
+                }
                 Map<String, Object> unHealthyContext = new HashMap<>();
-                unHealthyContext.put("appId", appId);
-                unHealthyContext.put("unHealthyMessage", result.getMessage());
+                unHealthyContext.put("appMsgs", appMsgs);
+                unHealthyContext.put("appMgmtUrl", "http://" + config.getString(SERVICE_HOST)
+ ":" + config.getInt(SERVICE_PORT) + "/#/integration/site");
+                unHealthyContext.put("healthCheckUrl", "http://" + config.getString(SERVICE_HOST)
+ ":" + HEALTH_CHECK_PORT + "/healthcheck");
                 context.put(UNHEALTHY_CONTEXT, unHealthyContext);
 
                 EagleMailClient client = new EagleMailClient(properties);
                 success = client.send(config.getString(CONF_MAIL_SENDER),
                         recipients,
                         config.hasPath(CONF_MAIL_CC) ? config.getString(CONF_MAIL_CC) : null,
-                        config.getString(CONF_MAIL_SUBJECT) + ": " + appId,
+                        config.getString(CONF_MAIL_SUBJECT),
                         config.getString(CONF_MAIL_TEMPLATE),
                         context,
                         null);
@@ -89,9 +97,9 @@ public class ApplicationHealthCheckEmailPublisher implements ApplicationHealthCh
             }
         }
         if (success) {
-            LOG.info("Successfully send unhealthy email of application {}", appId);
+            LOG.info("Successfully send unhealthy email");
         } else {
-            LOG.warn("Fail sending unhealthy email of application {} after tries {} times",
appId, MAX_RETRY_COUNT);
+            LOG.warn("Fail sending unhealthy email after tries {} times", MAX_RETRY_COUNT);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
index 4aea963..b292fa1 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
@@ -33,9 +33,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.lang.reflect.Constructor;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 
 public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckService {
@@ -55,6 +53,13 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
     private static final String HEALTH_PERIOD_PATH = "application.healthCheck.period";
     private static final String HEALTH_PUBLISHER_PATH = "application.healthCheck.publisher";
     private static final String HEALTH_PUBLISHER_IMPL_PATH = "application.healthCheck.publisher.publisherImpl";
+    private static final String HEALTH_CHECK_DAILY_SEND_HOUR_PATH = "application.healthCheck.publisher.dailySendHour";
+    private static final String SERVICE_PATH = "service";
+    private static final String TIMEZONE_PATH = "service.timezone";
+    private static final String HEALTHY = "OK";
+    private boolean hasSendDaily = false;
+
+    private TimeZone timeZone;
 
     @Inject
     private Injector currentInjector;
@@ -82,7 +87,8 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
                 clz = Thread.currentThread().getContextClassLoader().loadClass(className);
                 if (ApplicationHealthCheckPublisher.class.isAssignableFrom(clz)) {
                     Constructor<?> cotr = clz.getConstructor(Config.class);
-                    this.applicationHealthCheckPublisher = (ApplicationHealthCheckPublisher)cotr.newInstance(this.config.getConfig(HEALTH_PUBLISHER_PATH));
+                    this.applicationHealthCheckPublisher = (ApplicationHealthCheckPublisher)cotr.newInstance(
+                            this.config.getConfig(HEALTH_PUBLISHER_PATH).withFallback(this.config.getConfig(SERVICE_PATH)));
                 }
             } catch (Exception e) {
                 LOG.warn("exception found when create ApplicationHealthCheckPublisher instance
{}", e.getCause());
@@ -94,6 +100,7 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
     public void init(Environment environment) {
         this.environment = environment;
         registerAll();
+        this.timeZone = TimeZone.getTimeZone(config.getString(TIMEZONE_PATH));
     }
 
     private void registerAll() {
@@ -140,23 +147,54 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
     protected void runOneIteration() throws Exception {
         LOG.info("start application health check");
         registerAll();
+
+        boolean isDaily = false;
+        int dailySendHour = config.getInt(HEALTH_CHECK_DAILY_SEND_HOUR_PATH);
+
+        GregorianCalendar cal = new GregorianCalendar(timeZone);
+        if (cal.get(Calendar.HOUR_OF_DAY) % dailySendHour == 0 && !hasSendDaily)
{
+            isDaily = true;
+        }
+
+        if (cal.get(Calendar.HOUR_OF_DAY) % dailySendHour != 0) {
+            hasSendDaily = false;
+        }
+        Map<String, HealthCheck> copyAppHealthChecks = new HashMap<>();
         synchronized (lock) {
             for (String appId : appHealthChecks.keySet()) {
-                HealthCheck.Result result = appHealthChecks.get(appId).execute();
-                if (result.isHealthy()) {
-                    LOG.info("application {} is healthy", appId);
-                } else {
-                    LOG.warn("application {} is not healthy, {}", appId, result.getMessage(),
result.getError());
-                    if (this.applicationHealthCheckPublisher != null) {
-                        try {
-                            this.applicationHealthCheckPublisher.onUnHealthApplication(appId,
result);
-                        } catch (Exception e) {
-                            LOG.warn("failed to send email for unhealthy application {}",
appId, e.getCause());
-                        }
+                copyAppHealthChecks.put(appId, appHealthChecks.get(appId));
+            }
+        }
+
+        Map<String, HealthCheck.Result> results = new HashMap<>();
+        for (String appId : copyAppHealthChecks.keySet()) {
+            HealthCheck.Result result = copyAppHealthChecks.get(appId).execute();
+            if (result.isHealthy()) {
+                if (isDaily) {
+                    if (result.getMessage() == null || result.getMessage().isEmpty()) {
+                        results.put(appId, HealthCheck.Result.healthy(HEALTHY));
+                    } else {
+                        results.put(appId, result);
                     }
                 }
+                LOG.info("application {} is healthy", appId);
+            } else {
+                results.put(appId, result);
+                LOG.warn("application {} is not healthy, {}", appId, result.getMessage(),
result.getError());
+            }
+        }
+
+        if (this.applicationHealthCheckPublisher != null) {
+            try {
+                this.applicationHealthCheckPublisher.onUnHealthApplication(results);
+                if (isDaily) {
+                    hasSendDaily = true;
+                }
+            } catch (Exception e) {
+                LOG.warn("failed to send email for unhealthy applications", e);
             }
         }
+
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java
index d9c1eff..143e026 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java
@@ -23,7 +23,6 @@ import org.apache.eagle.app.Application;
 import org.apache.eagle.app.service.ApplicationListener;
 import org.apache.eagle.common.module.ModuleRegistry;
 import org.apache.eagle.metadata.model.ApplicationDesc;
-import org.apache.eagle.metadata.service.ApplicationEntityService;
 
 import java.lang.reflect.ParameterizedType;
 import java.util.Optional;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
b/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
index 51c1186..74f401a 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
@@ -23,8 +23,22 @@
 <body>
     #set ( $elem = $unHealthyContext )
 
-<p><b>Message: </b>$elem["appId"] has delayed</p>
-<p><b>Detail: </b>$elem["unHealthyMessage"]</p>
+
+<p><b>Health Check: </b>$elem["healthCheckUrl"]</p>
+<p><b>Appliaction Management: </b>$elem["appMgmtUrl"]</p>
+
+<table border="1">
+    <tr>
+        <th><b>Application ID</b></th>
+        <td><b>Health Check Message</b></td>
+    </tr>
+    #foreach($appId in ${elem["appMsgs"].keySet()})
+    <tr>
+        <th>$appId</th>
+        <td>${elem["appMsgs"].get($appId)}</td>
+    </tr>
+    #end
+</table>
 
 </body>
 </html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
index 02308c0..bc5b7c4 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
@@ -53,13 +53,14 @@ public class HadoopQueueRunningApplicationHealthCheck extends ApplicationHealthC
 
         client.getJerseyClient().setReadTimeout(60000);
 
+        String message = "";
         try {
             ApplicationEntity.Status status = getApplicationStatus();
             if (!status.toString().equals(ApplicationEntity.Status.RUNNING.toString())) {
-                String message = String.format("Application is not running, status is %s",
status.toString());
-                return Result.unhealthy(message);
+                message += String.format("Application is not RUNNING, status is %s. ", status.toString());
             }
 
+
             String query = String.format("%s[@site=\"%s\"]<@site>{max(timestamp)}",
                     Constants.GENERIC_METRIC_SERVICE,
                     hadoopQueueRunningAppConfig.eagleProps.site);
@@ -67,7 +68,7 @@ public class HadoopQueueRunningApplicationHealthCheck extends ApplicationHealthC
             GenericServiceAPIResponseEntity response = client
                     .search(query)
                     .metricName(HadoopClusterConstants.MetricName.HADOOP_CLUSTER_ALLOCATED_MEMORY)
-                    .startTime(System.currentTimeMillis() - 2 * 60 * 60000L)
+                    .startTime(System.currentTimeMillis() - 24 * 60 * 60000L)
                     .endTime(System.currentTimeMillis())
                     .pageSize(Integer.MAX_VALUE)
                     .send();
@@ -79,15 +80,15 @@ public class HadoopQueueRunningApplicationHealthCheck extends ApplicationHealthC
                 maxDelayTime = hadoopQueueRunningAppConfig.getConfig().getLong(MAX_DELAY_TIME_KEY);
             }
 
-            if (currentTimeStamp - currentProcessTimeStamp > maxDelayTime) {
-                String message = String.format("Current process time is %sms, delay %s hours",
+            if (!message.isEmpty() || currentTimeStamp - currentProcessTimeStamp > maxDelayTime)
{
+                message += String.format("Current process time is %sms, delay %s hours.",
                         currentProcessTimeStamp, (currentTimeStamp - currentProcessTimeStamp)
* 1.0 / 60000L / 60);
                 return Result.unhealthy(message);
             } else {
                 return Result.healthy();
             }
         } catch (Exception e) {
-            return Result.unhealthy(ExceptionUtils.getStackTrace(e));
+            return Result.unhealthy(printMessages(message, "An exception was caught when
fetch application current process time: ", ExceptionUtils.getStackTrace(e)));
         } finally {
             client.getJerseyClient().destroy();
             try {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
index 96d8672..20506c0 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
@@ -52,11 +52,11 @@ public class MRHistoryJobApplicationHealthCheck extends ApplicationHealthCheckBa
 
         client.getJerseyClient().setReadTimeout(eagleServiceConfig.readTimeoutSeconds * 1000);
 
+        String message = "";
         try {
             ApplicationEntity.Status status = getApplicationStatus();
             if (!status.toString().equals(ApplicationEntity.Status.RUNNING.toString())) {
-                String message = String.format("Application is not running, status is %s",
status.toString());
-                return Result.unhealthy(message);
+                message += String.format("Application is not RUNNING, status is %s. ", status.toString());
             }
 
             String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}",
@@ -78,15 +78,15 @@ public class MRHistoryJobApplicationHealthCheck extends ApplicationHealthCheckBa
                 maxDelayTime = mrHistoryJobConfig.getConfig().getLong(MAX_DELAY_TIME_KEY);
             }
 
-            if (currentTimeStamp - currentProcessTimeStamp > maxDelayTime) {
-                String message = String.format("Current process time is %sms, delay %s hours",
+            if (!message.isEmpty() || currentTimeStamp - currentProcessTimeStamp > maxDelayTime)
{
+                message += String.format("Current process time is %sms, delay %s hours.",
                         currentProcessTimeStamp, (currentTimeStamp - currentProcessTimeStamp)
* 1.0 / 60000L / 60);
                 return Result.unhealthy(message);
             } else {
                 return Result.healthy();
             }
         } catch (Exception e) {
-            return Result.unhealthy(ExceptionUtils.getStackTrace(e));
+            return Result.unhealthy(printMessages(message, "An exception was caught when
fetch application current process time: ", ExceptionUtils.getStackTrace(e)));
         } finally {
             client.getJerseyClient().destroy();
             try {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java
index 902590b..c802442 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationEagleServiceListener.java
@@ -27,6 +27,7 @@ import org.apache.eagle.jpm.mr.historyentity.*;
 import org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity;
 import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity;
 import org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity;
+import org.apache.eagle.jpm.util.Constants;
 import org.apache.eagle.jpm.util.MRJobTagName;
 import org.apache.eagle.log.entity.GenericMetricEntity;
 import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
@@ -107,7 +108,9 @@ public class JobEntityCreationEagleServiceListener implements HistoryJobEntityCr
                     ((JobExecutionAPIEntity) entity).getCurrentState());
 
                 metricEntities.addAll(jobExecutionMetricsCreationListener.generateMetrics((JobExecutionAPIEntity)entity));
-                emitFailedJob((JobExecutionAPIEntity)entity);
+                if (((JobExecutionAPIEntity)entity).getCurrentState().equals(Constants.JobState.FAILED.toString()))
{
+                    emitFailedJob((JobExecutionAPIEntity) entity);
+                }
             } else if (entity instanceof JobEventAPIEntity) {
                 jobEvents.add((JobEventAPIEntity) entity);
             } else if (entity instanceof TaskExecutionAPIEntity) {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
index bb6f390..fe6bd5a 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
@@ -52,11 +52,11 @@ public class SparkHistoryJobApplicationHealthCheck extends ApplicationHealthChec
 
         client.getJerseyClient().setReadTimeout(eagleServiceConfig.timeout * 1000);
 
+        String message = "";
         try {
             ApplicationEntity.Status status = getApplicationStatus();
             if (!status.toString().equals(ApplicationEntity.Status.RUNNING.toString())) {
-                String message = String.format("Application is not running, status is %s",
status.toString());
-                return Result.unhealthy(message);
+                message += String.format("Application is not RUNNING, status is %s. ", status.toString());
             }
 
             String query = String.format("%s[@site=\"%s\"]<@site>{max(endTime)}",
@@ -65,7 +65,7 @@ public class SparkHistoryJobApplicationHealthCheck extends ApplicationHealthChec
 
             GenericServiceAPIResponseEntity response = client
                     .search(query)
-                    .startTime(System.currentTimeMillis() - 12 * 60 * 60000L)
+                    .startTime(System.currentTimeMillis() - 24 * 60 * 60000L)
                     .endTime(System.currentTimeMillis())
                     .pageSize(10)
                     .send();
@@ -78,15 +78,15 @@ public class SparkHistoryJobApplicationHealthCheck extends ApplicationHealthChec
                 maxDelayTime = sparkHistoryJobAppConfig.getConfig().getLong(MAX_DELAY_TIME_KEY);
             }
 
-            if (currentTimeStamp - currentProcessTimeStamp > maxDelayTime * 3) {
-                String message = String.format("Current process time is %sms, delay %s hours",
+            if (!message.isEmpty() || currentTimeStamp - currentProcessTimeStamp > maxDelayTime
* 3) {
+                message += String.format("Current process time is %sms, delay %s hours.",
                         currentProcessTimeStamp, (currentTimeStamp - currentProcessTimeStamp)
* 1.0 / 60000L / 60);
                 return Result.unhealthy(message);
             } else {
                 return Result.healthy();
             }
         } catch (Exception e) {
-            return Result.unhealthy(ExceptionUtils.getStackTrace(e));
+            return Result.unhealthy(printMessages(message, "An exception was caught when
fetch application current process time: ", ExceptionUtils.getStackTrace(e)));
         } finally {
             client.getJerseyClient().destroy();
             try {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-server/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-server/src/main/resources/application.conf b/eagle-server/src/main/resources/application.conf
index db973c8..da0c7c9 100644
--- a/eagle-server/src/main/resources/application.conf
+++ b/eagle-server/src/main/resources/application.conf
@@ -101,11 +101,12 @@ application {
     period = 60
     publisher {
       publisherImpl = org.apache.eagle.app.service.impl.ApplicationHealthCheckEmailPublisher
+      dailySendHour = 11
       mail.smtp.host = "mail.host.com"
       mail.smtp.port = 25
       mail.smtp.recipients = "someone@email.com"
       mail.smtp.sender = "someone@email.com"
-      mail.smtp.subject = "Eagle Application is unhealthy"
+      mail.smtp.subject = "Eagle Application Health Check"
       mail.smtp.template = "HealthCheckTemplate.vm"
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
index 67669f8..4b740db 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
@@ -51,11 +51,11 @@ public class TopologyCheckApplicationHealthCheck extends ApplicationHealthCheckB
 
         client.getJerseyClient().setReadTimeout(topologyCheckAppConfig.getConfig().getInt("service.readTimeOutSeconds")
* 1000);
 
+        String message = "";
         try {
             ApplicationEntity.Status status = getApplicationStatus();
             if (!status.toString().equals(ApplicationEntity.Status.RUNNING.toString())) {
-                String message = String.format("Application is not running, status is %s",
status.toString());
-                return Result.unhealthy(message);
+                message += String.format("Application is not RUNNING, status is %s. ", status.toString());
             }
 
             String query = String.format("%s[@site=\"%s\"]<@site>{max(timestamp)}",
@@ -77,15 +77,15 @@ public class TopologyCheckApplicationHealthCheck extends ApplicationHealthCheckB
                 maxDelayTime = topologyCheckAppConfig.getConfig().getLong(MAX_DELAY_TIME_KEY);
             }
 
-            if (currentTimeStamp - currentProcessTimeStamp > maxDelayTime) {
-                String message = String.format("Current process time is %sms, delay %s hours",
+            if (!message.isEmpty() || currentTimeStamp - currentProcessTimeStamp > maxDelayTime)
{
+                message += String.format("Current process time is %sms, delay %s hours.",
                         currentProcessTimeStamp, (currentTimeStamp - currentProcessTimeStamp)
* 1.0 / 60000L / 60);
                 return Result.unhealthy(message);
             } else {
                 return Result.healthy();
             }
         } catch (Exception e) {
-            return Result.unhealthy(ExceptionUtils.getStackTrace(e));
+            return Result.unhealthy(printMessages(message, "An exception was caught when
fetch application current process time: ", ExceptionUtils.getStackTrace(e)));
         } finally {
             client.getJerseyClient().destroy();
             try {


Mime
View raw message