eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jinh...@apache.org
Subject incubator-eagle git commit: [EAGLE-784] application health check support notification
Date Fri, 18 Nov 2016 02:31:46 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master d6e78bec2 -> 3005c4be8


[EAGLE-784] application health check support notification

Author: wujinhu <wujinhu920@126.com>

Closes #666 from wujinhu/EAGLE-784.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/3005c4be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/3005c4be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/3005c4be

Branch: refs/heads/master
Commit: 3005c4be8f16f71839fa9358c8c10583a37821a6
Parents: d6e78be
Author: wujinhu <wujinhu920@126.com>
Authored: Fri Nov 18 10:31:38 2016 +0800
Committer: wujinhu <wujinhu920@126.com>
Committed: Fri Nov 18 10:31:38 2016 +0800

----------------------------------------------------------------------
 .../ApplicationHealthCheckPublisher.java        |  24 ++++
 .../ApplicationHealthCheckEmailPublisher.java   | 129 +++++++++++++++++++
 .../impl/ApplicationHealthCheckServiceImpl.java |  39 +++++-
 .../src/main/resources/HealthCheckTemplate.vm   |  30 +++++
 .../MRHistoryJobApplicationHealthCheck.java     |   3 +-
 .../src/main/resources/application.conf         |  13 ++
 6 files changed, 236 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3005c4be/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
new file mode 100644
index 0000000..9469521
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.app.service;
+
+import com.codahale.metrics.health.HealthCheck;
+
+public interface ApplicationHealthCheckPublisher {
+    void onUnHealthApplication(String appId, HealthCheck.Result result);
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3005c4be/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
new file mode 100644
index 0000000..033d326
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.app.service.impl;
+
+import com.codahale.metrics.health.HealthCheck;
+import com.typesafe.config.Config;
+import org.apache.eagle.alert.engine.publisher.email.AlertEmailConstants;
+import org.apache.eagle.alert.engine.publisher.email.EagleMailClient;
+import org.apache.eagle.app.service.ApplicationHealthCheckPublisher;
+import org.apache.velocity.VelocityContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+public class ApplicationHealthCheckEmailPublisher implements ApplicationHealthCheckPublisher
{
+    private static final Logger LOG = LoggerFactory.getLogger(ApplicationHealthCheckEmailPublisher.class);
+    private static final int MAX_RETRY_COUNT = 3;
+    private static final String CONF_MAIL_RECIPIENTS = "mail.smtp.recipients";
+    private static final String CONF_MAIL_SENDER = "mail.smtp.sender";
+    private static final String CONF_MAIL_SUBJECT = "mail.smtp.subject";
+    private static final String CONF_MAIL_CC = "mail.smtp.cc";
+    private static final String CONF_MAIL_TEMPLATE = "mail.smtp.template";
+    private static final String UNHEALTHY_CONTEXT = "unHealthyContext";
+
+    private Config config;
+
+    public ApplicationHealthCheckEmailPublisher(Config config) {
+        this.config = config;
+    }
+
+    @Override
+    public void onUnHealthApplication(String appId, HealthCheck.Result result) {
+        Properties properties = parseMailClientConfig();
+        if (properties == null) {
+            return;
+        }
+
+        int count = 0;
+        boolean success = false;
+        while (count++ < MAX_RETRY_COUNT && !success) {
+            LOG.info("Sending email, tried: " + count + ", max: " + MAX_RETRY_COUNT);
+            try {
+                String recipients = config.getString(CONF_MAIL_RECIPIENTS);
+                if (recipients == null || recipients.equals("")) {
+                    LOG.error("Recipients is null, skip sending emails ");
+                    return;
+                }
+
+                final VelocityContext context = new VelocityContext();
+                Map<String, Object> unHealthyContext = new HashMap<>();
+                unHealthyContext.put("appId", appId);
+                unHealthyContext.put("unHealthyMessage", result.getMessage());
+                context.put(UNHEALTHY_CONTEXT, unHealthyContext);
+
+                EagleMailClient client = new EagleMailClient(properties);
+                success = client.send(config.getString(CONF_MAIL_SENDER),
+                        recipients,
+                        config.hasPath(CONF_MAIL_CC) ? config.getString(CONF_MAIL_CC) : null,
+                        config.getString(CONF_MAIL_SUBJECT) + ": " + appId,
+                        config.getString(CONF_MAIL_TEMPLATE),
+                        context,
+                        null);
+
+                LOG.info("Success of sending email: " + success);
+                if (!success && count < MAX_RETRY_COUNT) {
+                    LOG.info("Sleep for a while before retrying");
+                    Thread.sleep(10 * 1000);
+                }
+            } catch (Exception e) {
+                LOG.warn("Sending mail exception", e);
+            }
+        }
+        if (success) {
+            LOG.info("Successfully send unhealthy email of application {}", appId);
+        } else {
+            LOG.warn("Fail sending unhealthy email of application {} after tries {} times",
appId, MAX_RETRY_COUNT);
+        }
+    }
+
+    private Properties parseMailClientConfig() {
+        Properties props = new Properties();
+
+        String mailHost = config.getString(AlertEmailConstants.CONF_MAIL_HOST);
+        int mailPort = config.getInt(AlertEmailConstants.CONF_MAIL_PORT);
+        if (mailHost == null || mailPort == 0 || mailHost.isEmpty()) {
+            LOG.warn("SMTP server is unset, will exit");
+            return null;
+        }
+        props.put(AlertEmailConstants.CONF_MAIL_HOST, mailHost);
+        props.put(AlertEmailConstants.CONF_MAIL_PORT, mailPort);
+
+        Boolean smtpAuth = config.hasPath(AlertEmailConstants.CONF_MAIL_AUTH) &&
config.getBoolean(AlertEmailConstants.CONF_MAIL_AUTH);
+        props.put(AlertEmailConstants.CONF_MAIL_AUTH, smtpAuth);
+        if (smtpAuth) {
+            props.put(AlertEmailConstants.CONF_AUTH_USER, config.getString(AlertEmailConstants.CONF_AUTH_USER));
+            props.put(AlertEmailConstants.CONF_AUTH_PASSWORD, config.getString(AlertEmailConstants.CONF_AUTH_PASSWORD));
+        }
+
+        String smtpConn = config.hasPath(AlertEmailConstants.CONF_MAIL_CONN) ? config.getString(AlertEmailConstants.CONF_MAIL_CONN)
: AlertEmailConstants.CONN_PLAINTEXT;
+        if (smtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_TLS)) {
+            props.put("mail.smtp.starttls.enable", "true");
+        }
+        if (smtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_SSL)) {
+            props.put("mail.smtp.socketFactory.port", "465");
+            props.put("mail.smtp.socketFactory.class",
+                    "javax.net.ssl.SSLSocketFactory");
+        }
+        props.put(AlertEmailConstants.CONF_MAIL_DEBUG, config.hasPath(AlertEmailConstants.CONF_MAIL_DEBUG)
&& config.getBoolean(AlertEmailConstants.CONF_MAIL_DEBUG));
+        return props;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3005c4be/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
index 45903e3..9d98b2f 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
@@ -22,6 +22,7 @@ import com.google.inject.Inject;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import io.dropwizard.setup.Environment;
+import org.apache.eagle.app.service.ApplicationHealthCheckPublisher;
 import org.apache.eagle.app.service.ApplicationHealthCheckService;
 import org.apache.eagle.app.service.ApplicationProviderService;
 import org.apache.eagle.app.spi.ApplicationProvider;
@@ -30,6 +31,7 @@ import org.apache.eagle.metadata.service.ApplicationEntityService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.lang.reflect.Constructor;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -40,12 +42,18 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
 
     private final ApplicationProviderService applicationProviderService;
     private final ApplicationEntityService applicationEntityService;
+    private ApplicationHealthCheckPublisher applicationHealthCheckPublisher;
     private final Config config;
     private Environment environment;
     private Map<String, HealthCheck> appHealthChecks = new HashMap<>();
     private final Object lock = new Object();
     private int initialDelay = 10;
-    private int period = 10;
+    private int period = 300;
+
+    private static final String HEALTH_INITIAL_DELAY_PATH = "application.healthCheck.initialDelay";
+    private static final String HEALTH_PERIOD_PATH = "application.healthCheck.period";
+    private static final String HEALTH_PUBLISHER_PATH = "application.healthCheck.publisher";
+    private static final String HEALTH_PUBLISHER_IMPL_PATH = "application.healthCheck.publisher.publisherImpl";
 
     @Inject
     public ApplicationHealthCheckServiceImpl(ApplicationProviderService applicationProviderService,
@@ -54,6 +62,28 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
         this.applicationProviderService = applicationProviderService;
         this.applicationEntityService = applicationEntityService;
         this.config = config;
+        if (this.config.hasPath(HEALTH_INITIAL_DELAY_PATH)) {
+            this.initialDelay = this.config.getInt(HEALTH_INITIAL_DELAY_PATH);
+        }
+
+        if (this.config.hasPath(HEALTH_PERIOD_PATH)) {
+            this.period = this.config.getInt(HEALTH_PERIOD_PATH);
+        }
+
+        this.applicationHealthCheckPublisher = null;
+        if (this.config.hasPath(HEALTH_PUBLISHER_PATH)) {
+            try {
+                String className = this.config.getString(HEALTH_PUBLISHER_IMPL_PATH);
+                Class<?> clz;
+                clz = Thread.currentThread().getContextClassLoader().loadClass(className);
+                if (ApplicationHealthCheckPublisher.class.isAssignableFrom(clz)) {
+                    Constructor<?> cotr = clz.getConstructor(Config.class);
+                    this.applicationHealthCheckPublisher = (ApplicationHealthCheckPublisher)cotr.newInstance(this.config.getConfig(HEALTH_PUBLISHER_PATH));
+                }
+            } catch (Exception e) {
+                LOG.warn("exception found when create ApplicationHealthCheckPublisher instance
{}", e.getCause());
+            }
+        }
     }
 
     @Override
@@ -113,6 +143,13 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
                     LOG.info("application {} is healthy", appId);
                 } else {
                     LOG.warn("application {} is not healthy, {}", appId, result.getMessage(),
result.getError());
+                    if (this.applicationHealthCheckPublisher != null) {
+                        try {
+                            this.applicationHealthCheckPublisher.onUnHealthApplication(appId,
result);
+                        } catch (Exception e) {
+                            LOG.warn("failed to send email for unhealthy application {}",
appId, e.getCause());
+                        }
+                    }
                 }
             }
         }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3005c4be/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
b/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
new file mode 100644
index 0000000..b368458
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
@@ -0,0 +1,30 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+    <meta name="viewport" content="width=device-width"/>
+</head>
+<body>
+    #set ( $elem = $unHealthyContext )
+
+<p><b>Message: </b>$elem["appId"] is delayed</p>
+<p><b>Detail: </b>$elem["unHealthyMessage"]</p>
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3005c4be/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
index dcbcaf7..ab078f5 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
@@ -19,6 +19,7 @@ package org.apache.eagle.jpm.mr.history;
 
 import com.codahale.metrics.health.HealthCheck;
 import com.typesafe.config.Config;
+import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.eagle.jpm.util.Constants;
 import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
 import org.apache.eagle.service.client.IEagleServiceClient;
@@ -79,7 +80,7 @@ public class MRHistoryJobApplicationHealthCheck extends HealthCheck {
                 return Result.healthy();
             }
         } catch (Exception e) {
-            return Result.unhealthy(e);
+            return Result.unhealthy(ExceptionUtils.getStackTrace(e.getCause()));
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3005c4be/eagle-server/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-server/src/main/resources/application.conf b/eagle-server/src/main/resources/application.conf
index 59df87b..db973c8 100644
--- a/eagle-server/src/main/resources/application.conf
+++ b/eagle-server/src/main/resources/application.conf
@@ -96,6 +96,19 @@ application {
     initialDelay: 10
     period: 10
   }
+  healthCheck {
+    initialDelay = 30
+    period = 60
+    publisher {
+      publisherImpl = org.apache.eagle.app.service.impl.ApplicationHealthCheckEmailPublisher
+      mail.smtp.host = "mail.host.com"
+      mail.smtp.port = 25
+      mail.smtp.recipients = "someone@email.com"
+      mail.smtp.sender = "someone@email.com"
+      mail.smtp.subject = "Eagle Application is unhealthy"
+      mail.smtp.template = "HealthCheckTemplate.vm"
+    }
+  }
 }
 
 # ---------------------------------------------


Mime
View raw message