eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject [3/3] incubator-eagle git commit: [EAGLE-545] hdfs/bhase/yarn topology health check
Date Mon, 10 Oct 2016 03:26:03 GMT
[EAGLE-545] hdfs/bhase/yarn topology health check

https://issues.apache.org/jira/browse/EAGLE-545

The app aims to monitor those services with a master-slave structured topology and provides metrics at the HOST level. Specifically, this app will support the following services healthy check
* HDFS: namenode, datanode, journalnode
* HBASE: hmaster, regionservers
* YARN: resourcemanager, nodemanagers, historyserver

Author: Zhao, Qingwen <qingwzhao@ebay.com>
Author: Qingwen Zhao <qingwen220@gmail.com>

Closes #449 from qingwen220/topology.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/9e873770
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/9e873770
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/9e873770

Branch: refs/heads/master
Commit: 9e87377052314f559918fc384f667b60c74f4394
Parents: 8ba1b84
Author: Zhao, Qingwen <qingwen220@gmail.com>
Authored: Mon Oct 10 11:25:42 2016 +0800
Committer: Qingwen Zhao <qingwen220@gmail.com>
Committed: Mon Oct 10 11:25:42 2016 +0800

----------------------------------------------------------------------
 .../environment/impl/StormExecutionRuntime.java |   1 +
 eagle-core/eagle-app/eagle-app-utils/pom.xml    |  45 +++
 .../apache/eagle/app/utils/AppConstants.java    |  26 ++
 .../eagle/app/utils/HadoopSecurityUtil.java     |  48 ++++
 .../eagle/app/utils/PathResolverHelper.java     |  37 +++
 .../app/utils/connection/InputStreamUtils.java  |  69 +++++
 .../utils/connection/URLConnectionUtils.java    | 103 +++++++
 .../eagle/app/utils/ha/HAURLSelector.java       |  29 ++
 eagle-core/eagle-app/pom.xml                    |   1 +
 ...spark.history.SparkHistoryJobAppProvider.xml |   5 +
 eagle-topology-check/eagle-topology-app/pom.xml |  85 ++++++
 .../assembly/eagle-topology-check-assembly.xml  |  66 +++++
 .../apache/eagle/topology/TopologyCheckApp.java |  52 ++++
 .../eagle/topology/TopologyCheckAppConfig.java  | 145 ++++++++++
 .../eagle/topology/TopologyCheckAppMain.java    |  26 ++
 .../topology/TopologyCheckAppProvider.java      |  28 ++
 .../eagle/topology/TopologyCheckMessageId.java  |  56 ++++
 .../topology/extractor/TopologyCrawler.java     |  28 ++
 .../extractor/TopologyEntityParser.java         |  43 +++
 .../extractor/TopologyEntityParserResult.java   |  65 +++++
 .../extractor/TopologyExtractorFactory.java     |  76 +++++
 .../extractor/hbase/HbaseTopologyCrawler.java   |  68 +++++
 .../hbase/HbaseTopologyEntityParser.java        | 164 +++++++++++
 .../extractor/hdfs/HdfsTopologyCrawler.java     |  65 +++++
 .../hdfs/HdfsTopologyEntityParser.java          | 280 +++++++++++++++++++
 .../extractor/mr/MRTopologyCrawler.java         |  56 ++++
 .../extractor/mr/MRTopologyEntityParser.java    | 217 ++++++++++++++
 .../topology/extractor/mr/YarnNodeInfo.java     | 101 +++++++
 .../extractor/mr/YarnNodeInfoWrapper.java       |  38 +++
 .../topology/extractor/mr/YarnNodeInfos.java    |  41 +++
 .../topology/resolver/TopologyRackResolver.java |  28 ++
 .../impl/DefaultTopologyRackResolver.java       |  45 +++
 .../impl/IPMaskTopologyRackResolver.java        |  54 ++++
 .../topology/storm/TopologyCheckAppSpout.java   |  85 ++++++
 .../topology/storm/TopologyDataExtractor.java   | 109 ++++++++
 .../topology/storm/TopologyDataPersistBolt.java | 142 ++++++++++
 .../topology/utils/EntityBuilderHelper.java     |  62 ++++
 .../apache/eagle/topology/utils/JMXBean.java    |  36 +++
 .../eagle/topology/utils/JMXQueryHelper.java    |  86 ++++++
 .../utils/ServiceNotResponseException.java      |  62 ++++
 ....eagle.topology.TopologyCheckAppProvider.xml | 155 ++++++++++
 ...org.apache.eagle.app.spi.ApplicationProvider |  19 ++
 .../src/main/resources/application.conf         |  60 ++++
 .../src/main/resources/log4j.properties         |  35 +++
 .../topology/TestHbaseTopologyCrawler.java      |  40 +++
 .../eagle/topology/TestHdfsTopologyCrawler.java |  40 +++
 .../apache/eagle/topology/TestJsonParser.java   |  80 ++++++
 .../eagle/topology/TestMRTopologyCrawler.java   |  40 +++
 .../TestNetMaskTopologyRackResolver.java        |  67 +++++
 .../src/test/resources/application.conf         |  59 ++++
 .../src/test/resources/log4j.properties         |  35 +++
 .../eagle-topology-entity/pom.xml               |  39 +++
 .../eagle/topology/TopologyConstants.java       |  96 +++++++
 .../entity/HBaseServiceTopologyAPIEntity.java   |  99 +++++++
 .../entity/HdfsServiceTopologyAPIEntity.java    | 101 +++++++
 .../entity/JournalNodeServiceAPIEntity.java     |  66 +++++
 .../entity/MRServiceTopologyAPIEntity.java      |  81 ++++++
 .../topology/entity/TopologyBaseAPIEntity.java  |  25 ++
 .../entity/TopologyEntityRepository.java        |  30 ++
 eagle-topology-check/pom.xml                    |  54 ++++
 pom.xml                                         |   1 +
 61 files changed, 3995 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
index 702a3e6..7817c73 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
@@ -16,6 +16,7 @@
  */
 package org.apache.eagle.app.environment.impl;
 
+import backtype.storm.utils.Utils;
 import org.apache.eagle.app.Application;
 import org.apache.eagle.app.environment.ExecutionRuntime;
 import org.apache.eagle.app.environment.ExecutionRuntimeProvider;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-core/eagle-app/eagle-app-utils/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-utils/pom.xml b/eagle-core/eagle-app/eagle-app-utils/pom.xml
new file mode 100644
index 0000000..1b3183d
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-utils/pom.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~  Licensed to the Apache Software Foundation (ASF) under one
+  ~  or more contributor license agreements.  See the NOTICE file
+  ~  distributed with this work for additional information
+  ~  regarding copyright ownership.  The ASF licenses this file
+  ~  to you under the Apache License, Version 2.0 (the
+  ~  "License"); you may not use this file except in compliance
+  ~  with the License.  You may obtain a copy of the License at
+  ~
+  ~      http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>eagle-app-parent</artifactId>
+        <groupId>org.apache.eagle</groupId>
+        <version>0.5.0-incubating-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>eagle-app-utils</artifactId>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/AppConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/AppConstants.java b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/AppConstants.java
new file mode 100644
index 0000000..65a7b44
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/AppConstants.java
@@ -0,0 +1,26 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.eagle.app.utils;
+
+public class AppConstants {
+
+    public enum CompressionType {
+        GZIP, NONE
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/HadoopSecurityUtil.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/HadoopSecurityUtil.java b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/HadoopSecurityUtil.java
new file mode 100644
index 0000000..571973c
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/HadoopSecurityUtil.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.app.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.io.IOException;
+
+/**
+ * This class provides util methods for Eagle connector communicating
+ * with secured cluster.
+ */
+public class HadoopSecurityUtil {
+
+    public static final String EAGLE_KEYTAB_FILE_KEY = "eagle.keytab.file";
+    public static final String EAGLE_PRINCIPAL_KEY = "eagle.kerberos.principal";
+
+    public static void login(Configuration kConfig) throws IOException {
+        String keytab = kConfig.get(EAGLE_KEYTAB_FILE_KEY);
+        String principal = kConfig.get(EAGLE_PRINCIPAL_KEY);
+        if ( keytab == null ||  principal == null || keytab.isEmpty() || principal.isEmpty()) {
+            return;
+        }
+
+        kConfig.setBoolean("hadoop.security.authorization", true);
+        kConfig.set("hadoop.security.authentication", "kerberos");
+        UserGroupInformation.setConfiguration(kConfig);
+        UserGroupInformation.loginUserFromKeytab(kConfig.get(EAGLE_PRINCIPAL_KEY), kConfig.get(EAGLE_KEYTAB_FILE_KEY));
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/PathResolverHelper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/PathResolverHelper.java b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/PathResolverHelper.java
new file mode 100644
index 0000000..81b0834
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/PathResolverHelper.java
@@ -0,0 +1,37 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.eagle.app.utils;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class PathResolverHelper {
+
+    public static String buildUrlPath(String baseUrl, String childUrl) {
+        try {
+            URI oldUri = new URI(baseUrl);
+            URI resolved = oldUri.resolve(childUrl);
+            return resolved.toString();
+        } catch (URISyntaxException e) {
+            e.printStackTrace();
+            return baseUrl;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/connection/InputStreamUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/connection/InputStreamUtils.java b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/connection/InputStreamUtils.java
new file mode 100644
index 0000000..7b9479f
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/connection/InputStreamUtils.java
@@ -0,0 +1,69 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.eagle.app.utils.connection;
+
+import org.apache.eagle.app.utils.AppConstants;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.zip.GZIPInputStream;
+
+public class InputStreamUtils {
+
+    private static final int CONNECTION_TIMEOUT = 10 * 1000;
+    private static final int READ_TIMEOUT = 5 * 60 * 1000;
+    private static final String GZIP_HTTP_HEADER = "Accept-Encoding";
+    private static final String GZIP_COMPRESSION = "gzip";
+
+    private static InputStream openGZIPInputStream(URL url, String auth, int timeout) throws IOException {
+        final URLConnection connection = url.openConnection();
+        connection.setConnectTimeout(CONNECTION_TIMEOUT);
+        connection.setReadTimeout(timeout);
+        connection.addRequestProperty(GZIP_HTTP_HEADER, GZIP_COMPRESSION);
+        if (null != auth) {
+            connection.setRequestProperty("Authorization", auth);
+        }
+        return new GZIPInputStream(connection.getInputStream());
+    }
+
+    private static InputStream openInputStream(URL url, String auth, int timeout) throws IOException {
+        URLConnection connection = url.openConnection();
+        connection.setConnectTimeout(timeout);
+        if (null != auth) {
+            connection.setRequestProperty("Authorization", auth);
+        }
+
+        return connection.getInputStream();
+    }
+
+    public static InputStream getInputStream(String urlString, String auth, AppConstants.CompressionType compressionType, int timeout) throws Exception {
+        final URL url = URLConnectionUtils.getUrl(urlString);
+        if (compressionType.equals(AppConstants.CompressionType.GZIP)) {
+            return openGZIPInputStream(url, auth, timeout);
+        } else { // CompressionType.NONE
+            return openInputStream(url, auth, timeout);
+        }
+    }
+
+    public static InputStream getInputStream(String urlString, String auth, AppConstants.CompressionType compressionType) throws Exception {
+        return getInputStream(urlString, auth, compressionType, READ_TIMEOUT);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/connection/URLConnectionUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/connection/URLConnectionUtils.java b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/connection/URLConnectionUtils.java
new file mode 100644
index 0000000..cb02852
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/connection/URLConnectionUtils.java
@@ -0,0 +1,103 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.eagle.app.utils.connection;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.*;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+
+public final class URLConnectionUtils {
+    //TODO: change some public method to private
+    private static final Logger LOG = LoggerFactory.getLogger(URLConnectionUtils.class);
+
+    public static URLConnection getConnection(String url) throws Exception {
+        if (url.startsWith("https://")) {
+            return getHTTPSConnection(url);
+        } else if (url.startsWith("http://")) {
+            return getHTTPConnection(url);
+        }
+        throw new Exception("Invalid input argument url: " + url);
+    }
+
+    private static URLConnection getHTTPConnection(String urlString) throws Exception {
+        final URL url = new URL(urlString);
+        return url.openConnection();
+    }
+
+    public static URL getUrl(String urlString) throws Exception {
+        if (urlString.toLowerCase().contains("https")) {
+            return getHTTPSUrl(urlString);
+        } else if (urlString.toLowerCase().contains("http")) {
+            return getHTTPUrl(urlString);
+        }
+        throw new Exception("Invalid input argument url: " + urlString);
+    }
+
+    private static URL getHTTPUrl(String urlString) throws MalformedURLException {
+        return new URL(urlString);
+    }
+
+    private static URL getHTTPSUrl(String urlString) throws MalformedURLException, NoSuchAlgorithmException, KeyManagementException {
+        // Create a trust manager that does not validate certificate chains
+        final TrustManager[] trustAllCerts = new TrustManager[] {new TrustAllX509TrustManager()};
+        // Install the all-trusting trust manager   
+        final SSLContext sc = SSLContext.getInstance("SSL");
+        sc.init(null, trustAllCerts, new java.security.SecureRandom());
+        HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+        // Create all-trusting host name verifier   
+        final HostnameVerifier allHostsValid = new HostnameVerifier() {
+            public boolean verify(String hostname, SSLSession session) {
+                return true;
+            }
+        };
+        HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
+        return new URL(urlString);
+    }
+
+    private static URLConnection getHTTPSConnection(String urlString) throws IOException, KeyManagementException, NoSuchAlgorithmException {
+        final URL url = getHTTPSUrl(urlString);
+        return url.openConnection();
+    }
+
+    public static class TrustAllX509TrustManager implements X509TrustManager {
+        @Override
+        public void checkClientTrusted(
+            java.security.cert.X509Certificate[] chain, String authType)
+            throws CertificateException {
+        }
+
+        @Override
+        public void checkServerTrusted(
+            java.security.cert.X509Certificate[] chain, String authType)
+            throws CertificateException {
+        }
+
+        @Override
+        public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+            return null;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/ha/HAURLSelector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/ha/HAURLSelector.java b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/ha/HAURLSelector.java
new file mode 100644
index 0000000..9267785
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-utils/src/main/java/org/apache/eagle/app/utils/ha/HAURLSelector.java
@@ -0,0 +1,29 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.eagle.app.utils.ha;
+
+import java.io.IOException;
+
+public interface HAURLSelector {
+
+    boolean checkUrl(String url);
+
+    void reSelectUrl() throws IOException;
+
+    String getSelectedUrl();
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-core/eagle-app/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/pom.xml b/eagle-core/eagle-app/pom.xml
index 5aff6a6..4e21c99 100644
--- a/eagle-core/eagle-app/pom.xml
+++ b/eagle-core/eagle-app/pom.xml
@@ -33,6 +33,7 @@
 
     <modules>
         <module>eagle-app-base</module>
+        <module>eagle-app-utils</module>
     </modules>
 
     <build>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
index 9f76da2..b0d5987 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
@@ -116,6 +116,11 @@
             <value>secret</value>
         </property>
         <property>
+            <name>eagleProps.eagle.service.basePath</name>
+            <description>eagleProps.eagle.service.basePath</description>
+            <value>/rest</value>
+        </property>
+        <property>
             <name>eagleProps.eagle.service.read.timeout</name>
             <displayName>eagleProps.eagle.service.read.timeout</displayName>
             <description>The maximum amount of time (in seconds) the app is trying to read from eagle service</description>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/pom.xml b/eagle-topology-check/eagle-topology-app/pom.xml
new file mode 100644
index 0000000..78ef6a5
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/pom.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  ~
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>eagle-topology-check</artifactId>
+        <groupId>org.apache.eagle</groupId>
+        <version>0.5.0-incubating-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>eagle-topology-app</artifactId>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-app-base</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-topology-entity</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-app-utils</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.json</groupId>
+            <artifactId>json</artifactId>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptor>src/assembly/eagle-topology-check-assembly.xml</descriptor>
+                    <finalName>eagle-topology-check-${project.version}</finalName>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <tarLongFileMode>posix</tarLongFileMode>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/assembly/eagle-topology-check-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/assembly/eagle-topology-check-assembly.xml b/eagle-topology-check/eagle-topology-app/src/assembly/eagle-topology-check-assembly.xml
new file mode 100644
index 0000000..244c5f0
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/assembly/eagle-topology-check-assembly.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  ~
+  -->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <id>assembly</id>
+    <formats>
+        <format>jar</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>false</useProjectArtifact>
+            <unpack>true</unpack>
+            <scope>runtime</scope>
+            <unpackOptions>
+                <excludes>
+                    <exclude>**/application.conf</exclude>
+                    <exclude>**/defaults.yaml</exclude>
+                    <exclude>**/*storm.yaml</exclude>
+                    <exclude>**/*storm.yaml.1</exclude>
+                    <exclude>**/log4j.properties</exclude>
+                </excludes>
+            </unpackOptions>
+            <excludes>
+                <exclude>org.apache.storm:storm-core</exclude>
+                <exclude>org.slf4j:slf4j-api</exclude>
+                <exclude>org.slf4j:log4j-over-slf4j</exclude>
+                <exclude>org.slf4j:slf4j-log4j12</exclude>
+                <exclude>log4j:log4j</exclude>
+                <exclude>asm:asm</exclude>
+                <exclude>org.apache.log4j.wso2:log4j</exclude>
+                <exclude>log4j:apache-log4j-extras</exclude>
+                <exclude>org.wso2.orbit.com.lmax:disruptor</exclude>
+            </excludes>
+        </dependencySet>
+    </dependencySets>
+    <fileSets>
+        <fileSet>
+            <directory>${project.build.outputDirectory}/</directory>
+            <outputDirectory>/</outputDirectory>
+            <excludes>
+                <exclude>**/application.conf</exclude>
+                <exclude>**/log4j.properties</exclude>
+                <exclude>**/*.yaml</exclude>
+            </excludes>
+        </fileSet>
+    </fileSets>
+</assembly>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java
new file mode 100644
index 0000000..7760e3d
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology;
+
+import backtype.storm.generated.StormTopology;
+import backtype.storm.topology.TopologyBuilder;
+import com.typesafe.config.Config;
+import org.apache.eagle.app.StormApplication;
+import org.apache.eagle.app.environment.impl.StormEnvironment;
+import org.apache.eagle.topology.storm.TopologyCheckAppSpout;
+import org.apache.eagle.topology.storm.TopologyDataPersistBolt;
+
+public class TopologyCheckApp extends StormApplication {
+    @Override
+    public StormTopology execute(Config config, StormEnvironment environment) {
+        TopologyCheckAppConfig topologyCheckAppConfig = TopologyCheckAppConfig.getInstance(config);
+
+        String spoutName = TopologyCheckAppConfig.TOPOLOGY_DATA_FETCH_SPOUT_NAME;
+        String persistBoltName = TopologyCheckAppConfig.TOPOLOGY_ENTITY_PERSIST_BOLT_NAME;
+
+        TopologyBuilder topologyBuilder = new TopologyBuilder();
+        topologyBuilder.setSpout(
+            spoutName,
+            new TopologyCheckAppSpout(topologyCheckAppConfig),
+            topologyCheckAppConfig.dataExtractorConfig.numDataFetcherSpout
+        ).setNumTasks(topologyCheckAppConfig.dataExtractorConfig.numDataFetcherSpout);
+
+        topologyBuilder.setBolt(
+            persistBoltName,
+            new TopologyDataPersistBolt(topologyCheckAppConfig),
+            topologyCheckAppConfig.dataExtractorConfig.numEntityPersistBolt
+        ).setNumTasks(topologyCheckAppConfig.dataExtractorConfig.numEntityPersistBolt).shuffleGrouping(spoutName);
+
+        return topologyBuilder.createTopology();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java
new file mode 100644
index 0000000..f6510da
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.topology.resolver.TopologyRackResolver;
+import org.apache.eagle.topology.resolver.impl.DefaultTopologyRackResolver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TopologyCheckAppConfig implements Serializable {
+
+    public static final String TOPOLOGY_DATA_FETCH_SPOUT_NAME = "topologyDataFetcherSpout";
+    public static final String TOPOLOGY_ENTITY_PERSIST_BOLT_NAME = "topologyEntityPersistBolt";
+
+    private static final int MAX_NUM_THREADS = 10;
+    private static final String HBASE_ZOOKEEPER_CLIENT_PORT = "2181";
+
+    private static final Logger LOG = LoggerFactory.getLogger(TopologyCheckAppConfig.class);
+
+    public DataExtractorConfig dataExtractorConfig;
+    public HBaseConfig hBaseConfig;
+    public HdfsConfig hdfsConfig;
+    public MRConfig mrConfig;
+    public List<TopologyConstants.TopologyType> topologyTypes;
+
+    public Config config;
+
+    private static TopologyCheckAppConfig configManager = new TopologyCheckAppConfig();
+
+    private TopologyCheckAppConfig() {
+        hBaseConfig = null;
+        hdfsConfig = null;
+        mrConfig = null;
+        dataExtractorConfig = new DataExtractorConfig();
+        topologyTypes = new ArrayList<>();
+    }
+
+    public static TopologyCheckAppConfig getInstance(Config config) {
+        configManager.init(config);
+        return configManager;
+    }
+
+    private void init(Config config) {
+        this.config = config;
+
+        this.dataExtractorConfig.site = config.getString("dataExtractorConfig.site");
+        this.dataExtractorConfig.fetchDataIntervalInSecs = config.getLong("dataExtractorConfig.fetchDataIntervalInSecs");
+        this.dataExtractorConfig.parseThreadPoolSize = MAX_NUM_THREADS;
+        if (config.hasPath("dataExtractorConfig.parseThreadPoolSize")) {
+            this.dataExtractorConfig.parseThreadPoolSize = config.getInt("dataExtractorConfig.parseThreadPoolSize");
+        }
+        this.dataExtractorConfig.numDataFetcherSpout = config.getInt("dataExtractorConfig.numDataFetcherSpout");
+        this.dataExtractorConfig.numEntityPersistBolt = config.getInt("dataExtractorConfig.numEntityPersistBolt");
+        String resolveCls = config.getString("dataExtractorConfig.rackResolverCls");
+        try {
+            this.dataExtractorConfig.resolverCls = (Class<? extends TopologyRackResolver>) Class.forName(resolveCls);
+        } catch (ClassNotFoundException e) {
+            LOG.warn("{} is not found, will use DefaultTopologyRackResolver instead", resolveCls);
+            this.dataExtractorConfig.resolverCls = DefaultTopologyRackResolver.class;
+            //e.printStackTrace();
+        }
+
+        if (config.hasPath("dataSourceConfig.hbase")) {
+            topologyTypes.add(TopologyConstants.TopologyType.HBASE);
+            hBaseConfig = new HBaseConfig();
+            hBaseConfig.eagleKeytab = config.getString("dataSourceConfig.hbase.kerberos.eagle.keytab");
+            hBaseConfig.eaglePrincipal = config.getString("dataSourceConfig.hbase.kerberos.eagle.principal");
+            hBaseConfig.hbaseMasterPrincipal = config.getString("dataSourceConfig.hbase.kerberos.master.principal");
+            hBaseConfig.zkQuorum = config.getString("dataSourceConfig.hbase.zkQuorum");
+            hBaseConfig.zkRoot = config.getString("dataSourceConfig.hbase.zkZnodeParent");
+            hBaseConfig.zkClientPort = getOptionalConfig("dataSourceConfig.hbase.zkPropertyClientPort", HBASE_ZOOKEEPER_CLIENT_PORT);
+            hBaseConfig.zkRetryTimes = getOptionalConfig("dataSourceConfig.hbase.zkRetryTimes", "5");
+        }
+
+        if (config.hasPath("dataSourceConfig.mr")) {
+            topologyTypes.add(TopologyConstants.TopologyType.MR);
+            mrConfig = new MRConfig();
+            mrConfig.rmUrls =  config.getString("dataSourceConfig.mr.rmUrl").split(",\\s*");
+            mrConfig.historyServerUrl = getOptionalConfig("dataSourceConfig.mr.historyServerUrl", null);
+        }
+
+        if (config.hasPath("dataSourceConfig.hdfs")) {
+            topologyTypes.add(TopologyConstants.TopologyType.HDFS);
+            hdfsConfig = new HdfsConfig();
+            hdfsConfig.namenodeUrls = config.getString("dataSourceConfig.hdfs.namenodeUrl").split(",\\s*");
+        }
+    }
+
+    public static class DataExtractorConfig implements Serializable {
+        public String site;
+        public int numDataFetcherSpout;
+        public int numEntityPersistBolt;
+        public long fetchDataIntervalInSecs;
+        public int parseThreadPoolSize;
+        public Class<? extends TopologyRackResolver> resolverCls;
+    }
+
+    public static class HBaseConfig implements Serializable {
+        public String zkQuorum;
+        public String zkRoot;
+        public String zkClientPort;
+        public String zkRetryTimes;
+        public String hbaseMasterPrincipal;
+        public String eaglePrincipal;
+        public String eagleKeytab;
+    }
+
+    public static class MRConfig implements Serializable {
+        public String [] rmUrls;
+        public String historyServerUrl;
+    }
+
+    public static class HdfsConfig implements Serializable {
+        public String [] namenodeUrls;
+    }
+
+    private String getOptionalConfig(String key, String defaultValue) {
+        if (this.config.hasPath(key)) {
+            return config.getString(key);
+        } else {
+            return defaultValue;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppMain.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppMain.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppMain.java
new file mode 100644
index 0000000..c8e1168
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppMain.java
@@ -0,0 +1,26 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.eagle.topology;
+
+public class TopologyCheckAppMain {
+
+    public static void main(String[] args) {
+        new TopologyCheckApp().run(args);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppProvider.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppProvider.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppProvider.java
new file mode 100644
index 0000000..5969a7a
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppProvider.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology;
+
+import org.apache.eagle.app.spi.AbstractApplicationProvider;
+
+public class TopologyCheckAppProvider extends AbstractApplicationProvider<TopologyCheckApp> {
+    @Override
+    public TopologyCheckApp getApplication() {
+        return new TopologyCheckApp();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckMessageId.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckMessageId.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckMessageId.java
new file mode 100644
index 0000000..a0303a9
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckMessageId.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology;
+
+import com.google.common.base.Objects;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+
+import java.util.Calendar;
+
+public class TopologyCheckMessageId {
+    private String topologyType;
+    private Long timestamp;
+
+    public TopologyCheckMessageId(TopologyConstants.TopologyType topologyType, Long timestamp) {
+        this.topologyType = topologyType.toString();
+        this.timestamp = timestamp;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        final TopologyCheckMessageId other = (TopologyCheckMessageId) obj;
+        return Objects.equal(this.topologyType, other.topologyType)
+                && Objects.equal(this.timestamp, other.timestamp);
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder().append(topologyType).append(timestamp).toHashCode();
+    }
+
+    @Override
+    public String toString() {
+        Calendar calendar = Calendar.getInstance();
+        calendar.setTimeInMillis(timestamp);
+        return String.format("topologyType=%s, timestamp=%s", topologyType, calendar.getTime());
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyCrawler.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyCrawler.java
new file mode 100644
index 0000000..55eba15
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyCrawler.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology.extractor;
+
+public interface TopologyCrawler {
+
+    /**
+     * Fetch raw data and emit the parsed result to the next bolt
+     * @return
+     */
+    void extract();
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyEntityParser.java
new file mode 100644
index 0000000..6a860ad
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyEntityParser.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology.extractor;
+
+import org.apache.eagle.topology.TopologyConstants;
+
+import java.io.IOException;
+
+public interface TopologyEntityParser {
+    /**
+     * Parse hadoop topology and return the topology entity results
+     * @return the topology entity result
+     */
+    public TopologyEntityParserResult parse(long timestamp) throws IOException;
+
+    /**
+     * Get topology type for the parser
+     * @return topology type
+     */
+    public TopologyConstants.TopologyType getTopologyType();
+
+    /**
+     * Get hadoop version for the parser
+     * @return
+     */
+    public TopologyConstants.HadoopVersion getHadoopVersion();
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyEntityParserResult.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyEntityParserResult.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyEntityParserResult.java
new file mode 100644
index 0000000..f5746f7
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyEntityParserResult.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology.extractor;
+
+import org.apache.eagle.log.entity.GenericMetricEntity;
+import org.apache.eagle.topology.TopologyConstants;
+import org.apache.eagle.topology.entity.TopologyBaseAPIEntity;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TopologyEntityParserResult {
+    private TopologyConstants.HadoopVersion version;
+    private List<TopologyBaseAPIEntity> masterNodes = new ArrayList<>();
+    private List<TopologyBaseAPIEntity> slaveNodes = new ArrayList<>();
+    private List<GenericMetricEntity> metrics = new ArrayList<>();
+
+    public List<TopologyBaseAPIEntity> getSlaveNodes() {
+        return slaveNodes;
+    }
+
+    public void setSlaveNodes(List<TopologyBaseAPIEntity> slaveNodes) {
+        this.slaveNodes = slaveNodes;
+    }
+
+    public List<TopologyBaseAPIEntity> getMasterNodes() {
+        return masterNodes;
+    }
+
+    public void setMasterNodes(List<TopologyBaseAPIEntity> masterNodes) {
+        this.masterNodes = masterNodes;
+    }
+
+    public List<GenericMetricEntity> getMetrics() {
+        return metrics;
+    }
+
+    public void setMetrics(List<GenericMetricEntity> metrics) {
+        this.metrics = metrics;
+    }
+
+    public TopologyConstants.HadoopVersion getVersion() {
+        return version;
+    }
+    public void setVersion(TopologyConstants.HadoopVersion version) {
+        this.version = version;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java
new file mode 100644
index 0000000..17dec20
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology.extractor;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import org.apache.eagle.topology.TopologyCheckAppConfig;
+import org.apache.eagle.topology.TopologyConstants;
+import org.apache.eagle.topology.extractor.hbase.HbaseTopologyCrawler;
+
+import org.apache.eagle.topology.extractor.hdfs.HdfsTopologyCrawler;
+import org.apache.eagle.topology.extractor.mr.MRTopologyCrawler;
+import org.apache.eagle.topology.resolver.TopologyRackResolver;
+import org.apache.eagle.topology.resolver.impl.DefaultTopologyRackResolver;
+import org.slf4j.Logger;
+
+import java.lang.reflect.Constructor;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TopologyExtractorFactory {
+
+    private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(TopologyExtractorFactory.class);
+
+    private TopologyExtractorFactory() {}
+
+    private static Map<String, Constructor<? extends TopologyCrawler>> extractorMap = Collections.synchronizedMap(new HashMap<>());
+
+    private static void registerTopologyExtractor(String topologyType, Class<? extends TopologyCrawler> clazz) {
+        Constructor<? extends TopologyCrawler> constructor = null;
+        try {
+            constructor = clazz.getConstructor(TopologyCheckAppConfig.class, TopologyRackResolver.class, SpoutOutputCollector.class);
+        } catch (NoSuchMethodException e) {
+            e.printStackTrace();
+        }
+        if (constructor != null) {
+            extractorMap.put(topologyType, constructor);
+        }
+    }
+
+    public static TopologyCrawler create(TopologyConstants.TopologyType topologyType, TopologyCheckAppConfig config, TopologyRackResolver rackResolver, SpoutOutputCollector collector) {
+        if (extractorMap.containsKey(topologyType.toString().toUpperCase())) {
+            Constructor<? extends TopologyCrawler> constructor = extractorMap.get(topologyType.name());
+            try {
+                return constructor.newInstance(config, rackResolver, collector);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        } else {
+            LOG.error("Unsupported topology type {}", topologyType.toString());
+        }
+        return null;
+    }
+
+    static {
+        registerTopologyExtractor(TopologyConstants.TopologyType.HBASE.name(), HbaseTopologyCrawler.class);
+        registerTopologyExtractor(TopologyConstants.TopologyType.HDFS.name(), HdfsTopologyCrawler.class);
+        registerTopologyExtractor(TopologyConstants.TopologyType.MR.name(), MRTopologyCrawler.class);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java
new file mode 100644
index 0000000..398178f
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology.extractor.hbase;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.tuple.Values;
+import org.apache.eagle.topology.TopologyCheckAppConfig;
+import org.apache.eagle.topology.TopologyCheckMessageId;
+import org.apache.eagle.topology.TopologyConstants;
+import org.apache.eagle.topology.extractor.TopologyEntityParserResult;
+import org.apache.eagle.topology.extractor.TopologyCrawler;
+import org.apache.eagle.topology.resolver.TopologyRackResolver;
+import org.apache.eagle.topology.utils.EntityBuilderHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class HbaseTopologyCrawler implements TopologyCrawler {
+
+    private static final Logger LOG = LoggerFactory.getLogger(HbaseTopologyCrawler.class);
+
+    private HbaseTopologyEntityParser parser;
+    private SpoutOutputCollector collector;
+    private String site;
+
+    public HbaseTopologyCrawler(TopologyCheckAppConfig config, TopologyRackResolver rackResolver, SpoutOutputCollector collector) {
+        this.site = config.dataExtractorConfig.site;
+        this.parser = new HbaseTopologyEntityParser(this.site, config.hBaseConfig, rackResolver);
+        this.collector = collector;
+    }
+
+    @Override
+    public void extract() {
+        long updateTimestamp = System.currentTimeMillis();
+        TopologyEntityParserResult result = null;
+        try {
+            result = parser.parse(updateTimestamp);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        if (result == null) {
+            LOG.warn("No data fetched");
+            result = new TopologyEntityParserResult();
+        }
+        if (result.getMasterNodes().isEmpty()) {
+            result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.HMASTER_ROLE, 0, site, updateTimestamp));
+        }
+        TopologyCheckMessageId messageId = new TopologyCheckMessageId(TopologyConstants.TopologyType.HBASE, updateTimestamp);
+        this.collector.emit(new Values(TopologyConstants.HBASE_INSTANCE_SERVICE_NAME, result), messageId);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
new file mode 100644
index 0000000..f93665d
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology.extractor.hbase;
+
+import org.apache.eagle.app.utils.HadoopSecurityUtil;
+import org.apache.eagle.topology.TopologyCheckAppConfig;
+import org.apache.eagle.topology.TopologyConstants;
+import org.apache.eagle.topology.extractor.TopologyEntityParserResult;
+import org.apache.eagle.topology.entity.HBaseServiceTopologyAPIEntity;
+import org.apache.eagle.topology.extractor.TopologyEntityParser;
+import org.apache.eagle.topology.resolver.TopologyRackResolver;
+import org.apache.eagle.topology.utils.EntityBuilderHelper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.ServerLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.eagle.topology.TopologyConstants.*;
+
+public class HbaseTopologyEntityParser implements TopologyEntityParser {
+
+    private Configuration hBaseConfiguration;
+    private String site;
+    private Boolean kerberosEnable = false;
+    private TopologyRackResolver rackResolver;
+
+    public  HbaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig, TopologyRackResolver resolver) {
+        this.site = site;
+        this.rackResolver = resolver;
+        this.hBaseConfiguration = HBaseConfiguration.create();
+        this.hBaseConfiguration.set("hbase.zookeeper.quorum", hBaseConfig.zkQuorum);
+        this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", hBaseConfig.zkClientPort);
+        this.hBaseConfiguration.set("zookeeper.znode.parent", hBaseConfig.zkRoot);
+        this.hBaseConfiguration.set("hbase.client.retries.number", hBaseConfig.zkRetryTimes);
+        // kerberos authentication
+        this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal);
+        this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab);
+        if (hBaseConfig.eaglePrincipal != null && hBaseConfig.eagleKeytab != null
+                && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) {
+            this.kerberosEnable = true;
+            this.hBaseConfiguration.set("hbase.security.authentication", "kerberos");
+            this.hBaseConfiguration.set("hbase.master.kerberos.principal", hBaseConfig.hbaseMasterPrincipal);
+        }
+    }
+
+    private HBaseAdmin getHBaseAdmin() throws IOException {
+        if (this.kerberosEnable) {
+            HadoopSecurityUtil.login(hBaseConfiguration);
+        }
+        return new HBaseAdmin(this.hBaseConfiguration);
+    }
+
+    @Override
+    public TopologyEntityParserResult parse(long timestamp) throws IOException {
+        long deadServers = 0;
+        long liveServers = 0;
+        TopologyEntityParserResult result = new TopologyEntityParserResult();
+        HBaseAdmin admin = null;
+        try {
+            admin = getHBaseAdmin();
+            ClusterStatus status = admin.getClusterStatus();
+            deadServers = status.getDeadServers();
+            liveServers = status.getServersSize();
+            result.setVersion(HadoopVersion.V2);
+            for (ServerName liveServer : status.getServers()) {
+                ServerLoad load = status.getLoad(liveServer);
+                result.getSlaveNodes().add(parseServer(liveServer, load, TopologyConstants.REGIONSERVER_ROLE, TopologyConstants.REGIONSERVER_LIVE_STATUS, timestamp));
+            }
+            for (ServerName deadServer : status.getDeadServerNames()) {
+                ServerLoad load = status.getLoad(deadServer);
+                result.getSlaveNodes().add(parseServer(deadServer, load, TopologyConstants.REGIONSERVER_ROLE, TopologyConstants.REGIONSERVER_DEAD_STATUS, timestamp));
+            }
+            ServerName master = status.getMaster();
+            if (master != null) {
+                ServerLoad load = status.getLoad(master);
+                result.getMasterNodes().add(parseServer(master, load, TopologyConstants.HMASTER_ROLE, TopologyConstants.HMASTER_ACTIVE_STATUS, timestamp));
+            }
+            for (ServerName backupMaster : status.getBackupMasters()) {
+                ServerLoad load = status.getLoad(backupMaster);
+                result.getMasterNodes().add(parseServer(backupMaster, load, TopologyConstants.HMASTER_ROLE, TopologyConstants.HMASTER_STANDBY_STATUS, timestamp));
+            }
+            double liveRatio = liveServers * 1d / (liveServers + deadServers);
+            result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.REGIONSERVER_ROLE, liveRatio, site, timestamp));
+            return result;
+        } catch (RuntimeException e) {
+            e.printStackTrace();
+        } finally {
+            if (admin != null) {
+                try {
+                    admin.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+        return result;
+    }
+
+    private HBaseServiceTopologyAPIEntity parseServer(ServerName serverName, ServerLoad serverLoad, String role, String status, long timestamp) {
+        if (serverName == null) {
+            return null;
+        }
+        HBaseServiceTopologyAPIEntity entity = createEntity(role, serverName.getHostname(), timestamp);
+        parseServerLoad(entity, serverLoad);
+        entity.setStatus(status);
+        return entity;
+    }
+
+    private void parseServerLoad(HBaseServiceTopologyAPIEntity entity, ServerLoad load) {
+        if (load == null) {
+            return;
+        }
+        entity.setMaxHeapMB(load.getMaxHeapMB());
+        entity.setUsedHeapMB(load.getUsedHeapMB());
+        entity.setNumRegions(load.getNumberOfRegions());
+        entity.setNumRequests(load.getNumberOfRequests());
+    }
+
+    private HBaseServiceTopologyAPIEntity createEntity(String roleType, String hostname, long timestamp) {
+        HBaseServiceTopologyAPIEntity entity = new HBaseServiceTopologyAPIEntity();
+        Map<String, String> tags = new HashMap<String, String>();
+        entity.setTags(tags);
+        tags.put(SITE_TAG, site);
+        tags.put(ROLE_TAG, roleType);
+        tags.put(HOSTNAME_TAG, hostname);
+        String rack = rackResolver.resolve(hostname);
+        tags.put(RACK_TAG, rack);
+        entity.setLastUpdateTime(timestamp);
+        entity.setTimestamp(timestamp);
+        return entity;
+    }
+
+    @Override
+    public TopologyConstants.TopologyType getTopologyType() {
+        return TopologyType.HBASE;
+    }
+
+    @Override
+    public TopologyConstants.HadoopVersion getHadoopVersion() {
+        return HadoopVersion.V2;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9e873770/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyCrawler.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyCrawler.java
new file mode 100644
index 0000000..7030221
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyCrawler.java
@@ -0,0 +1,65 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.eagle.topology.extractor.hdfs;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.tuple.Values;
+import org.apache.eagle.topology.TopologyCheckAppConfig;
+import org.apache.eagle.topology.TopologyCheckMessageId;
+import org.apache.eagle.topology.TopologyConstants;
+import org.apache.eagle.topology.extractor.TopologyEntityParserResult;
+import org.apache.eagle.topology.extractor.TopologyCrawler;
+import org.apache.eagle.topology.resolver.TopologyRackResolver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class HdfsTopologyCrawler implements TopologyCrawler {
+
+    private static final Logger LOG = LoggerFactory.getLogger(HdfsTopologyCrawler.class);
+
+    private HdfsTopologyEntityParser parser;
+    private SpoutOutputCollector collector;
+
+    public HdfsTopologyCrawler(TopologyCheckAppConfig config, TopologyRackResolver rackResolver, SpoutOutputCollector collector) {
+        this.parser = new HdfsTopologyEntityParser(config.dataExtractorConfig.site, config.hdfsConfig, rackResolver);
+        this.collector = collector;
+    }
+
+    @Override
+    public void extract() {
+        long updateTimestamp = System.currentTimeMillis();
+        TopologyEntityParserResult result = null;
+        try {
+            result = parser.parse(updateTimestamp);
+        } catch (IOException e) {
+            e.printStackTrace();
+            return;
+        }
+        if (result == null || result.getMasterNodes().isEmpty()) {
+            LOG.warn("No data fetched");
+            return;
+        }
+        TopologyCheckMessageId messageId = new TopologyCheckMessageId(TopologyConstants.TopologyType.HDFS, updateTimestamp);
+        this.collector.emit(new Values(TopologyConstants.HDFS_INSTANCE_SERVICE_NAME, result), messageId);
+    }
+
+
+}


Mime
View raw message