eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject incubator-eagle git commit: [EAGLE-799] Can not get hbase info by service api when the master becomes not available
Date Thu, 24 Nov 2016 13:14:51 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 8042f9e44 -> de061d599


[EAGLE-799] Can not get hbase info by service api when the master becomes not available

Can not get hbase info by service api when the master becomes not available

Author: yupu <yupu@ebay.com>

Closes #680 from puyulu/EAGLE-799.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/de061d59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/de061d59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/de061d59

Branch: refs/heads/master
Commit: de061d599c35735abd6b9447d37949289827e1e3
Parents: 8042f9e
Author: yupu <yupu@ebay.com>
Authored: Thu Nov 24 21:14:43 2016 +0800
Committer: Zhao, Qingwen <qingwzhao@apache.org>
Committed: Thu Nov 24 21:14:43 2016 +0800

----------------------------------------------------------------------
 .../extractor/hbase/HbaseTopologyEntityParser.java     |  5 +++--
 .../extractor/hdfs/HdfsTopologyEntityParser.java       | 13 +++++++++++++
 .../topology/extractor/mr/MRTopologyEntityParser.java  |  9 +++++----
 .../eagle/topology/storm/TopologyDataPersistBolt.java  |  2 +-
 .../topology/entity/HdfsServiceTopologyAPIEntity.java  | 13 +++++++++++++
 5 files changed, 35 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
index 77781a3..f3eac23 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
@@ -46,7 +46,7 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser {
     private Boolean kerberosEnable = false;
     private TopologyRackResolver rackResolver;
 
-    public  HbaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig,
TopologyRackResolver resolver) {
+    public HbaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig,
TopologyRackResolver resolver) {
         this.site = site;
         this.rackResolver = resolver;
         this.hBaseConfiguration = HBaseConfiguration.create();
@@ -56,7 +56,7 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser {
         this.hBaseConfiguration.set("hbase.client.retries.number", hBaseConfig.zkRetryTimes);
         // kerberos authentication
         if (hBaseConfig.eaglePrincipal != null && hBaseConfig.eagleKeytab != null
-                && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty())
{
+            && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty())
{
             this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal);
             this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab);
             this.kerberosEnable = true;
@@ -103,6 +103,7 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser
{
             }
             double liveRatio = liveServers * 1d / (liveServers + deadServers);
             result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.REGIONSERVER_ROLE,
liveRatio, site, timestamp));
+            result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.HMASTER_ROLE,
1d, site, timestamp));
             return result;
         } catch (RuntimeException e) {
             e.printStackTrace();

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
index 627584a..72dce80 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
@@ -72,6 +72,8 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser {
     private static final String DATA_NODE_CAPACITY = "capacity";
     private static final String DATA_NODE_ADMIN_STATE = "adminState";
     private static final String DATA_NODE_FAILED_VOLUMN = "volfails";
+    private static final String DATA_NODE_VERSION = "version";
+    private static final String NAME_NODE_VERSION = "Version";
 
     private static final String DATA_NODE_DECOMMISSIONED = "Decommissioned";
     private static final String DATA_NODE_DECOMMISSIONED_STATE = "decommissioned";
@@ -115,9 +117,18 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser
{
         final String urlString = buildFSNamesystemURL(url);
         final Map<String, JMXBean> jmxBeanMap = JMXQueryHelper.query(urlString);
         final JMXBean bean = jmxBeanMap.get(JMX_FS_NAME_SYSTEM_BEAN_NAME);
+
+        final String nameNodeUrlString = buildNamenodeInfo(url);
+        final Map<String, JMXBean> nameNodeMap = JMXQueryHelper.query(nameNodeUrlString);
+        final JMXBean nameNodeBean = nameNodeMap.get(JMX_NAMENODE_INFO);
+
         if (bean == null || bean.getPropertyMap() == null) {
             throw new ServiceNotResponseException("Invalid JMX format, FSNamesystem bean
is null!");
         }
+        if (nameNodeBean == null || nameNodeBean.getPropertyMap() == null) {
+            throw new ServiceNotResponseException("Invalid JMX format, NameNode bean is null!");
+        }
+
         final String hostname = (String) bean.getPropertyMap().get(HA_NAME);
         HdfsServiceTopologyAPIEntity result = createHdfsServiceEntity(TopologyConstants.NAME_NODE_ROLE,
hostname, updateTime);
         final String state = (String) bean.getPropertyMap().get(HA_STATE);
@@ -128,6 +139,7 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser
{
         result.setUsedCapacityTB(Double.toString(capacityUsedGB / 1024));
         final Integer blocksTotal = (Integer) bean.getPropertyMap().get(BLOCKS_TOTAL);
         result.setNumBlocks(Integer.toString(blocksTotal));
+        result.setVersion((String) nameNodeBean.getPropertyMap().get(NAME_NODE_VERSION));
         return result;
     }
 
@@ -240,6 +252,7 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser
{
             } else {
                 entity.setStatus(TopologyConstants.DATA_NODE_LIVE_STATUS);
             }
+            entity.setVersion(String.valueOf(liveNode.get(DATA_NODE_VERSION)));
             numLiveNodes++;
             result.getSlaveNodes().add(entity);
         }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java
b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java
index 671d07b..52148b7 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java
@@ -46,7 +46,7 @@ import static org.apache.eagle.topology.TopologyConstants.*;
 
 public class MRTopologyEntityParser implements TopologyEntityParser {
 
-    private String [] rmUrls;
+    private String[] rmUrls;
     private String historyServerUrl;
     private String site;
     private TopologyRackResolver rackResolver;
@@ -91,9 +91,10 @@ public class MRTopologyEntityParser implements TopologyEntityParser {
                 // reSelect url
             }
         }
-        if (result.getMasterNodes().isEmpty()) {
-            result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.RESOURCE_MANAGER_ROLE,
0, site, timestamp));
-        }
+
+        double value = result.getMasterNodes().isEmpty() ? 0 : result.getMasterNodes().size()
* 1d / rmUrls.length;
+        result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.RESOURCE_MANAGER_ROLE,
value, site, timestamp));
+
         doCheckHistoryServer(timestamp, result);
         return result;
     }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java
b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java
index e0ff3cd..afd3518 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java
@@ -90,7 +90,7 @@ public class TopologyDataPersistBolt extends BaseRichBolt {
             GenericServiceAPIResponseEntity<TopologyBaseAPIEntity> response = client.search().query(query).pageSize(Integer.MAX_VALUE).send();
             if (response.isSuccess() && response.getObj() != null) {
                 for (TopologyBaseAPIEntity entity : response.getObj()) {
-                    if (!availableHostnames.contains(generateKey(entity))) {
+                    if (availableHostnames != null && availableHostnames.size() >
0 && !availableHostnames.contains(generateKey(entity))) {
                         entitiesForDeletion.add(entity);
                     }
                 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
index baca97b..d5ee41e 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
@@ -43,6 +43,8 @@ public class HdfsServiceTopologyAPIEntity extends TopologyBaseAPIEntity
{
     private long writtenTxidDiff;
     @Column("g")
     private long lastUpdateTime;
+    @Column("h")
+    private String version;
 
     public long getLastUpdateTime() {
         return lastUpdateTime;
@@ -106,5 +108,16 @@ public class HdfsServiceTopologyAPIEntity extends TopologyBaseAPIEntity
{
         this.writtenTxidDiff = writtenTxidDiff;
         valueChanged("writtenTxidDiff");
     }
+
+    public String getVersion() {
+        return version;
+    }
+
+    public void setVersion(String version) {
+        this.version = version;
+        valueChanged("version");
+    }
+
+
 }
 


Mime
View raw message