eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject incubator-eagle git commit: [EAGLE-816] Fix host problem in jmx collector script
Date Wed, 30 Nov 2016 10:01:08 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master f39ffe915 -> 68a3e8bde


[EAGLE-816] Fix host problem in jmx collector script

[EAGLE-816] Fix host problem in jmx collector script

Fix host name bug in jmx collector scripts, which current always only use current fadn instead
of configured host, it's a bug.

Author: Hao Chen <hao@apache.org>

Closes #701 from haoch/EAGLE-816.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/68a3e8bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/68a3e8bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/68a3e8bd

Branch: refs/heads/master
Commit: 68a3e8bde3418add34c0b585a206236ee227c6e0
Parents: f39ffe9
Author: Hao Chen <hao@apache.org>
Authored: Wed Nov 30 18:00:51 2016 +0800
Committer: Hao Chen <hao@apache.org>
Committed: Wed Nov 30 18:00:51 2016 +0800

----------------------------------------------------------------------
 .../hadoop_jmx_collector/hadoop_ha_checker.py   | 19 +++++++++++++++++--
 .../hadoop_jmx_collector/metric_collector.py    | 20 ++++++++++----------
 2 files changed, 27 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/68a3e8bd/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py b/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
index e8bf4ae..dcca28e 100644
--- a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
+++ b/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
@@ -17,17 +17,19 @@
 #
 
 from metric_collector import MetricCollector, JmxReader, YarnWSReader, Runner
-import logging,socket
+import logging,socket,string
 
 class HadoopNNHAChecker(MetricCollector):
     def run(self):
         hosts = []
-
+        host_name_list = []
         for input in self.config["input"]:
             if not input.has_key("host"):
                 input["host"] = socket.getfqdn()
             if input.has_key("component") and input["component"] == "namenode":
                 hosts.append(input)
+                host_name_list.append(input["host"])
+
         if not bool(hosts):
             logging.warn("non hosts are configured as 'namenode' in 'input' config, exit")
             return
@@ -35,7 +37,10 @@ class HadoopNNHAChecker(MetricCollector):
         logging.info("Checking namenode HA: " + str(hosts))
         total_count = len(hosts)
 
+        all_hosts_name = string.join(host_name_list,",")
+
         self.collect({
+            "host": all_hosts_name,
             "component": "namenode",
             "metric": "hadoop.namenode.hastate.total.count",
             "value": total_count
@@ -63,18 +68,21 @@ class HadoopNNHAChecker(MetricCollector):
                 logging.exception("failed to read jmx from " + host["host"] + ":" + host["port"])
                 failed_count += 1
         self.collect({
+            "host": all_hosts_name,
             "component": "namenode",
             "metric": "hadoop.namenode.hastate.active.count",
             "value": active_count
         })
 
         self.collect({
+            "host": all_hosts_name,
             "component": "namenode",
             "metric": "hadoop.namenode.hastate.standby.count",
             "value": standby_count
         })
 
         self.collect({
+            "host": all_hosts_name,
             "component": "namenode",
             "metric": "hadoop.namenode.hastate.failed.count",
             "value": failed_count
@@ -83,19 +91,23 @@ class HadoopNNHAChecker(MetricCollector):
 class HadoopRMHAChecker(MetricCollector):
     def run(self):
         hosts = []
+        all_hosts = []
         for input in self.config["input"]:
             if not input.has_key("host"):
                 input["host"] = socket.getfqdn()
             if input.has_key("component") and input["component"] == "resourcemanager":
                 hosts.append(input)
+                all_hosts.append(input["host"])
         if not bool(hosts):
             logging.warn("Non hosts are configured as 'resourcemanager' in 'input' config,
exit")
             return
 
         logging.info("Checking resource manager HA: " + str(hosts))
         total_count = len(hosts)
+        all_hosts_name = string.join(all_hosts,",")
 
         self.collect({
+            "host": all_hosts_name,
             "component": "resourcemanager",
             "metric": "hadoop.resourcemanager.hastate.total.count",
             "value": total_count
@@ -120,18 +132,21 @@ class HadoopRMHAChecker(MetricCollector):
                 failed_count += 1
 
         self.collect({
+            "host": all_hosts_name,
             "component": "resourcemanager",
             "metric": "hadoop.resourcemanager.hastate.active.count",
             "value": active_count
         })
 
         self.collect({
+            "host": all_hosts_name,
             "component": "resourcemanager",
             "metric": "hadoop.resourcemanager.hastate.standby.count",
             "value": standby_count
         })
 
         self.collect({
+            "host": all_hosts_name,
             "component": "resourcemanager",
             "metric": "hadoop.resourcemanager.hastate.failed.count",
             "value": failed_count

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/68a3e8bd/eagle-external/hadoop_jmx_collector/metric_collector.py
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/metric_collector.py b/eagle-external/hadoop_jmx_collector/metric_collector.py
index 2e1520a..6eeec5a 100644
--- a/eagle-external/hadoop_jmx_collector/metric_collector.py
+++ b/eagle-external/hadoop_jmx_collector/metric_collector.py
@@ -251,7 +251,7 @@ class MetricCollector(threading.Thread):
         if msg.has_key("value"):
             msg["value"] = float(str(msg["value"]))
         if not msg.has_key("host") or len(msg["host"]) == 0:
-            msg["host"] = self.fqdn
+            raise Exception("host is null: " + str(msg))
         if not msg.has_key("site"):
             msg["site"] = self.config["env"]["site"]
         self.sender.send(msg)
@@ -327,18 +327,18 @@ class JmxMetricCollector(MetricCollector):
         for input in self.input_components:
             try:
                 beans = JmxReader(input["host"], input["port"], input["https"]).open().get_jmx_beans()
-                self.on_beans(input["component"], beans)
+                self.on_beans(input, beans)
             except Exception as e:
                 logging.exception("Failed to read jmx for " + str(input))
 
     def filter_bean(self, bean, mbean_domain):
         return mbean_domain in self.selected_domain
 
-    def on_beans(self, component, beans):
+    def on_beans(self, source, beans):
         for bean in beans:
-            self.on_bean(component,bean)
+            self.on_bean(source,bean)
 
-    def on_bean(self, component, bean):
+    def on_bean(self, source, bean):
         # mbean is of the form "domain:key=value,...,foo=bar"
         mbean = bean[u'name']
         mbean_domain, mbean_attribute = mbean.rstrip().split(":", 1)
@@ -351,18 +351,19 @@ class JmxMetricCollector(MetricCollector):
         metric_prefix_name = self.__build_metric_prefix(mbean_attribute, context)
 
         for key, value in bean.iteritems():
-            self.on_bean_kv(metric_prefix_name, component,key, value)
+            self.on_bean_kv(metric_prefix_name, source, key, value)
 
         for listener in self.listeners:
-            listener.on_bean(component, bean.copy())
+            listener.on_bean(source, bean.copy())
 
-    def on_bean_kv(self, prefix,component, key, value):
+    def on_bean_kv(self, prefix,source, key, value):
         # Skip Tags
         if re.match(r'tag.*', key):
             return
         metric_name = (prefix + '.' + key).lower()
         self.on_metric({
-            "component": component,
+            "component": source["component"],
+            "host": source["host"],
             "metric": metric_name,
             "value": value
         })
@@ -384,7 +385,6 @@ class JmxMetricCollector(MetricCollector):
             metric_prefix_name = '.'.join([i[1] for i in mbean_list])
         return ("hadoop." + metric_prefix_name).replace(" ", "").lower()
 
-
 class JmxMetricListener:
     def init(self, collector):
         self.collector = collector


Mime
View raw message