ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-12285 - Alerts: HDFS Capacity Utilization Alert Always Shows 0% (jonathanhurley)
Date Mon, 06 Jul 2015 16:08:25 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 1c6c6a459 -> 9c1c7f935


AMBARI-12285 - Alerts: HDFS Capacity Utilization Alert Always Shows 0% (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9c1c7f93
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9c1c7f93
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9c1c7f93

Branch: refs/heads/trunk
Commit: 9c1c7f9356490f0717b491bff33c4db84d2717ba
Parents: 1c6c6a4
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Sun Jul 5 23:56:25 2015 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Mon Jul 6 11:53:23 2015 -0400

----------------------------------------------------------------------
 .../python/ambari_agent/alerts/metric_alert.py  |  10 +-
 .../src/test/python/ambari_agent/TestAlerts.py  |  66 +++++++++
 .../server/upgrade/UpgradeCatalog210.java       | 144 +++++++++++++------
 .../common-services/HDFS/2.1.0.2.0/alerts.json  |  16 +--
 .../stacks/BIGTOP/0.8/services/HDFS/alerts.json |  16 +--
 5 files changed, 192 insertions(+), 60 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9c1c7f93/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
index fea2f6c..ece6063 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
@@ -262,6 +262,14 @@ class MetricAlert(BaseAlert):
 
     
 class JmxMetric:
+  DYNAMIC_CODE_TEMPLATE = """
+# ensure that division yields a float, use // for integer division
+from __future__ import division
+
+def f(args):
+  return {0}
+"""
+
   def __init__(self, jmx_info):
     self.custom_module = None
     self.property_list = jmx_info['property_list']
@@ -271,7 +279,7 @@ class JmxMetric:
       realcode = re.sub('(\{(\d+)\})', 'args[\g<2>]', jmx_info['value'])
       
       self.custom_module =  imp.new_module(str(uuid.uuid4()))
-      code = 'def f(args):\n  return ' + realcode
+      code = self.DYNAMIC_CODE_TEMPLATE.format(realcode)
       exec code in self.custom_module.__dict__
     
     for p in self.property_list:

http://git-wip-us.apache.org/repos/asf/ambari/blob/9c1c7f93/ambari-agent/src/test/python/ambari_agent/TestAlerts.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestAlerts.py b/ambari-agent/src/test/python/ambari_agent/TestAlerts.py
index 4570842..26f1617 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestAlerts.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestAlerts.py
@@ -1011,6 +1011,32 @@ class TestAlerts(TestCase):
     # try to request a dictionary object instead of a property
     self.assertEquals(configuration["foo-site"], alert._get_configuration_value("{{foo-site}}"))
 
+
+  @patch.object(MetricAlert, "_load_jmx")
+  def test_metric_alert_floating_division(self, ma_load_jmx_mock):
+    definition_json = self._get_metric_alert_definition_with_float_division()
+    configuration = {'hdfs-site' :
+      { 'dfs.datanode.http.address': 'c6401.ambari.apache.org:80'}
+    }
+
+    collector = AlertCollector()
+    cluster_configuration = self.__get_cluster_configuration()
+    self.__update_cluster_configuration(cluster_configuration, configuration)
+
+    alert = MetricAlert(definition_json, definition_json['source'], None)
+    alert.set_helpers(collector, cluster_configuration)
+    alert.set_cluster("c1", "c6401.ambari.apache.org")
+
+    # 10 / 5
+    ma_load_jmx_mock.return_value = ([10, 5], None)
+
+    alert.collect()
+    alerts = collector.alerts()
+    self.assertEquals(0, len(collector.alerts()))
+    self.assertEquals('OK', alerts[0]['state'])
+    self.assertEquals('(Unit Tests) OK: 10 5 2.0', alerts[0]['text'])
+
+
   def __get_cluster_configuration(self):
     """
     Gets an instance of the cluster cache where the file read and write
@@ -1151,6 +1177,46 @@ class TestAlerts(TestCase):
       }
     }
 
+  def _get_metric_alert_definition_with_float_division(self):
+    return {
+      "name": "DataNode CPU Check",
+      "service": "HDFS",
+      "component": "DATANODE",
+      "label": "DataNode Process",
+      "interval": 6,
+      "scope": "host",
+      "enabled": True,
+      "uuid": "c1f73191-4481-4435-8dae-fd380e4c0be1",
+      "source": {
+        "type": "METRIC",
+        "uri": {
+          "http": "{{hdfs-site/dfs.datanode.http.address}}",
+          "https": "{{hdfs-site/dfs.datanode.https.address}}",
+          "https_property": "{{hdfs-site/dfs.http.policy}}",
+          "https_property_value": "HTTPS_ONLY"
+        },
+        "jmx": {
+          "property_list": [
+            "someJmxObject/value",
+            "someOtherJmxObject/value"
+          ],
+          "value": "{0} / {1}"
+        },
+        "reporting": {
+          "ok": {
+            "text": "(Unit Tests) OK: {0} {1} {2}",
+          },
+          "warning": {
+            "text": "(Unit Tests) Warning: {0} {1} {2}",
+            "value": 150
+          },
+          "critical": {
+            "text": "(Unit Tests) Critical: {0} {1} {2}",
+            "value": 200
+          }
+        }
+      }
+    }
 
   def _get_web_alert_definition(self):
     return {

http://git-wip-us.apache.org/repos/asf/ambari/blob/9c1c7f93/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
index 0be06c8..eb1bab5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
@@ -18,12 +18,28 @@
 
 package org.apache.ambari.server.upgrade;
 
-import com.google.gson.Gson;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.persist.Transactional;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.regex.Matcher;
+
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.persistence.criteria.CriteriaBuilder;
+import javax.persistence.criteria.CriteriaDelete;
+import javax.persistence.criteria.Root;
+
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.configuration.Configuration;
@@ -45,7 +61,6 @@ import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.state.Host;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.StackId;
-import org.apache.ambari.server.state.alert.AlertDefinitionFactory;
 import org.apache.ambari.server.state.stack.OsFamily;
 import org.apache.ambari.server.utils.VersionUtils;
 import org.apache.commons.lang.StringUtils;
@@ -53,26 +68,13 @@ import org.eclipse.persistence.internal.databaseaccess.FieldTypeDefinition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.persistence.EntityManager;
-import javax.persistence.Query;
-import javax.persistence.criteria.CriteriaBuilder;
-import javax.persistence.criteria.CriteriaDelete;
-import javax.persistence.criteria.Root;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.regex.Matcher;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.persist.Transactional;
 
 
 /**
@@ -1067,7 +1069,6 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
     AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
     AlertDefinitionDAO alertDefinitionDAO = injector.getInstance(AlertDefinitionDAO.class);
     Clusters clusters = ambariManagementController.getClusters();
-    AlertDefinitionFactory alertDefinitionFactory = injector.getInstance(AlertDefinitionFactory.class);
 
     List<String> metricAlerts = Arrays.asList("namenode_cpu", "namenode_hdfs_blocks_health",
             "namenode_hdfs_capacity_utilization", "namenode_rpc_latency",
@@ -1151,12 +1152,13 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
             rootJson.addProperty("default_port", new Integer(8019));
 
             // save the changes
-            updateAlertDefinitionEntitySource("hdfs_zookeeper_failover_controller_process",
rootJson.toString(), UUID.randomUUID().toString());
+            updateAlertDefinitionEntitySource("hdfs_zookeeper_failover_controller_process",
+                rootJson.toString(), UUID.randomUUID().toString());
           }
 
           // update oozie web ui alert
-          AlertDefinitionEntity oozieWebUIAlertDefinitionEntity =  alertDefinitionDAO.findByName(cluster.getClusterId(),
-                            "oozie_server_webui");
+          AlertDefinitionEntity oozieWebUIAlertDefinitionEntity = alertDefinitionDAO.findByName(
+              cluster.getClusterId(), "oozie_server_webui");
 
           if (oozieWebUIAlertDefinitionEntity != null) {
             String source = oozieWebUIAlertDefinitionEntity.getSource();
@@ -1172,7 +1174,41 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
                     "{{cluster-env/smokeuser_principal_name}}");
 
             // save the changes
-            updateAlertDefinitionEntitySource("oozie_server_webui", rootJson.toString(),
UUID.randomUUID().toString());
+            updateAlertDefinitionEntitySource("oozie_server_webui", rootJson.toString(),
+                UUID.randomUUID().toString());
+          }
+
+          // update HDFS metric alerts that had changes to their text
+          List<String> hdfsMetricAlertsFloatDivision = Arrays.asList(
+              "namenode_hdfs_capacity_utilization", "datanode_storage");
+
+          for (String metricAlertName : hdfsMetricAlertsFloatDivision) {
+            AlertDefinitionEntity entity = alertDefinitionDAO.findByName(cluster.getClusterId(),
+                metricAlertName);
+
+            if (null == entity) {
+              continue;
+            }
+
+            String source = entity.getSource();
+            JsonObject rootJson = new JsonParser().parse(source).getAsJsonObject();
+            JsonObject reporting = rootJson.getAsJsonObject("reporting");
+            JsonObject ok = reporting.getAsJsonObject("ok");
+            JsonObject warning = reporting.getAsJsonObject("warning");
+            JsonObject critical = reporting.getAsJsonObject("critical");
+
+            JsonElement okText = ok.remove("text");
+            ok.addProperty("text", okText.getAsString().replace("{2:d}", "{2:.0f}"));
+
+            JsonElement warningText = warning.remove("text");
+            warning.addProperty("text", warningText.getAsString().replace("{2:d}", "{2:.0f}"));
+
+            JsonElement criticalText = critical.remove("text");
+            critical.addProperty("text", criticalText.getAsString().replace("{2:d}", "{2:.0f}"));
+
+            // save the changes
+            updateAlertDefinitionEntitySource(metricAlertName, rootJson.toString(),
+                UUID.randomUUID().toString());
           }
         }
       }
@@ -1316,17 +1352,39 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
   }
 
   private int getHbaseRamRecomendations(int totalMem) {
-    if (totalMem <= 4) return 1;
-    if (4 < totalMem && totalMem <= 8) return 1;
-    if (8 < totalMem && totalMem <= 16) return 2;
-    if (16 < totalMem && totalMem <= 24) return 4;
-    if (24 < totalMem && totalMem <= 48) return 8;
-    if (48 < totalMem && totalMem <= 64) return 8;
-    if (64 < totalMem && totalMem <= 72) return 8;
-    if (72 < totalMem && totalMem <= 96) return 16;
-    if (96 < totalMem && totalMem <= 128) return 24;
-    if (128 < totalMem && totalMem <= 256) return 32;
-    if (256 < totalMem) return 64;
+    if (totalMem <= 4) {
+      return 1;
+    }
+    if (4 < totalMem && totalMem <= 8) {
+      return 1;
+    }
+    if (8 < totalMem && totalMem <= 16) {
+      return 2;
+    }
+    if (16 < totalMem && totalMem <= 24) {
+      return 4;
+    }
+    if (24 < totalMem && totalMem <= 48) {
+      return 8;
+    }
+    if (48 < totalMem && totalMem <= 64) {
+      return 8;
+    }
+    if (64 < totalMem && totalMem <= 72) {
+      return 8;
+    }
+    if (72 < totalMem && totalMem <= 96) {
+      return 16;
+    }
+    if (96 < totalMem && totalMem <= 128) {
+      return 24;
+    }
+    if (128 < totalMem && totalMem <= 256) {
+      return 32;
+    }
+    if (256 < totalMem) {
+      return 64;
+    }
     return -1;
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/9c1c7f93/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json
index 4e62eab..f5a10f6 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json
@@ -235,14 +235,14 @@
           },
           "reporting": {
             "ok": {
-              "text": "Capacity Used:[{2:d}%, {0}], Capacity Remaining:[{1}]"
+              "text": "Capacity Used:[{2:.0f}%, {0}], Capacity Remaining:[{1}]"
             },
             "warning": {
-              "text": "Capacity Used:[{2:d}%, {0}], Capacity Remaining:[{1}]",
+              "text": "Capacity Used:[{2:.0f}%, {0}], Capacity Remaining:[{1}]",
               "value": 80
             },          
             "critical": {
-              "text": "Capacity Used:[{2:d}%, {0}], Capacity Remaining:[{1}]",
+              "text": "Capacity Used:[{2:.0f}%, {0}], Capacity Remaining:[{1}]",
               "value": 90
             },
             "units" : "%"
@@ -252,7 +252,7 @@
               "Hadoop:service=NameNode,name=FSNamesystemState/CapacityUsed",
               "Hadoop:service=NameNode,name=FSNamesystemState/CapacityRemaining"
             ],
-            "value": "{0}/({0} + {1}) * 100"
+            "value": "{0}/({0} + {1}) * 100.0"
           }
         }
       },
@@ -627,14 +627,14 @@
           },
           "reporting": {
             "ok": {
-              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:d}% Used, {1}]"
+              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:.0f}% Used, {1}]"
             },
             "warning": {
-              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:d}% Used, {1}]",
+              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:.0f}% Used, {1}]",
               "value": 80
             },
             "critical": {
-              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:d}% Used, {1}]",
+              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:.0f}% Used, {1}]",
               "value": 90
             },
             "units" : "%"
@@ -644,7 +644,7 @@
               "Hadoop:service=DataNode,name=FSDatasetState-*/Remaining",
               "Hadoop:service=DataNode,name=FSDatasetState-*/Capacity"
             ],
-            "value": "({1} - {0})/{1} * 100"
+            "value": "({1} - {0})/{1} * 100.0"
           }
         }
       }    

http://git-wip-us.apache.org/repos/asf/ambari/blob/9c1c7f93/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/alerts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/alerts.json
b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/alerts.json
index cee8209..2d03a8e 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/alerts.json
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/alerts.json
@@ -229,14 +229,14 @@
           },
           "reporting": {
             "ok": {
-              "text": "Capacity Used:[{2:d}%, {0}], Capacity Remaining:[{1}]"
+              "text": "Capacity Used:[{2:.0f}%, {0}], Capacity Remaining:[{1}]"
             },
             "warning": {
-              "text": "Capacity Used:[{2:d}%, {0}], Capacity Remaining:[{1}]",
+              "text": "Capacity Used:[{2:.0f}%, {0}], Capacity Remaining:[{1}]",
               "value": 80
             },
             "critical": {
-              "text": "Capacity Used:[{2:d}%, {0}], Capacity Remaining:[{1}]",
+              "text": "Capacity Used:[{2:.0f}%, {0}], Capacity Remaining:[{1}]",
               "value": 90
             },
             "units" : "%"
@@ -246,7 +246,7 @@
               "Hadoop:service=NameNode,name=FSNamesystemState/CapacityUsed",
               "Hadoop:service=NameNode,name=FSNamesystemState/CapacityRemaining"
             ],
-            "value": "{0}/({0} + {1}) * 100"
+            "value": "{0}/({0} + {1}) * 100.0"
           }
         }
       },
@@ -583,14 +583,14 @@
           },
           "reporting": {
             "ok": {
-              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:d}% Used, {1}]"
+              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:.0f}% Used, {1}]"
             },
             "warning": {
-              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:d}% Used, {1}]",
+              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:.0f}% Used, {1}]",
               "value": 80
             },
             "critical": {
-              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:d}% Used, {1}]",
+              "text": "Remaining Capacity:[{0}], Total Capacity:[{2:.0f}% Used, {1}]",
               "value": 90
             },
             "units" : "%"
@@ -600,7 +600,7 @@
               "Hadoop:service=DataNode,name=FSDatasetState-*/Remaining",
               "Hadoop:service=DataNode,name=FSDatasetState-*/Capacity"
             ],
-            "value": "({1} - {0})/{1} * 100"
+            "value": "({1} - {0})/{1} * 100.0"
           }
         }
       }


Mime
View raw message