ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [25/45] git commit: AMBARI-7047. BE: Config validation type of values less than recommended should be WARNs
Date Thu, 28 Aug 2014 17:50:00 GMT
AMBARI-7047. BE: Config validation type of values less than recommended should be WARNs


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fa10e3bf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fa10e3bf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fa10e3bf

Branch: refs/heads/branch-alerts-dev
Commit: fa10e3bff6f18aeee1d93eecf1f15ad0d7657235
Parents: d87cf7f
Author: Srimanth Gunturi <sgunturi@hortonworks.com>
Authored: Wed Aug 27 13:51:39 2014 -0700
Committer: Srimanth Gunturi <sgunturi@hortonworks.com>
Committed: Wed Aug 27 18:31:29 2014 -0700

----------------------------------------------------------------------
 .../stacks/HDP/2.0.6/services/stack_advisor.py  | 56 +++++++++++---------
 .../stacks/HDP/2.1/services/stack_advisor.py    | 14 ++---
 .../stacks/2.0.6/common/test_stack_advisor.py   | 36 ++++++++++---
 3 files changed, 67 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/fa10e3bf/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index ad0dcc0..083286e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -415,58 +415,66 @@ class HDP206StackAdvisor(StackAdvisor):
       "YARN": ["yarn-site", self.validateYARNConfigurations]
     }.get(serviceName, None)
 
-  def toConfigurationValidationErrors(self, items, siteName):
+  def toConfigurationValidationProblems(self, validationProblems, siteName):
     result = []
-    for item in items:
-      if item["message"] is not None:
-        error = { "type": 'configuration', "level": 'ERROR', "message": item["message"],
"config-type": siteName, "config-name": item["config-name"] }
-        result.append(error)
+    for validationProblem in validationProblems:
+      validationItem = validationProblem.get("item", None)
+      if validationItem is not None:
+        problem = { "type": 'configuration', "level": validationItem["level"], "message":
validationItem["message"],
+                  "config-type": siteName, "config-name": validationProblem["config-name"]
}
+        result.append(problem)
     return result
 
+  def getWarnItem(self, message):
+    return {"level": "WARN", "message": message}
+
+  def getErrorItem(self, message):
+    return {"level": "ERROR", "message": message}
+
   def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName):
     if not propertyName in properties:
-      return "Value should be set"
+      return self.getErrorItem("Value should be set")
     value = to_number(properties[propertyName])
     if value is None:
-      return "Value should be integer"
+      return self.getErrorItem("Value should be integer")
     defaultValue = to_number(recommendedDefaults[propertyName])
     if defaultValue is None:
       return None
     if value < defaultValue:
-      return "Value is less than the recommended default of {0}".format(defaultValue)
+      return self.getWarnItem("Value is less than the recommended default of {0}".format(defaultValue))
     return None
 
   def validateXmxValue(self, properties, recommendedDefaults, propertyName):
     if not propertyName in properties:
-      return "Value should be set"
+      return self.getErrorItem("Value should be set")
     value = properties[propertyName]
     defaultValue = recommendedDefaults[propertyName]
     if defaultValue is None:
-      return "Config's default value can't be null or undefined"
+      return self.getErrorItem("Config's default value can't be null or undefined")
     if not checkXmxValueFormat(value):
-      return 'Invalid value format'
+      return self.getErrorItem('Invalid value format')
     valueInt = formatXmxSizeToBytes(getXmxSize(value))
     defaultValueXmx = getXmxSize(defaultValue)
     defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
     if valueInt < defaultValueInt:
-      return "Value is less than the recommended default of -Xmx" + defaultValueXmx
+      return self.getWarnItem("Value is less than the recommended default of -Xmx" + defaultValueXmx)
     return None
 
   def validateMapReduce2Configurations(self, properties, recommendedDefaults):
-    validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'mapreduce.map.java.opts')},
-                        {"config-name": 'mapreduce.reduce.java.opts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'mapreduce.reduce.java.opts')},
-                        {"config-name": 'mapreduce.task.io.sort.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.task.io.sort.mb')},
-                        {"config-name": 'mapreduce.map.memory.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.map.memory.mb')},
-                        {"config-name": 'mapreduce.reduce.memory.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.reduce.memory.mb')},
-                        {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
-                        {"config-name": 'yarn.app.mapreduce.am.command-opts', "message":
self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')}
]
-    return self.toConfigurationValidationErrors(validationItems, "mapred-site")
+    validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties,
recommendedDefaults, 'mapreduce.map.java.opts')},
+                        {"config-name": 'mapreduce.reduce.java.opts', "item": self.validateXmxValue(properties,
recommendedDefaults, 'mapreduce.reduce.java.opts')},
+                        {"config-name": 'mapreduce.task.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.task.io.sort.mb')},
+                        {"config-name": 'mapreduce.map.memory.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.map.memory.mb')},
+                        {"config-name": 'mapreduce.reduce.memory.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.reduce.memory.mb')},
+                        {"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
+                        {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
+    return self.toConfigurationValidationProblems(validationItems, "mapred-site")
 
   def validateYARNConfigurations(self, properties, recommendedDefaults):
-    validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message":
self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
-                        {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message":
self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
-                        {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message":
self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')}
]
-    return self.toConfigurationValidationErrors(validationItems, "yarn-site")
+    validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
+                        {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
+                        {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
+    return self.toConfigurationValidationProblems(validationItems, "yarn-site")
 
 
 # Validation helper methods

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa10e3bf/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index e4168d7..98fee5d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -89,14 +89,14 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
       return validator
 
   def validateHiveConfigurations(self, properties, recommendedDefaults):
-    validationItems = [ {"config-name": 'hive.tez.container.size', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'hive.tez.container.size')},
-                        {"config-name": 'hive.tez.java.opts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'hive.tez.java.opts')},
-                        {"config-name": 'hive.auto.convert.join.noconditionaltask.size',
"message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')}
]
-    return self.toConfigurationValidationErrors(validationItems, "hive-site")
+    validationItems = [ {"config-name": 'hive.tez.container.size', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'hive.tez.container.size')},
+                        {"config-name": 'hive.tez.java.opts', "item": self.validateXmxValue(properties,
recommendedDefaults, 'hive.tez.java.opts')},
+                        {"config-name": 'hive.auto.convert.join.noconditionaltask.size',
"item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')}
]
+    return self.toConfigurationValidationProblems(validationItems, "hive-site")
 
   def validateTezConfigurations(self, properties, recommendedDefaults):
-    validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.am.resource.memory.mb')},
-                        {"config-name": 'tez.am.java.opts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'tez.am.java.opts')} ]
-    return self.toConfigurationValidationErrors(validationItems, "tez-site")
+    validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.am.resource.memory.mb')},
+                        {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties,
recommendedDefaults, 'tez.am.java.opts')} ]
+    return self.toConfigurationValidationProblems(validationItems, "tez-site")
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa10e3bf/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index d32fa56..898797a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -99,9 +99,9 @@ class TestHDP206StackAdvisor(TestCase):
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
     expectedItems = [
-      {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host":
"host1"},
-      {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host":
"host1"},
-      {"message": "Host is not used", "host": "host2"}
+      {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "level":
"WARN", "host": "host1"},
+      {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "level":
"WARN", "host": "host1"},
+      {"message": "Host is not used", "level": "ERROR", "host": "host2"}
     ]
     self.assertValidationResult(expectedItems, result)
 
@@ -120,7 +120,27 @@ class TestHDP206StackAdvisor(TestCase):
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
     expectedItems = [
-      {"message": "Cardinality violation, cardinality=ALL, hosts count=1"}
+      {"message": "Cardinality violation, cardinality=ALL, hosts count=1", "level": "ERROR"}
+    ]
+    self.assertValidationResult(expectedItems, result)
+
+  def test_validationWarnMessagesIfLessThanDefault(self):
+    servicesInfo = [
+      {
+        "name": "YARN",
+        "components": []
+      }
+    ]
+    services = self.prepareServices(servicesInfo)
+    services["configurations"] = {"yarn-site":{"properties":{"yarn.nodemanager.resource.memory-mb":
"0",
+                                                             "yarn.scheduler.minimum-allocation-mb":
"str"}}}
+    hosts = self.prepareHosts([])
+    result = self.stackAdvisor.validateConfigurations(services, hosts)
+
+    expectedItems = [
+      {"message": "Value is less than the recommended default of 2046", "level": "WARN"},
+      {"message": "Value should be integer", "level": "ERROR"},
+      {"message": "Value should be set", "level": "ERROR"}
     ]
     self.assertValidationResult(expectedItems, result)
 
@@ -139,7 +159,7 @@ class TestHDP206StackAdvisor(TestCase):
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
     expectedItems = [
-      {"message": "Host is not used", "host": "host1"}
+      {"message": "Host is not used", "host": "host1", "level": "ERROR"}
     ]
     self.assertValidationResult(expectedItems, result)
 
@@ -157,7 +177,7 @@ class TestHDP206StackAdvisor(TestCase):
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
     expectedItems = [
-      {"message": "Cardinality violation, cardinality=0-1, hosts count=2"}
+      {"message": "Cardinality violation, cardinality=0-1, hosts count=2", "level": "ERROR"}
     ]
     self.assertValidationResult(expectedItems, result)
 
@@ -175,7 +195,7 @@ class TestHDP206StackAdvisor(TestCase):
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
     expectedItems = [
-      {"message": "Host is not used", "host": "host2"}
+      {"message": "Host is not used", "host": "host2", "level": "ERROR"}
     ]
     self.assertValidationResult(expectedItems, result)
 
@@ -244,7 +264,7 @@ class TestHDP206StackAdvisor(TestCase):
   def assertValidationResult(self, expectedItems, result):
     actualItems = []
     for item in result["items"]:
-      next = { "message": item["message"] }
+      next = {"message": item["message"], "level": item["level"]}
       try:
         next["host"] = item["host"]
       except KeyError, err:


Mime
View raw message