Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id EB43711809 for ; Thu, 28 Aug 2014 17:49:37 +0000 (UTC) Received: (qmail 7545 invoked by uid 500); 28 Aug 2014 17:49:37 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 7498 invoked by uid 500); 28 Aug 2014 17:49:37 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 7404 invoked by uid 99); 28 Aug 2014 17:49:37 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Aug 2014 17:49:37 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 7D201A04D3C; Thu, 28 Aug 2014 17:49:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jonathanhurley@apache.org To: commits@ambari.apache.org Date: Thu, 28 Aug 2014 17:50:00 -0000 Message-Id: In-Reply-To: <2e4152e5cdac41a598a3a33c6457d634@git.apache.org> References: <2e4152e5cdac41a598a3a33c6457d634@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [25/45] git commit: AMBARI-7047. BE: Config validation type of values less than recommended should be WARNs AMBARI-7047. BE: Config validation type of values less than recommended should be WARNs Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fa10e3bf Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fa10e3bf Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fa10e3bf Branch: refs/heads/branch-alerts-dev Commit: fa10e3bff6f18aeee1d93eecf1f15ad0d7657235 Parents: d87cf7f Author: Srimanth Gunturi Authored: Wed Aug 27 13:51:39 2014 -0700 Committer: Srimanth Gunturi Committed: Wed Aug 27 18:31:29 2014 -0700 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 56 +++++++++++--------- .../stacks/HDP/2.1/services/stack_advisor.py | 14 ++--- .../stacks/2.0.6/common/test_stack_advisor.py | 36 ++++++++++--- 3 files changed, 67 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fa10e3bf/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index ad0dcc0..083286e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -415,58 +415,66 @@ class HDP206StackAdvisor(StackAdvisor): "YARN": ["yarn-site", self.validateYARNConfigurations] }.get(serviceName, None) - def toConfigurationValidationErrors(self, items, siteName): + def toConfigurationValidationProblems(self, validationProblems, siteName): result = [] - for item in items: - if item["message"] is not None: - error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type": siteName, "config-name": item["config-name"] } - result.append(error) + for validationProblem in validationProblems: + validationItem = validationProblem.get("item", None) + if validationItem is not None: + problem = { "type": 'configuration', "level": validationItem["level"], "message": validationItem["message"], + "config-type": siteName, "config-name": validationProblem["config-name"] } + result.append(problem) return result + def getWarnItem(self, message): + return {"level": "WARN", "message": message} + + def getErrorItem(self, message): + return {"level": "ERROR", "message": message} + def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName): if not propertyName in properties: - return "Value should be set" + return self.getErrorItem("Value should be set") value = to_number(properties[propertyName]) if value is None: - return "Value should be integer" + return self.getErrorItem("Value should be integer") defaultValue = to_number(recommendedDefaults[propertyName]) if defaultValue is None: return None if value < defaultValue: - return "Value is less than the recommended default of {0}".format(defaultValue) + return self.getWarnItem("Value is less than the recommended default of {0}".format(defaultValue)) return None def validateXmxValue(self, properties, recommendedDefaults, propertyName): if not propertyName in properties: - return "Value should be set" + return self.getErrorItem("Value should be set") value = properties[propertyName] defaultValue = recommendedDefaults[propertyName] if defaultValue is None: - return "Config's default value can't be null or undefined" + return self.getErrorItem("Config's default value can't be null or undefined") if not checkXmxValueFormat(value): - return 'Invalid value format' + return self.getErrorItem('Invalid value format') valueInt = formatXmxSizeToBytes(getXmxSize(value)) defaultValueXmx = getXmxSize(defaultValue) defaultValueInt = formatXmxSizeToBytes(defaultValueXmx) if valueInt < defaultValueInt: - return "Value is less than the recommended default of -Xmx" + defaultValueXmx + return self.getWarnItem("Value is less than the recommended default of -Xmx" + defaultValueXmx) return None def validateMapReduce2Configurations(self, properties, recommendedDefaults): - validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')}, - {"config-name": 'mapreduce.reduce.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')}, - {"config-name": 'mapreduce.task.io.sort.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')}, - {"config-name": 'mapreduce.map.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')}, - {"config-name": 'mapreduce.reduce.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')}, - {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}, - {"config-name": 'yarn.app.mapreduce.am.command-opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ] - return self.toConfigurationValidationErrors(validationItems, "mapred-site") + validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')}, + {"config-name": 'mapreduce.reduce.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')}, + {"config-name": 'mapreduce.task.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')}, + {"config-name": 'mapreduce.map.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')}, + {"config-name": 'mapreduce.reduce.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')}, + {"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}, + {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ] + return self.toConfigurationValidationProblems(validationItems, "mapred-site") def validateYARNConfigurations(self, properties, recommendedDefaults): - validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')}, - {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')}, - {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ] - return self.toConfigurationValidationErrors(validationItems, "yarn-site") + validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')}, + {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')}, + {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ] + return self.toConfigurationValidationProblems(validationItems, "yarn-site") # Validation helper methods http://git-wip-us.apache.org/repos/asf/ambari/blob/fa10e3bf/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py index e4168d7..98fee5d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -89,14 +89,14 @@ class HDP21StackAdvisor(HDP206StackAdvisor): return validator def validateHiveConfigurations(self, properties, recommendedDefaults): - validationItems = [ {"config-name": 'hive.tez.container.size', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')}, - {"config-name": 'hive.tez.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')}, - {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ] - return self.toConfigurationValidationErrors(validationItems, "hive-site") + validationItems = [ {"config-name": 'hive.tez.container.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')}, + {"config-name": 'hive.tez.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')}, + {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ] + return self.toConfigurationValidationProblems(validationItems, "hive-site") def validateTezConfigurations(self, properties, recommendedDefaults): - validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, - {"config-name": 'tez.am.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ] - return self.toConfigurationValidationErrors(validationItems, "tez-site") + validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, + {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ] + return self.toConfigurationValidationProblems(validationItems, "tez-site") http://git-wip-us.apache.org/repos/asf/ambari/blob/fa10e3bf/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index d32fa56..898797a 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -99,9 +99,9 @@ class TestHDP206StackAdvisor(TestCase): result = self.stackAdvisor.validateComponentLayout(services, hosts) expectedItems = [ - {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host": "host1"}, - {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host": "host1"}, - {"message": "Host is not used", "host": "host2"} + {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "level": "WARN", "host": "host1"}, + {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "level": "WARN", "host": "host1"}, + {"message": "Host is not used", "level": "ERROR", "host": "host2"} ] self.assertValidationResult(expectedItems, result) @@ -120,7 +120,27 @@ class TestHDP206StackAdvisor(TestCase): result = self.stackAdvisor.validateComponentLayout(services, hosts) expectedItems = [ - {"message": "Cardinality violation, cardinality=ALL, hosts count=1"} + {"message": "Cardinality violation, cardinality=ALL, hosts count=1", "level": "ERROR"} + ] + self.assertValidationResult(expectedItems, result) + + def test_validationWarnMessagesIfLessThanDefault(self): + servicesInfo = [ + { + "name": "YARN", + "components": [] + } + ] + services = self.prepareServices(servicesInfo) + services["configurations"] = {"yarn-site":{"properties":{"yarn.nodemanager.resource.memory-mb": "0", + "yarn.scheduler.minimum-allocation-mb": "str"}}} + hosts = self.prepareHosts([]) + result = self.stackAdvisor.validateConfigurations(services, hosts) + + expectedItems = [ + {"message": "Value is less than the recommended default of 2046", "level": "WARN"}, + {"message": "Value should be integer", "level": "ERROR"}, + {"message": "Value should be set", "level": "ERROR"} ] self.assertValidationResult(expectedItems, result) @@ -139,7 +159,7 @@ class TestHDP206StackAdvisor(TestCase): result = self.stackAdvisor.validateComponentLayout(services, hosts) expectedItems = [ - {"message": "Host is not used", "host": "host1"} + {"message": "Host is not used", "host": "host1", "level": "ERROR"} ] self.assertValidationResult(expectedItems, result) @@ -157,7 +177,7 @@ class TestHDP206StackAdvisor(TestCase): result = self.stackAdvisor.validateComponentLayout(services, hosts) expectedItems = [ - {"message": "Cardinality violation, cardinality=0-1, hosts count=2"} + {"message": "Cardinality violation, cardinality=0-1, hosts count=2", "level": "ERROR"} ] self.assertValidationResult(expectedItems, result) @@ -175,7 +195,7 @@ class TestHDP206StackAdvisor(TestCase): result = self.stackAdvisor.validateComponentLayout(services, hosts) expectedItems = [ - {"message": "Host is not used", "host": "host2"} + {"message": "Host is not used", "host": "host2", "level": "ERROR"} ] self.assertValidationResult(expectedItems, result) @@ -244,7 +264,7 @@ class TestHDP206StackAdvisor(TestCase): def assertValidationResult(self, expectedItems, result): actualItems = [] for item in result["items"]: - next = { "message": item["message"] } + next = {"message": item["message"], "level": item["level"]} try: next["host"] = item["host"] except KeyError, err: