ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject ambari git commit: AMBARI-11556. Ambari does not give a warning if tez.am.resource.memory.mb is greater than yarn max container size (aonishuk)
Date Mon, 01 Jun 2015 12:46:47 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk ee96fdebc -> 6f13d10ac


AMBARI-11556. Ambari does not give a warning if tez.am.resource.memory.mb is greater than
yarn max container size  (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6f13d10a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6f13d10a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6f13d10a

Branch: refs/heads/trunk
Commit: 6f13d10ac7b86bb8dc53e7a9da4573fb6116fc49
Parents: ee96fde
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Mon Jun 1 15:46:40 2015 +0300
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Mon Jun 1 15:46:40 2015 +0300

----------------------------------------------------------------------
 .../stacks/HDP/2.2/services/stack_advisor.py    | 16 +++++++++
 .../stacks/2.2/common/test_stack_advisor.py     | 38 ++++++++++++++++++++
 2 files changed, 54 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6f13d10a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 7743fd6..bcadd70 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -582,6 +582,22 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                         {"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.task.resource.memory.mb')},
                         {"config-name": 'tez.runtime.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.runtime.io.sort.mb')},
                         {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item":
self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},]
+
+    tez_site = properties
+    prop_name1 = 'tez.am.resource.memory.mb'
+    prop_name2 = 'tez.task.resource.memory.mb'
+    yarnSiteProperties = getSiteProperties(configurations, "yarn-site")
+    if yarnSiteProperties:
+      yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+      if int(tez_site[prop_name1]) > yarnMaxAllocationSize:
+          validationItems.append({"config-name": prop_name1,
+                                  "item": self.getWarnItem(
+                                      "{0} should be less than YARN max allocation size ({1})".format(prop_name1,
yarnMaxAllocationSize))})
+      if int(tez_site[prop_name2]) > yarnMaxAllocationSize:
+          validationItems.append({"config-name": prop_name2,
+                                  "item": self.getWarnItem(
+                                      "{0} should be less than YARN max allocation size ({1})".format(prop_name2,
yarnMaxAllocationSize))})
+
     return self.toConfigurationValidationProblems(validationItems, "tez-site")
 
   def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):

http://git-wip-us.apache.org/repos/asf/ambari/blob/6f13d10a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index dffca7b..e87a8d0 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2183,6 +2183,44 @@ class TestHDP22StackAdvisor(TestCase):
     self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services,
hosts)
     self.assertEqual("kms://http@myhost1:2222/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
 
+  def test_validateTezConfigurationsEnv(self):
+    configurations = {
+        "yarn-site": {
+            "properties": {
+                "yarn.scheduler.minimum-allocation-mb": "100",
+                "yarn.scheduler.maximum-allocation-mb": "2048"
+            }
+        }
+    }
+
+    recommendedDefaults = {'tez.task.resource.memory.mb': '1024',
+                           'tez.runtime.io.sort.mb' : '256',
+                           'tez.runtime.unordered.output.buffer.size-mb' : '256',
+                           'tez.am.resource.memory.mb' : '1024'}
+
+    properties = {'tez.task.resource.memory.mb': '2050',
+                  'tez.runtime.io.sort.mb' : '256',
+                  'tez.runtime.unordered.output.buffer.size-mb' : '256',
+                  'tez.am.resource.memory.mb' : '2050'}
+
+
+    res_expected = [{'config-name': 'tez.am.resource.memory.mb',
+                 'config-type': 'tez-site',
+                 'level': 'WARN',
+                 'message': "tez.am.resource.memory.mb should be less than YARN max allocation
size (2048)",
+                 'type': 'configuration',
+                 'level': 'WARN'},
+                    {'config-name': 'tez.task.resource.memory.mb',
+                 'config-type': 'tez-site',
+                 'level': 'WARN',
+                 'message': "tez.task.resource.memory.mb should be less than YARN max allocation
size (2048)",
+                 'type': 'configuration',
+                 'level': 'WARN'}]
+
+    res = self.stackAdvisor.validateTezConfigurations(properties, recommendedDefaults, configurations,
'', '')
+    self.assertEquals(res, res_expected)
+
+
   def test_validateHDFSConfigurationsEnv(self):
     configurations = {}
 


Mime
View raw message