ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen)
Date Fri, 29 Jul 2016 07:27:05 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 fefb156ac -> 5e929b9bb


AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency
change (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5e929b9b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5e929b9b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5e929b9b

Branch: refs/heads/branch-2.4
Commit: 5e929b9bbff338be7458f33034184c7fb9af13ee
Parents: fefb156
Author: Dmytro Sen <dsen@apache.org>
Authored: Fri Jul 29 10:26:05 2016 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Fri Jul 29 10:26:58 2016 +0300

----------------------------------------------------------------------
 .../stacks/HDP/2.2/services/stack_advisor.py    | 14 ++++++++------
 .../stacks/2.2/common/test_stack_advisor.py     | 20 +++-----------------
 2 files changed, 11 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/5e929b9b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index b01af59..24447a3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -91,12 +91,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       # Above is the default calculated 'maximum' values derived purely from hosts.
       # However, there are 'maximum' and other attributes that actually change based on the
values
       #  of other configs. We need to update those values.
-
-      # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
-      # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
is always 5120 during first recommendation.
-      # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
-      # in this case (if 'changed-configurations' are empty).
-      if ('changed-configurations' in services.keys() and services['changed-configurations']
and "yarn-site" in services["configurations"]):
+      if ("yarn-site" in services["configurations"]):
         if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]):
           putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
           putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
@@ -383,9 +378,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size,
yarnMaxAllocationSize)
 
     putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]),
container_size))
+
     putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
     putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
 
+    if "yarn-site" in services["configurations"]:
+      if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
+        putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
+      if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
+        putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+
     putHiveSiteProperty("hive.prewarm.enabled", "false")
     putHiveSiteProperty("hive.prewarm.numcontainers", "3")
     putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true")

http://git-wip-us.apache.org/repos/asf/ambari/blob/5e929b9b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 658d818..75a26d2 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1162,24 +1162,10 @@ class TestHDP22StackAdvisor(TestCase):
     self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
     self.assertEquals(configurations, expected)
 
-    # With no 'changed-configurations', we should get updated 'maximum's from recommendations
-    # Else update from services.
-    #
-    # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
-    # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
is always 5120 during first recommendation.
-    # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
-    # in this case (if 'changed-configurations' are empty).
-    #
-    # Test not empty 'changed-configurations':
-    services["changed-configurations"] = [
-      {
-        "type": "yarn-env",
-        "name": "min_user_id"
-      }
-    ]
+    # Test - with no 'changed-configurations', we should get updated 'maximum's.
+    services.pop("changed-configurations", None)
     services.pop("configurations", None)
-    services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb":
'4321', "yarn.nodemanager.resource.cpu-vcores": '9'}},
-                                  "yarn-env": {"properties": {"min_user_id": "500"}}}
+    services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb":
'4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}}
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"]
= '9'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"]
= '9'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"]
= '4321'


Mime
View raw message