ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpapirkovs...@apache.org
Subject [2/2] ambari git commit: AMBARI-17296. Ambari displays a warning about config values not being at optimal values right after a clean install with no customization. (aonishuk via mpapirkovskyy)
Date Mon, 18 Jul 2016 14:57:57 GMT
AMBARI-17296. Ambari displays a warning about config values not being at optimal values right
after a clean install with no customization. (aonishuk via mpapirkovskyy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a0fac264
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a0fac264
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a0fac264

Branch: refs/heads/branch-2.4
Commit: a0fac264d06b3664bd99a47a7c53df905e056e64
Parents: 77c2dc6
Author: Myroslav Papirkovskyi <mpapyrkovskyy@hortonworks.com>
Authored: Mon Jul 18 17:33:24 2016 +0300
Committer: Myroslav Papirkovskyi <mpapyrkovskyy@hortonworks.com>
Committed: Mon Jul 18 17:35:34 2016 +0300

----------------------------------------------------------------------
 .../stacks/HDP/2.2/services/stack_advisor.py    | 11 +++++++++-
 .../stacks/HDP/2.3/services/stack_advisor.py    | 10 ++++++++--
 .../src/main/resources/stacks/stack_advisor.py  |  5 ++++-
 .../stacks/2.2/common/test_stack_advisor.py     | 21 +++++++++++++++++---
 .../stacks/2.3/common/test_stack_advisor.py     |  4 +++-
 5 files changed, 43 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a0fac264/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 1598d0e..1bc5da7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -68,10 +68,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
       putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
       putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
+
       # Above is the default calculated 'maximum' values derived purely from hosts.
       # However, there are 'maximum' and other attributes that actually change based on the
values
       #  of other configs. We need to update those values.
-      if ("yarn-site" in services["configurations"]):
+
+      # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
+      # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
is always 5120 during first recommendation.
+      # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
+      # in this case (if 'changed-configurations' are empty).
+      if ('changed-configurations' in services.keys() and services['changed-configurations']
and "yarn-site" in services["configurations"]):
         if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]):
           putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
           putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
@@ -349,6 +355,9 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size,
yarnMaxAllocationSize)
 
     putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]),
container_size))
+    putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
+    putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+
     putHiveSiteProperty("hive.prewarm.enabled", "false")
     putHiveSiteProperty("hive.prewarm.numcontainers", "3")
     putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true")

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0fac264/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 928fa95..6081a04 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -81,8 +81,14 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     super(HDP23StackAdvisor, self).recommendTezConfigurations(configurations, clusterData,
services, hosts)
 
     putTezProperty = self.putProperty(configurations, "tez-site")
-    if "hive-site" in services["configurations"] and "hive.tez.container.size" in services["configurations"]["hive-site"]["properties"]:
-      putTezProperty("tez.task.resource.memory.mb", services["configurations"]["hive-site"]["properties"]["hive.tez.container.size"])
+
+    if "HIVE" in self.getServiceNames(services):
+      if not "hive-site" in configurations:
+        self.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+
+      if "hive-site" in configurations and "hive.tez.container.size" in configurations["hive-site"]["properties"]:
+        putTezProperty("tez.task.resource.memory.mb", configurations["hive-site"]["properties"]["hive.tez.container.size"])
+
     # remove 2gb limit for tez.runtime.io.sort.mb
     # in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in other case
comment calculation code below
     taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0fac264/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 47f4500..fb30f4d 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -364,7 +364,7 @@ class DefaultStackAdvisor(StackAdvisor):
     stackName = services["Versions"]["stack_name"]
     stackVersion = services["Versions"]["stack_version"]
     hostsList = self.getActiveHosts([host["Hosts"] for host in hosts["items"]])
-    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    servicesList = self.getServiceNames(services)
 
     layoutRecommendations = self.createComponentLayoutRecommendations(services, hosts)
 
@@ -1070,3 +1070,6 @@ class DefaultStackAdvisor(StackAdvisor):
       return (int(cardinality),int(cardinality))
 
     return (None, None)
+
+  def getServiceNames(self, services):
+    return [service["StackServices"]["service_name"] for service in services["services"]]

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0fac264/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 5f272b5..208ee8c 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1136,10 +1136,24 @@ class TestHDP22StackAdvisor(TestCase):
     self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
     self.assertEquals(configurations, expected)
 
-    # Test - with no 'changed-configurations', we should get updated 'maximum's.
-    services.pop("changed-configurations", None)
+    # With no 'changed-configurations', we should get updated 'maximum's from recommendations
+    # Else update from services.
+    #
+    # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
+    # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
is always 5120 during first recommendation.
+    # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
+    # in this case (if 'changed-configurations' are empty).
+    #
+    # Test not empty 'changed-configurations':
+    services["changed-configurations"] = [
+      {
+        "type": "yarn-env",
+        "name": "min_user_id"
+      }
+    ]
     services.pop("configurations", None)
-    services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb":
'4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}}
+    services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb":
'4321', "yarn.nodemanager.resource.cpu-vcores": '9'}},
+                                  "yarn-env": {"properties": {"min_user_id": "500"}}}
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"]
= '9'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"]
= '9'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"]
= '4321'
@@ -1225,6 +1239,7 @@ class TestHDP22StackAdvisor(TestCase):
         },
        'property_attributes': {
          'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
+         'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'},
          'hive.server2.authentication.pam.services': {'delete': 'true'},
          'hive.server2.custom.authentication.class': {'delete': 'true'},
          'hive.server2.authentication.kerberos.principal': {'delete': 'true'},

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0fac264/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index a30d5fc..faad01c 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -663,7 +663,8 @@ class TestHDP23StackAdvisor(TestCase):
           },
          'atlas.cluster.name': {'delete': 'true'},
          'atlas.rest.address': {'delete': 'true'},
-         'datanucleus.rdbms.datastoreAdapterClassName': {'delete': 'true'}
+         'datanucleus.rdbms.datastoreAdapterClassName': {'delete': 'true'},
+         'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'}
         }
       },
       'hiveserver2-site': {
@@ -919,6 +920,7 @@ class TestHDP23StackAdvisor(TestCase):
         },
         'property_attributes': {
           'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
+          'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'},
           'hive.server2.authentication.pam.services': {'delete': 'true'},
           'hive.server2.custom.authentication.class': {'delete': 'true'},
           'hive.server2.authentication.kerberos.principal': {'delete': 'true'},


Mime
View raw message