Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 08ADF200B50 for ; Fri, 29 Jul 2016 09:27:07 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 0732F160A79; Fri, 29 Jul 2016 07:27:07 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 29916160A61 for ; Fri, 29 Jul 2016 09:27:06 +0200 (CEST) Received: (qmail 25390 invoked by uid 500); 29 Jul 2016 07:27:05 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 25381 invoked by uid 99); 29 Jul 2016 07:27:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 29 Jul 2016 07:27:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 42EA4E0103; Fri, 29 Jul 2016 07:27:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dsen@apache.org To: commits@ambari.apache.org Message-Id: <54c8b2fc4b5f4a83bbff50ebe562abff@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen) Date: Fri, 29 Jul 2016 07:27:05 +0000 (UTC) archived-at: Fri, 29 Jul 2016 07:27:07 -0000 Repository: ambari Updated Branches: refs/heads/branch-2.4 fefb156ac -> 5e929b9bb AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5e929b9b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5e929b9b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5e929b9b Branch: refs/heads/branch-2.4 Commit: 5e929b9bbff338be7458f33034184c7fb9af13ee Parents: fefb156 Author: Dmytro Sen Authored: Fri Jul 29 10:26:05 2016 +0300 Committer: Dmytro Sen Committed: Fri Jul 29 10:26:58 2016 +0300 ---------------------------------------------------------------------- .../stacks/HDP/2.2/services/stack_advisor.py | 14 ++++++++------ .../stacks/2.2/common/test_stack_advisor.py | 20 +++----------------- 2 files changed, 11 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5e929b9b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index b01af59..24447a3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -91,12 +91,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): # Above is the default calculated 'maximum' values derived purely from hosts. # However, there are 'maximum' and other attributes that actually change based on the values # of other configs. We need to update those values. - - # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"]. - # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation. - # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] - # in this case (if 'changed-configurations' are empty). - if ('changed-configurations' in services.keys() and services['changed-configurations'] and "yarn-site" in services["configurations"]): + if ("yarn-site" in services["configurations"]): if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]): putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) @@ -383,9 +378,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor): container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size, yarnMaxAllocationSize) putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), container_size)) + putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + if "yarn-site" in services["configurations"]: + if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: + putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) + if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: + putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + putHiveSiteProperty("hive.prewarm.enabled", "false") putHiveSiteProperty("hive.prewarm.numcontainers", "3") putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true") http://git-wip-us.apache.org/repos/asf/ambari/blob/5e929b9b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 658d818..75a26d2 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1162,24 +1162,10 @@ class TestHDP22StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - # With no 'changed-configurations', we should get updated 'maximum's from recommendations - # Else update from services. - # - # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"]. - # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation. - # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] - # in this case (if 'changed-configurations' are empty). - # - # Test not empty 'changed-configurations': - services["changed-configurations"] = [ - { - "type": "yarn-env", - "name": "min_user_id" - } - ] + # Test - with no 'changed-configurations', we should get updated 'maximum's. + services.pop("changed-configurations", None) services.pop("configurations", None) - services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}, - "yarn-env": {"properties": {"min_user_id": "500"}}} + services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}} expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321'