Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 62D4A183F1 for ; Thu, 1 Oct 2015 19:15:10 +0000 (UTC) Received: (qmail 94799 invoked by uid 500); 1 Oct 2015 19:14:48 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 94772 invoked by uid 500); 1 Oct 2015 19:14:48 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 94763 invoked by uid 99); 1 Oct 2015 19:14:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 01 Oct 2015 19:14:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E0E42E154C; Thu, 1 Oct 2015 19:14:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aonishuk@apache.org To: commits@ambari.apache.org Date: Thu, 01 Oct 2015 19:14:47 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] ambari git commit: AMBARI-13286. During installation of Storm/Atlas, Ambari overwrites and warns about wrong dependent configuration (aonishuk) Repository: ambari Updated Branches: refs/heads/branch-2.1 94e7bf1f8 -> 97d7c2a8c refs/heads/trunk 8cd16dc8a -> 2df0abec1 AMBARI-13286. During installation of Storm/Atlas, Ambari overwrites and warns about wrong dependent configuration (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2df0abec Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2df0abec Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2df0abec Branch: refs/heads/trunk Commit: 2df0abec1cd3b2651712abe04861ce4b5e4e8b48 Parents: 8cd16dc Author: Andrew Onishuk Authored: Thu Oct 1 22:14:21 2015 +0300 Committer: Andrew Onishuk Committed: Thu Oct 1 22:14:21 2015 +0300 ---------------------------------------------------------------------- .../stacks/HDP/2.2/services/stack_advisor.py | 7 +- .../stacks/2.2/common/test_stack_advisor.py | 102 +++++++++---------- .../stacks/2.3/common/test_stack_advisor.py | 2 +- 3 files changed, 58 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2df0abec/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 8bf0b32..91d163d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -352,7 +352,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveSiteProperty("hive.server2.enable.doAs", "true") yarn_queues = "default" - capacitySchedulerProperties = services['configurations']["capacity-scheduler"]["properties"] if "capacity-scheduler" in services['configurations'] else {} + capacitySchedulerProperties = {} + if "capacity-scheduler" in services['configurations'] and "capacity-scheduler" in services['configurations']["capacity-scheduler"]["properties"]: + properties = str(services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"]).split('\n') + for property in properties: + key,sep,value = property.partition("=") + capacitySchedulerProperties[key] = value if "yarn.scheduler.capacity.root.queues" in capacitySchedulerProperties: yarn_queues = str(capacitySchedulerProperties["yarn.scheduler.capacity.root.queues"]) # Interactive Queues property attributes http://git-wip-us.apache.org/repos/asf/ambari/blob/2df0abec/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 97cf4ca..aa9a302 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -993,7 +993,7 @@ class TestHDP22StackAdvisor(TestCase): 'hive-site': { 'properties': { 'hive.server2.enable.doAs': 'true', - 'hive.server2.tez.default.queues': "queue1,queue2", + 'hive.server2.tez.default.queues': "default", 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', 'hive.auto.convert.join.noconditionaltask.size': '268435456', @@ -1040,7 +1040,7 @@ class TestHDP22StackAdvisor(TestCase): 'hive.server2.authentication.kerberos.keytab': {'delete': 'true'}, 'hive.server2.authentication.ldap.url': {'delete': 'true'}, 'hive.server2.tez.default.queues': { - 'entries': [{'value': 'queue1', 'label': 'queue1 queue'}, {'value': 'queue2', 'label': 'queue2 queue'}] + 'entries': [{'value': 'default', 'label': 'default queue'}] } } }, @@ -1307,55 +1307,55 @@ class TestHDP22StackAdvisor(TestCase): # test 'hive.server2.tez.default.queues' leaf queues services["configurations"]['capacity-scheduler']['properties'] = { - "yarn.scheduler.capacity.maximum-am-resource-percent": "0.2", - "yarn.scheduler.capacity.maximum-applications": "10000", - "yarn.scheduler.capacity.node-locality-delay": "40", - "yarn.scheduler.capacity.queue-mappings-override.enable": "false", - "yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator", - "yarn.scheduler.capacity.root.accessible-node-labels": "*", - "yarn.scheduler.capacity.root.acl_administer_queue": "*", - "yarn.scheduler.capacity.root.capacity": "100", - "yarn.scheduler.capacity.root.default.a.a1.acl_administer_queue": "*", - "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications": "*", - "yarn.scheduler.capacity.root.default.a.a1.capacity": "75", - "yarn.scheduler.capacity.root.default.a.a1.maximum-capacity": "100", - "yarn.scheduler.capacity.root.default.a.a1.minimum-user-limit-percent": "100", - "yarn.scheduler.capacity.root.default.a.a1.ordering-policy": "fifo", - "yarn.scheduler.capacity.root.default.a.a1.state": "RUNNING", - "yarn.scheduler.capacity.root.default.a.a1.user-limit-factor": "1", - "yarn.scheduler.capacity.root.default.a.a2.acl_administer_queue": "*", - "yarn.scheduler.capacity.root.default.a.a2.acl_submit_applications": "*", - "yarn.scheduler.capacity.root.default.a.a2.capacity": "25", - "yarn.scheduler.capacity.root.default.a.a2.maximum-capacity": "25", - "yarn.scheduler.capacity.root.default.a.a2.minimum-user-limit-percent": "100", - "yarn.scheduler.capacity.root.default.a.a2.ordering-policy": "fifo", - "yarn.scheduler.capacity.root.default.a.a2.state": "RUNNING", - "yarn.scheduler.capacity.root.default.a.a2.user-limit-factor": "1", - "yarn.scheduler.capacity.root.default.a.acl_administer_queue": "*", - "yarn.scheduler.capacity.root.default.a.acl_submit_applications": "*", - "yarn.scheduler.capacity.root.default.a.capacity": "50", - "yarn.scheduler.capacity.root.default.a.maximum-capacity": "100", - "yarn.scheduler.capacity.root.default.a.minimum-user-limit-percent": "100", - "yarn.scheduler.capacity.root.default.a.ordering-policy": "fifo", - "yarn.scheduler.capacity.root.default.a.queues": "a1,a2", - "yarn.scheduler.capacity.root.default.a.state": "RUNNING", - "yarn.scheduler.capacity.root.default.a.user-limit-factor": "1", - "yarn.scheduler.capacity.root.default.acl_submit_applications": "*", - "yarn.scheduler.capacity.root.default.b.acl_administer_queue": "*", - "yarn.scheduler.capacity.root.default.b.acl_submit_applications": "*", - "yarn.scheduler.capacity.root.default.b.capacity": "50", - "yarn.scheduler.capacity.root.default.b.maximum-capacity": "50", - "yarn.scheduler.capacity.root.default.b.minimum-user-limit-percent": "100", - "yarn.scheduler.capacity.root.default.b.ordering-policy": "fifo", - "yarn.scheduler.capacity.root.default.b.state": "RUNNING", - "yarn.scheduler.capacity.root.default.b.user-limit-factor": "1", - "yarn.scheduler.capacity.root.default.capacity": "100", - "yarn.scheduler.capacity.root.default.maximum-capacity": "100", - "yarn.scheduler.capacity.root.default.queues": "a,b", - "yarn.scheduler.capacity.root.default.state": "RUNNING", - "yarn.scheduler.capacity.root.default.user-limit-factor": "1", - "yarn.scheduler.capacity.root.queues": "default" - } + "capacity-scheduler" : "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" + "yarn.scheduler.capacity.maximum-applications=10000\n" + "yarn.scheduler.capacity.node-locality-delay=40\n" + "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" + "yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator\n" + "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.default.a.a1.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.a.a1.capacity=75\n" + "yarn.scheduler.capacity.root.default.a.a1.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.a.a1.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.a.a1.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.a.a1.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.a.a1.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.a.a2.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.a.a2.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.a.a2.capacity=25\n" + "yarn.scheduler.capacity.root.default.a.a2.maximum-capacity=25\n" + "yarn.scheduler.capacity.root.default.a.a2.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.a.a2.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.a.a2.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.a.a2.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.a.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.a.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.a.capacity=50\n" + "yarn.scheduler.capacity.root.default.a.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.a.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.a.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.a.queues=a1,a2\n" + "yarn.scheduler.capacity.root.default.a.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.a.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.b.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.b.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.b.capacity=50\n" + "yarn.scheduler.capacity.root.default.b.maximum-capacity=50\n" + "yarn.scheduler.capacity.root.default.b.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.b.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.b.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.b.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.capacity=100\n" + "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.queues=a,b\n" + "yarn.scheduler.capacity.root.default.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.queues=default"} + expected['hive-site']['properties']['hive.server2.tez.default.queues'] = 'default.a.a1,default.a.a2,default.b' expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'] = { 'entries': [{'value': 'default.a.a1', 'label': 'default.a.a1 queue'}, {'value': 'default.a.a2', 'label': 'default.a.a2 queue'}, {'value': 'default.b', 'label': 'default.b queue'}] http://git-wip-us.apache.org/repos/asf/ambari/blob/2df0abec/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 680dcde..e0c6d28 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -417,7 +417,7 @@ class TestHDP23StackAdvisor(TestCase): "configurations": { "capacity-scheduler": { "properties": { - "yarn.scheduler.capacity.root.queues": "queue1,queue2" + "capacity-scheduler" :"yarn.scheduler.capacity.root.queues=queue1,queue2" } }, "hive-env": {