Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 922FA200B40 for ; Thu, 2 Jun 2016 00:35:10 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 90A05160A4D; Wed, 1 Jun 2016 22:35:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B13B5160A4C for ; Thu, 2 Jun 2016 00:35:09 +0200 (CEST) Received: (qmail 1124 invoked by uid 500); 1 Jun 2016 22:35:08 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 1115 invoked by uid 99); 1 Jun 2016 22:35:08 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Jun 2016 22:35:08 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C89D4E0498; Wed, 1 Jun 2016 22:35:08 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: bhuvnesh2703@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-16992: Recommend hawq_rm_nvcore_limit_perseg in HAWQ service advisor (mithmatt via bhuvnesh2703) Date: Wed, 1 Jun 2016 22:35:08 +0000 (UTC) archived-at: Wed, 01 Jun 2016 22:35:10 -0000 Repository: ambari Updated Branches: refs/heads/branch-2.4 a2e2f8e57 -> c04b543b7 AMBARI-16992: Recommend hawq_rm_nvcore_limit_perseg in HAWQ service advisor (mithmatt via bhuvnesh2703) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c04b543b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c04b543b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c04b543b Branch: refs/heads/branch-2.4 Commit: c04b543b75e5a69473f6a7b5ce0d5506837ff40e Parents: a2e2f8e Author: Bhuvnesh Chaudhary Authored: Wed Jun 1 15:37:08 2016 -0700 Committer: Bhuvnesh Chaudhary Committed: Wed Jun 1 15:37:08 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/service_advisor.py | 4 +++ .../stacks/2.3/HAWQ/test_service_advisor.py | 26 +++++++++++++++++++- .../stacks/2.3/common/test_stack_advisor.py | 12 ++++++--- 3 files changed, 37 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c04b543b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py index 94e1854..32e23f9 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py @@ -142,6 +142,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): hawqHosts = hawqMasterHosts.union(hawqSegmentHosts) numSegments = len(hawqSegmentHosts) minHawqHostsMemory = min([host['Hosts']['total_mem'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts]) + minHawqHostsCoreCount = min([host['Hosts']['cpu_count'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts]) if "hawq-site" in services["configurations"]: hawq_site = services["configurations"]["hawq-site"]["properties"] @@ -167,6 +168,9 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): buckets = factor * numSegments putHawqSiteProperty('default_hash_table_bucket_number', buckets) + if "hawq_rm_nvcore_limit_perseg" in hawq_site: + putHawqSiteProperty('hawq_rm_nvcore_limit_perseg', minHawqHostsCoreCount) + # update YARN RM urls with the values from yarn-site if YARN is installed if "YARN" in servicesList and "yarn-site" in services["configurations"]: yarn_site = services["configurations"]["yarn-site"]["properties"] http://git-wip-us.apache.org/repos/asf/ambari/blob/c04b543b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py index 6fbdf45..6531454 100644 --- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py @@ -141,7 +141,8 @@ class TestHAWQ200ServiceAdvisor(TestCase): }, "hawq-site": { "properties": { - "hawq_rm_memory_limit_perseg": "67108864KB" + "hawq_rm_memory_limit_perseg": "67108864KB", + "hawq_rm_nvcore_limit_perseg": "16" } } } @@ -184,30 +185,52 @@ class TestHAWQ200ServiceAdvisor(TestCase): { "Hosts": { "host_name": "c6401.ambari.apache.org", + "cpu_count" : 2, "total_mem": 33554432 } }, { "Hosts": { "host_name": "c6402.ambari.apache.org", + "cpu_count" : 4, "total_mem": 33554433 } }, { "Hosts": { "host_name": "c6403.ambari.apache.org", + "cpu_count" : 1, "total_mem": 33554434 } }, { "Hosts": { "host_name": "c6404.ambari.apache.org", + "cpu_count" : 2, "total_mem": 33554435 } } ] } + ## Test if hawq_rm_nvcore_limit_perseg is set correctly + + # Case 1: + # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2 + # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1 + # Recommend hawq_rm_nvcore_limit_perseg as 2 + self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2") + + # Case 2: + # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 2, c6404.ambari.apache.org - 2 + # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1 + # Recommend hawq_rm_nvcore_limit_perseg as 2 + hosts["items"][1]["Hosts"]["cpu_count"] = 2 + self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2") + + ## Test if vm.overcommit_memory is set correctly # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432) @@ -221,6 +244,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1") ## Test if hawq_rm_memory_limit_perseg is set correctly + # Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB as vm.overcommit_memory = 1 and in MB hosts["items"][0]["Hosts"]["total_mem"] = 2097152 self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) http://git-wip-us.apache.org/repos/asf/ambari/blob/c04b543b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index b931dc6..424a386 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1994,25 +1994,29 @@ class TestHDP23StackAdvisor(TestCase): { "Hosts": { "host_name": "c6401.ambari.apache.org", - "total_mem": 12345678 + "cpu_count" : 2, + "total_mem": 33554432 } }, { "Hosts": { "host_name": "c6402.ambari.apache.org", - "total_mem": 12345678 + "cpu_count" : 4, + "total_mem": 33554433 } }, { "Hosts": { "host_name": "c6403.ambari.apache.org", - "total_mem": 12345678 + "cpu_count" : 1, + "total_mem": 33554434 } }, { "Hosts": { "host_name": "c6404.ambari.apache.org", - "total_mem": 12345678 + "cpu_count" : 2, + "total_mem": 33554435 } } ]