ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhuvnesh2...@apache.org
Subject ambari git commit: AMBARI-16992: Recommend hawq_rm_nvcore_limit_perseg in HAWQ service advisor (mithmatt via bhuvnesh2703)
Date Wed, 01 Jun 2016 22:35:08 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 a2e2f8e57 -> c04b543b7


AMBARI-16992: Recommend hawq_rm_nvcore_limit_perseg in HAWQ service advisor (mithmatt via
bhuvnesh2703)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c04b543b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c04b543b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c04b543b

Branch: refs/heads/branch-2.4
Commit: c04b543b75e5a69473f6a7b5ce0d5506837ff40e
Parents: a2e2f8e
Author: Bhuvnesh Chaudhary <bhuvnesh2703@gmail.com>
Authored: Wed Jun 1 15:37:08 2016 -0700
Committer: Bhuvnesh Chaudhary <bhuvnesh2703@gmail.com>
Committed: Wed Jun 1 15:37:08 2016 -0700

----------------------------------------------------------------------
 .../HAWQ/2.0.0/service_advisor.py               |  4 +++
 .../stacks/2.3/HAWQ/test_service_advisor.py     | 26 +++++++++++++++++++-
 .../stacks/2.3/common/test_stack_advisor.py     | 12 ++++++---
 3 files changed, 37 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/c04b543b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
index 94e1854..32e23f9 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
@@ -142,6 +142,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
       hawqHosts = hawqMasterHosts.union(hawqSegmentHosts)
       numSegments = len(hawqSegmentHosts)
       minHawqHostsMemory = min([host['Hosts']['total_mem'] for host in hosts['items'] if
host['Hosts']['host_name'] in hawqHosts])
+      minHawqHostsCoreCount = min([host['Hosts']['cpu_count'] for host in hosts['items']
if host['Hosts']['host_name'] in hawqHosts])
 
     if "hawq-site" in services["configurations"]:
       hawq_site = services["configurations"]["hawq-site"]["properties"]
@@ -167,6 +168,9 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
           buckets = factor * numSegments
         putHawqSiteProperty('default_hash_table_bucket_number', buckets)
 
+      if "hawq_rm_nvcore_limit_perseg" in hawq_site:
+        putHawqSiteProperty('hawq_rm_nvcore_limit_perseg', minHawqHostsCoreCount)
+
       # update YARN RM urls with the values from yarn-site if YARN is installed
       if "YARN" in servicesList and "yarn-site" in services["configurations"]:
         yarn_site = services["configurations"]["yarn-site"]["properties"]

http://git-wip-us.apache.org/repos/asf/ambari/blob/c04b543b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
index 6fbdf45..6531454 100644
--- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
@@ -141,7 +141,8 @@ class TestHAWQ200ServiceAdvisor(TestCase):
       },
       "hawq-site": {
         "properties": {
-          "hawq_rm_memory_limit_perseg": "67108864KB"
+          "hawq_rm_memory_limit_perseg": "67108864KB",
+          "hawq_rm_nvcore_limit_perseg": "16"
         }
       }
     }
@@ -184,30 +185,52 @@ class TestHAWQ200ServiceAdvisor(TestCase):
         {
           "Hosts": {
             "host_name": "c6401.ambari.apache.org",
+            "cpu_count" : 2,
             "total_mem": 33554432
           }
         },
         {
           "Hosts": {
             "host_name": "c6402.ambari.apache.org",
+            "cpu_count" : 4,
             "total_mem": 33554433
           }
         },
         {
           "Hosts": {
             "host_name": "c6403.ambari.apache.org",
+            "cpu_count" : 1,
             "total_mem": 33554434
           }
         },
         {
           "Hosts": {
             "host_name": "c6404.ambari.apache.org",
+            "cpu_count" : 2,
             "total_mem": 33554435
           }
         }
       ]
     }
 
+    ## Test if hawq_rm_nvcore_limit_perseg is set correctly
+
+    # Case 1:
+    # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org
- 2
+    # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1
+    # Recommend hawq_rm_nvcore_limit_perseg as 2
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations,
None, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"],
"2")
+
+    # Case 2:
+    # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 2, c6404.ambari.apache.org
- 2
+    # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1
+    # Recommend hawq_rm_nvcore_limit_perseg as 2
+    hosts["items"][1]["Hosts"]["cpu_count"] = 2
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations,
None, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"],
"2")
+
+
     ## Test if vm.overcommit_memory is set correctly
 
     # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432)
@@ -221,6 +244,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
     self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"],
"1")
 
     ## Test if hawq_rm_memory_limit_perseg is set correctly
+
     # Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB
as vm.overcommit_memory = 1 and in MB
     hosts["items"][0]["Hosts"]["total_mem"] = 2097152
     self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations,
None, services, hosts)

http://git-wip-us.apache.org/repos/asf/ambari/blob/c04b543b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index b931dc6..424a386 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1994,25 +1994,29 @@ class TestHDP23StackAdvisor(TestCase):
         {
           "Hosts": {
             "host_name": "c6401.ambari.apache.org",
-            "total_mem": 12345678
+            "cpu_count" : 2,
+            "total_mem": 33554432
           }
         },
         {
           "Hosts": {
             "host_name": "c6402.ambari.apache.org",
-            "total_mem": 12345678
+            "cpu_count" : 4,
+            "total_mem": 33554433
           }
         },
         {
           "Hosts": {
             "host_name": "c6403.ambari.apache.org",
-            "total_mem": 12345678
+            "cpu_count" : 1,
+            "total_mem": 33554434
           }
         },
         {
           "Hosts": {
             "host_name": "c6404.ambari.apache.org",
-            "total_mem": 12345678
+            "cpu_count" : 2,
+            "total_mem": 33554435
           }
         }
       ]


Mime
View raw message