ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhuvnesh2...@apache.org
Subject ambari git commit: AMBARI-16745: Recommend value for hawq_rm_memory_limit_perseg for HAWQ (bhuvnesh2703)
Date Thu, 19 May 2016 19:14:31 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 56d266525 -> 9987b97e9


AMBARI-16745: Recommend value for hawq_rm_memory_limit_perseg for HAWQ (bhuvnesh2703)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9987b97e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9987b97e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9987b97e

Branch: refs/heads/branch-2.4
Commit: 9987b97e923e77ae276f59c4e6c48e3138155860
Parents: 56d2665
Author: Bhuvnesh Chaudhary <bhuvnesh2703@gmail.com>
Authored: Thu May 19 12:15:38 2016 -0700
Committer: Bhuvnesh Chaudhary <bhuvnesh2703@gmail.com>
Committed: Thu May 19 12:15:38 2016 -0700

----------------------------------------------------------------------
 .../HAWQ/2.0.0/service_advisor.py               | 32 ++++++++++++---
 .../stacks/2.3/HAWQ/test_service_advisor.py     | 43 +++++++++++++++++++-
 2 files changed, 67 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9987b97e/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
index 276dd3a..0590604 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
@@ -23,6 +23,7 @@ import re
 import socket
 import sys
 import traceback
+import math
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
@@ -162,13 +163,32 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
             putHawqSiteProperty(hs_prop, yarn_site[ys_prop])
 
     # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater
than 32GB
-    if "hawq-sysctl-env" in services["configurations"]:
+    if "hawq-sysctl-env" in services["configurations"] and "vm.overcommit_memory" in services["configurations"]["hawq-sysctl-env"]["properties"]:
       MEM_THRESHOLD = 33554432 # 32GB, minHawqHostsMemory is represented in kB
-      hawq_sysctl_env = services["configurations"]["hawq-sysctl-env"]["properties"]
-      if "vm.overcommit_memory" in hawq_sysctl_env:
-        propertyValue = "2" if minHawqHostsMemory >= MEM_THRESHOLD else "1"
-        putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services)
-        putHawqSysctlEnvProperty("vm.overcommit_memory", propertyValue)
+      vm_overcommit_mem_value = "2" if minHawqHostsMemory >= MEM_THRESHOLD else "1"
+      putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services)
+      putHawqSysctlEnvProperty("vm.overcommit_memory", vm_overcommit_mem_value)
+
+      # Set the value for hawq_rm_memory_limit_perseg based on vm.overcommit value and RAM
available on HAWQ Hosts
+      # HAWQ Hosts with the minimum amount of RAM is considered for calculations
+      # Available RAM Formula = SWAP + RAM * vm.overcommit_ratio / 100
+      # Assumption: vm.overcommit_ratio = 50 (default on Linux), SWAP not considered for
recommendation
+      host_ram_kb = minHawqHostsMemory / 2 if vm_overcommit_mem_value == "2" else minHawqHostsMemory
+      host_ram_gb = float(host_ram_kb) / (1024 * 1024)
+      recommended_mem_percentage = {
+        host_ram_gb <= 64: .75,
+        64 < host_ram_gb <= 512: .85,
+        host_ram_gb > 512: .95
+      }[True]
+      recommended_mem = math.ceil(host_ram_gb * recommended_mem_percentage)
+      unit = "GB"
+      # If RAM on a host is very low ~ 2 GB, ceil function may result in making it equal
to total mem,
+      # in that case we recommend the value in MB not GB
+      if recommended_mem >= host_ram_gb:
+        recommended_mem = math.ceil(float(host_ram_kb)/1024 * recommended_mem_percentage)
+        unit = "MB"
+      # hawq_rm_memory_limit_perseg does not support decimal value so trim decimal using
int
+      putHawqSiteProperty("hawq_rm_memory_limit_perseg", "{0}{1}".format(int(recommended_mem),
unit))
 
     # set output.replace-datanode-on-failure in HAWQ hdfs-client depending on the cluster
size
     if "hdfs-client" in services["configurations"]:

http://git-wip-us.apache.org/repos/asf/ambari/blob/9987b97e/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
index 50f3a1f..905d455 100644
--- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
@@ -33,7 +33,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
     with open(hawq200ServiceAdvisorPath, 'rb') as fp:
       service_advisor = imp.load_module('stack_advisor_impl', fp, hawq200ServiceAdvisorPath,
('.py', 'rb', imp.PY_SOURCE))
 
-    stackAdvisorClass = getattr(stack_advisor, 'StackAdvisor')
+    stackAdvisorClass = getattr(stack_advisor, 'DefaultStackAdvisor')
     self.stackAdvisor = stackAdvisorClass()
 
     serviceAdvisorClass = getattr(service_advisor, 'HAWQ200ServiceAdvisor')
@@ -46,6 +46,11 @@ class TestHAWQ200ServiceAdvisor(TestCase):
         "properties": {
           "vm.overcommit_memory": "1"
         }
+      },
+      "hawq-site": {
+        "properties": {
+          "hawq_rm_memory_limit_perseg": "64GB"
+        }
       }
     }
 
@@ -120,4 +125,38 @@ class TestHAWQ200ServiceAdvisor(TestCase):
     # Case 2: One machine has total_mem below 32GB
     hosts["items"][0]["Hosts"]["total_mem"] = 33554431
     self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations,
None, services, hosts)
-    self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"],
"1")
\ No newline at end of file
+    self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"],
"1")
+
+    ## Test if hawq_rm_memory_limit_perseg is set correctly
+    # Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB
as vm.overcommit_memory = 1 and in MB
+    hosts["items"][0]["Hosts"]["total_mem"] = 2097152
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations,
None, services, hosts)
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"],
"1536MB")
+
+    # Case 2: Minimum host memory is ~ 16 GB, recommended val must be .75% of 16GB as vm.overcommit_memory
= 1 and in GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 16777216
+    hosts["items"][1]["Hosts"]["total_mem"] = 26777216
+    hosts["items"][3]["Hosts"]["total_mem"] = 36777216
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations,
None, services, hosts)
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"],
"12GB")
+
+    # Case 2: Minimum host memory is ~ 64 GB, recommended val must be .75% of 32GB as vm.overcommit_memory
= 2 and in GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 67108864
+    hosts["items"][1]["Hosts"]["total_mem"] = 77108864
+    hosts["items"][3]["Hosts"]["total_mem"] = 87108864
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations,
None, services, hosts)
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"],
"24GB")
+
+    # Case 4: Minimum host memory is ~ 512 GB, recommended val must be .85% of 256GB as vm.overcommit_memory
= 2 and in GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 536870912
+    hosts["items"][1]["Hosts"]["total_mem"] = 636870912
+    hosts["items"][3]["Hosts"]["total_mem"] = 736870912
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations,
None, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"],
"218GB")
+
+    # Case 5: Minimum host memory is ~ 1024 GB, recommended val must be .95% of 512GB as
vm.overcommit_memory = 2 and in GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 1073741824
+    hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
+    hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations,
None, services, hosts)
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"],
"436GB")


Mime
View raw message