Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3DB7417EC6 for ; Tue, 14 Apr 2015 23:24:56 +0000 (UTC) Received: (qmail 14903 invoked by uid 500); 14 Apr 2015 23:24:56 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 14875 invoked by uid 500); 14 Apr 2015 23:24:56 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 14763 invoked by uid 99); 14 Apr 2015 23:24:56 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 14 Apr 2015 23:24:56 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 01DFAE0435; Tue, 14 Apr 2015 23:24:55 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: srimanth@apache.org To: commits@ambari.apache.org Date: Tue, 14 Apr 2015 23:24:57 -0000 Message-Id: In-Reply-To: <84766e05c62e431097d410f56fedcc2b@git.apache.org> References: <84766e05c62e431097d410f56fedcc2b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/4] ambari git commit: AMBARI-10475. BE: Extend stack-advisor to recommend property_value_attributes for HBase (mpapirkovskyy via srimanth) AMBARI-10475. BE: Extend stack-advisor to recommend property_value_attributes for HBase (mpapirkovskyy via srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/61cd7e8f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/61cd7e8f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/61cd7e8f Branch: refs/heads/trunk Commit: 61cd7e8f5f611b3aa05cfeccb328cccda6b92641 Parents: 731c56e Author: Srimanth Gunturi Authored: Tue Apr 14 11:31:34 2015 -0700 Committer: Srimanth Gunturi Committed: Tue Apr 14 16:13:16 2015 -0700 ---------------------------------------------------------------------- .../0.96.0.2.0/configuration/hbase-env.xml | 4 ++-- .../stacks/HDP/2.0.6/services/stack_advisor.py | 4 ++-- .../stacks/HDP/2.2/services/stack_advisor.py | 21 +++++++++++++++----- 3 files changed, 20 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/61cd7e8f/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml index abef070..105bb40 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml @@ -41,7 +41,7 @@ 1024 6554 MB - 250 + 256 @@ -68,7 +68,7 @@ and the -Xmn ratio (hbase_regionserver_xmn_ratio) exceeds this value. 1024 16384 MB - 250 + 256 http://git-wip-us.apache.org/repos/asf/ambari/blob/61cd7e8f/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index e692411..38d6eda 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -113,7 +113,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if configType not in config: config[configType] = {} def appendPropertyAttribute(key, attribute, attributeValue): - if"property_attributes" not in config[configType]: + if "property_attributes" not in config[configType]: config[configType]["property_attributes"] = {} if key not in config[configType]["property_attributes"]: config[configType]["property_attributes"][key] = {} @@ -147,7 +147,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putHDFSProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256)) def recommendHbaseEnvConfigurations(self, configurations, clusterData, services, hosts): - putHbaseProperty = self.putProperty(configurations, "hbase-env") + putHbaseProperty = self.putProperty(configurations, "hbase-env", services) putHbaseProperty('hbase_regionserver_heapsize', int(clusterData['hbaseRam']) * 1024) putHbaseProperty('hbase_master_heapsize', int(clusterData['hbaseRam']) * 1024) http://git-wip-us.apache.org/repos/asf/ambari/blob/61cd7e8f/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 9a2d373..da7ba08 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -125,7 +125,18 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts): super(HDP22StackAdvisor, self).recommendHbaseEnvConfigurations(configurations, clusterData, services, hosts) - putHbaseSiteProperty = self.putProperty(configurations, "hbase-site") + putHbaseEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hbase-env", services) + + rs_hosts = self.getHostsWithComponent("HBASE", "HBASE_REGIONSERVER", services, hosts) + if rs_hosts is not None and len(rs_hosts) > 0: + min_ram = rs_hosts[0]["Hosts"]["total_mem"] + for host in rs_hosts["items"]: + host_ram = host["Hosts"]["total_mem"] + min_ram = min(min_ram, host_ram) + + putHbaseEnvPropertyAttributes('hbase_regionserver_heapsize', 'max', max(1024, int(min_ram*0.8/1024))) + + putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services) putHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", '0.4') servicesList = [service["StackServices"]["service_name"] for service in services["services"]] @@ -154,7 +165,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): hbase_bucketcache_percentage_in_combinedcache_str = "{0:.4f}".format(math.ceil(hbase_bucketcache_percentage_in_combinedcache * 10000) / 10000.0) # Set values in hbase-site - putHbaseProperty = self.putProperty(configurations, "hbase-site") + putHbaseProperty = self.putProperty(configurations, "hbase-site", services) putHbaseProperty('hfile.block.cache.size', hfile_block_cache_size) putHbaseProperty('hbase.regionserver.global.memstore.upperLimit', hbase_regionserver_global_memstore_size) putHbaseProperty('hbase.bucketcache.ioengine', 'offheap') @@ -162,17 +173,17 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHbaseProperty('hbase.bucketcache.percentage.in.combinedcache', hbase_bucketcache_percentage_in_combinedcache_str) # Enable in hbase-env - putHbaseEnvProperty = self.putProperty(configurations, "hbase-env") + putHbaseEnvProperty = self.putProperty(configurations, "hbase-env", services) putHbaseEnvProperty('hbase_max_direct_memory_size', regionserver_max_direct_memory_size) putHbaseEnvProperty('hbase_regionserver_heapsize', regionserver_heap_size) else: # Disable - putHbaseProperty = self.putProperty(configurations, "hbase-site") + putHbaseProperty = self.putProperty(configurations, "hbase-site", services) putHbaseProperty('hbase.bucketcache.ioengine', '') putHbaseProperty('hbase.bucketcache.size', '') putHbaseProperty('hbase.bucketcache.percentage.in.combinedcache', '') - putHbaseEnvProperty = self.putProperty(configurations, "hbase-env") + putHbaseEnvProperty = self.putProperty(configurations, "hbase-env", services) putHbaseEnvProperty('hbase_max_direct_memory_size', '')