Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ECEEF18DE4 for ; Sat, 19 Sep 2015 15:26:41 +0000 (UTC) Received: (qmail 19143 invoked by uid 500); 19 Sep 2015 15:26:41 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 19112 invoked by uid 500); 19 Sep 2015 15:26:41 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 19103 invoked by uid 99); 19 Sep 2015 15:26:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 19 Sep 2015 15:26:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AE70EDFD87; Sat, 19 Sep 2015 15:26:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dsen@apache.org To: commits@ambari.apache.org Message-Id: <11350d67cdcd461a9a961c182999a2fe@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-13161 Master heap setting in distributed mode should be limited (dsen) Date: Sat, 19 Sep 2015 15:26:41 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/branch-2.1 944d9a731 -> 957a6919e AMBARI-13161 Master heap setting in distributed mode should be limited (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/957a6919 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/957a6919 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/957a6919 Branch: refs/heads/branch-2.1 Commit: 957a6919eef00466891073e827827aeafb663f8f Parents: 944d9a7 Author: Dmytro Sen Authored: Sat Sep 19 18:24:07 2015 +0300 Committer: Dmytro Sen Committed: Sat Sep 19 18:26:31 2015 +0300 ---------------------------------------------------------------------- .../0.1.0/configuration/ams-hbase-env.xml | 24 ++++++++++- .../0.1.0/configuration/ams-hbase-site.xml | 20 +++++++++ .../0.1.0/package/scripts/ams_service.py | 5 +++ .../0.1.0/package/scripts/hbase.py | 22 +++++----- .../stacks/HDP/2.0.6/services/stack_advisor.py | 44 +++++++++++++++----- .../AMBARI_METRICS/test_metrics_collector.py | 19 +++++---- .../stacks/2.0.6/common/test_stack_advisor.py | 17 ++++++++ .../stacks/2.2/common/test_stack_advisor.py | 2 + 8 files changed, 121 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/957a6919/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml index 68b99ca..7274a12 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml @@ -33,11 +33,21 @@ hbase_regionserver_heapsize - 1024m + 512m HBase RegionServer Heap Size. In embedded mode, total heap size is sum of master and regionserver heap sizes. + + + ams-hbase-site + hbase.cluster.distributed + + + ams-hbase-site + hbase.rootdir + + regionserver_xmn_size @@ -63,11 +73,21 @@ hbase_master_heapsize - 1024m + 512m HBase Master Heap Size. In embedded mode, total heap size is sum of master and regionserver heap sizes. + + + ams-hbase-site + hbase.cluster.distributed + + + ams-hbase-site + hbase.rootdir + + max_open_files_limit http://git-wip-us.apache.org/repos/asf/ambari/blob/957a6919/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml index b286cac..165b87e 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml @@ -313,4 +313,24 @@ a normal table and would return items in rowkey order for scans + + + dfs.client.read.shortcircuit + true + Enable/Disable short circuit read for your client. + Hadoop servers should be configured to allow short circuit read + for the hbase user for this to take effect + + + + ams-hbase-site + hbase.cluster.distributed + + + ams-hbase-site + hbase.rootdir + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/957a6919/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py index 8b2fa95..1afe1d8 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py @@ -22,6 +22,7 @@ from resource_management import * from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from hbase_service import hbase_service +import os @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def ams_service(name, action): @@ -53,6 +54,10 @@ def ams_service(name, action): Execute(format("{sudo} rm -rf {hbase_tmp_dir}/*.tmp") ) + if not params.is_hbase_distributed and os.path.exists(format("{zookeeper_data_dir}")): + Execute(format("{sudo} rm -rf {zookeeper_data_dir}/*") + ) + daemon_cmd = format("{cmd} start") Execute(daemon_cmd, user=params.ams_user http://git-wip-us.apache.org/repos/asf/ambari/blob/957a6919/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py index 1d9bf23..b69d67a 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py @@ -173,6 +173,17 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' hbase_TemplateConfig( format("hbase_client_jaas.conf"), user=params.hbase_user) hbase_TemplateConfig( format("ams_zookeeper_jaas.conf"), user=params.hbase_user) + if name != "client": + Directory( params.hbase_pid_dir, + owner = params.hbase_user, + recursive = True + ) + + Directory (params.hbase_log_dir, + owner = params.hbase_user, + recursive = True + ) + if name == "master": if params.is_hbase_distributed: @@ -216,17 +227,6 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' File(format("{params.hbase_pid_dir}/distributed_mode"), action="delete", owner=params.hbase_user) - if name != "client": - Directory( params.hbase_pid_dir, - owner = params.hbase_user, - recursive = True - ) - - Directory (params.hbase_log_dir, - owner = params.hbase_user, - recursive = True - ) - if params.hbase_log4j_props is not None: File(format("{params.hbase_conf_dir}/log4j.properties"), mode=0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/957a6919/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index b2332b0..5568e60 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -369,6 +369,15 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putHbaseEnvProperty("hbase_master_heapsize", str(hbase_heapsize) + "m") putHbaseEnvProperty("hbase_master_xmn_size", hbase_xmn_size) + # If no local DN in distributed mode + if rootDir.startswith("hdfs://"): + dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE") + if set(amsCollectorHosts).intersection(dn_hosts): + collector_cohosted_with_dn = "true" + else: + collector_cohosted_with_dn = "false" + putAmsHbaseSiteProperty("dfs.client.read.shortcircuit", collector_cohosted_with_dn) + #split points scriptDir = os.path.dirname(os.path.abspath(__file__)) metricsDir = os.path.join(scriptDir, '../../../../common-services/AMBARI_METRICS/0.1.0/package') @@ -651,6 +660,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.rootdir', host["Hosts"])}]) validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.tmp.dir', host["Hosts"])}]) + dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE") if not hbase_rootdir.startswith("hdfs"): mountPoints = [] for mountPoint in host["Hosts"]["disk_info"]: @@ -669,7 +679,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor): # if METRICS_COLLECTOR is co-hosted with DATANODE # cross-check dfs.datanode.data.dir and hbase.rootdir # they shouldn't share same disk partition IO - dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE") hdfs_site = getSiteProperties(configurations, "hdfs-site") dfs_datadirs = hdfs_site.get("dfs.datanode.data.dir").split(",") if hdfs_site and "dfs.datanode.data.dir" in hdfs_site else [] if dn_hosts and collectorHostName in dn_hosts and ams_site and \ @@ -679,8 +688,18 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if dfs_datadir_mountpoint == hbase_rootdir_mountpoint: item = self.getWarnItem("Consider not using {0} partition for storing metrics data. " "{0} is already used by datanode to store HDFS data".format(hbase_rootdir_mountpoint)) - validationItems.extend([{"config-name":'hbase.rootdir', "item": item}]) + validationItems.extend([{"config-name": 'hbase.rootdir', "item": item}]) break + # If no local DN in distributed mode + elif collectorHostName not in dn_hosts and distributed.lower() == "true": + item = self.getWarnItem("It's recommended to install Datanode component on {0} " + "to speed up IO operations between HDFS and Metrics " + "Collector in distributed mode ".format(collectorHostName)) + validationItems.extend([{"config-name": "hbase.cluster.distributed", "item": item}]) + # Short circuit read should be enabled in distibuted mode + # if local DN installed + else: + validationItems.extend([{"config-name": "dfs.client.read.shortcircuit", "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, "dfs.client.read.shortcircuit")}]) return self.toConfigurationValidationProblems(validationItems, "ams-hbase-site") @@ -757,16 +776,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor): validationItems.extend([{"config-name": heapPropertyToIncrease, "item": unusedMemoryHbaseItem}]) pass - # Check RS memory in distributed mode since we set default as 512m - hbase_site = getSiteProperties(configurations, "ams-hbase-site") - hbase_rootdir = hbase_site.get("hbase.rootdir") - regionServerMinMemItem = None - if hbase_rootdir and hbase_rootdir.startswith("hdfs://"): - regionServerMinMemItem = self.validateMinMemorySetting(properties, 1024, 'hbase_regionserver_heapsize') - validationItems.extend([ {"config-name": "hbase_regionserver_heapsize", "item": regionServerItem}, - {"config-name": "hbase_regionserver_heapsize", "item": regionServerMinMemItem}, {"config-name": "hbase_master_heapsize", "item": masterItem}, {"config-name": "hbase_master_heapsize", "item": masterHostItem}, {"config-name": "hbase_log_dir", "item": logDirItem} @@ -890,6 +901,19 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return None + def validatorEqualsToRecommendedItem(self, properties, recommendedDefaults, + propertyName): + if not propertyName in properties: + return self.getErrorItem("Value should be set for %s" % propertyName) + value = properties.get(propertyName) + if not propertyName in recommendedDefaults: + return self.getErrorItem("Value should be recommended for %s" % propertyName) + recommendedValue = recommendedDefaults.get(propertyName) + if value != recommendedValue: + return self.getWarnItem("It is recommended to set value {0} " + "for property {1}".format(recommendedValue, propertyName)) + return None + def validateMinMemorySetting(self, properties, defaultValue, propertyName): if not propertyName in properties: return self.getErrorItem("Value should be set") http://git-wip-us.apache.org/repos/asf/ambari/blob/957a6919/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py index 7b08541..7add394 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py +++ b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py @@ -50,7 +50,7 @@ class TestOozieClient(RMFTestCase): not_if = 'ls /var/run/ambari-metrics-collector//hbase-ams-regionserver.pid >/dev/null 2>&1 && ps `cat /var/run/ambari-metrics-collector//hbase-ams-regionserver.pid` >/dev/null 2>&1', user = 'ams' ) - self.assertResourceCalled('Execute', 'ambari-sudo.sh rm -rf /var/lib/ambari-metrics-collector/hbase-tmp/*.tmp /var/lib/ambari-metrics-collector/hbase-tmp/zookeeper/*', + self.assertResourceCalled('Execute', 'ambari-sudo.sh rm -rf /var/lib/ambari-metrics-collector/hbase-tmp/*.tmp', ) self.assertResourceCalled('Execute', '/usr/sbin/ambari-metrics-collector --config /etc/ambari-metrics-collector/conf --distributed start', user = 'ams' @@ -210,6 +210,15 @@ class TestOozieClient(RMFTestCase): owner = 'ams', template_tag = None, ) + self.assertResourceCalled('Directory', '/var/run/ambari-metrics-collector/', + owner = 'ams', + recursive = True + ) + self.assertResourceCalled('Directory', '/var/log/ambari-metrics-collector', + owner = 'ams', + recursive = True + ) + if name == 'master': self.assertResourceCalled('HdfsResource', 'hdfs://localhost:8020/apps/hbase/data', security_enabled = False, @@ -255,14 +264,6 @@ class TestOozieClient(RMFTestCase): ) self.assertResourceCalled('File', '/var/run/ambari-metrics-collector//distributed_mode', action=["create"], mode=0644, owner='ams') - self.assertResourceCalled('Directory', '/var/run/ambari-metrics-collector/', - owner = 'ams', - recursive = True - ) - self.assertResourceCalled('Directory', '/var/log/ambari-metrics-collector', - owner = 'ams', - recursive = True - ) self.assertResourceCalled('File', '/etc/ams-hbase/conf/log4j.properties', owner = 'ams', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/957a6919/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 16ba278..79a6edb 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -1394,3 +1394,20 @@ class TestHDP206StackAdvisor(TestCase): self.assertEquals(self.stack_advisor_impl.getMountPointForDir("file:///var/log", ["/var", "/"]), "/var") self.assertEquals(self.stack_advisor_impl.getMountPointForDir("hdfs:///hdfs_path", ["/var", "/"]), None) self.assertEquals(self.stack_advisor_impl.getMountPointForDir("relative/path", ["/var", "/"]), None) + + def test_getValidatorEqualsToRecommendedItem(self): + properties = {"property1": "value1"} + recommendedDefaults = {"property1": "value1"} + self.assertEquals(self.stackAdvisor.validatorEqualsToRecommendedItem(properties, recommendedDefaults, "property1"), None) + properties = {"property1": "value1"} + recommendedDefaults = {"property1": "value2"} + expected = {'message': 'It is recommended to set value value2 for property property1', 'level': 'WARN'} + self.assertEquals(self.stackAdvisor.validatorEqualsToRecommendedItem(properties, recommendedDefaults, "property1"), expected) + properties = {} + recommendedDefaults = {"property1": "value2"} + expected = {'level': 'ERROR', 'message': 'Value should be set for property1'} + self.assertEquals(self.stackAdvisor.validatorEqualsToRecommendedItem(properties, recommendedDefaults, "property1"), expected) + properties = {"property1": "value1"} + recommendedDefaults = {} + expected = {'level': 'ERROR', 'message': 'Value should be recommended for property1'} + self.assertEquals(self.stackAdvisor.validatorEqualsToRecommendedItem(properties, recommendedDefaults, "property1"), expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/957a6919/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 5bb99ae..054bf96 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -2161,6 +2161,8 @@ class TestHDP22StackAdvisor(TestCase): services["configurations"]['ams-hbase-site']['properties']['hbase.rootdir'] = 'hdfs://host1/amshbase' expected['ams-hbase-site']['properties']['hbase.rootdir'] = 'hdfs://host1/amshbase' expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512m' + # services["configurations"]['ams-hbase-site']['properties']['dfs.client.read.shortcircuit'] = 'true' + expected['ams-hbase-site']['properties']['dfs.client.read.shortcircuit'] = 'true' # Distributed mode, low memory, no splitpoints recommended services["configurations"]['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512m'