Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id EEE321824B for ; Fri, 12 Feb 2016 18:30:48 +0000 (UTC) Received: (qmail 1695 invoked by uid 500); 12 Feb 2016 18:30:48 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 1639 invoked by uid 500); 12 Feb 2016 18:30:48 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 866 invoked by uid 99); 12 Feb 2016 18:30:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 12 Feb 2016 18:30:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 12716E69A7; Fri, 12 Feb 2016 18:30:48 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: yusaku@apache.org To: commits@ambari.apache.org Date: Fri, 12 Feb 2016 18:30:59 -0000 Message-Id: <89a9ef0d4ac241d89c7f5c815046fef3@git.apache.org> In-Reply-To: <40975445cf2040c4afed4ffe5bbb663b@git.apache.org> References: <40975445cf2040c4afed4ffe5bbb663b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [13/19] ambari git commit: AMBARI-14982 : Change AMS configs to deal with slow writes to WASB (avijayan) AMBARI-14982 : Change AMS configs to deal with slow writes to WASB (avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9120682e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9120682e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9120682e Branch: refs/heads/2.2.1-maint Commit: 9120682ed5de26c6163c6bc07935782fb645ff32 Parents: 984d9e9 Author: Aravindan Vijayan Authored: Thu Feb 11 10:16:48 2016 -0800 Committer: Aravindan Vijayan Committed: Thu Feb 11 10:16:48 2016 -0800 ---------------------------------------------------------------------- .../timeline/HBaseTimelineMetricStore.java | 18 +++++----- .../metrics/timeline/PhoenixHBaseAccessor.java | 21 ++++++----- .../timeline/TimelineMetricConfiguration.java | 10 ++++++ .../0.1.0/configuration/ams-site.xml | 37 +++++++++++++++----- .../stacks/HDP/2.0.6/services/stack_advisor.py | 5 ++- .../stacks/2.2/common/test_stack_advisor.py | 5 ++- 6 files changed, 69 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9120682e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index c4e946a..8678071 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -142,14 +142,16 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin aggregatorDailyThread.start(); } - int initDelay = configuration.getTimelineMetricsServiceWatcherInitDelay(); - int delay = configuration.getTimelineMetricsServiceWatcherDelay(); - // Start the watchdog - executorService.scheduleWithFixedDelay( - new TimelineMetricStoreWatcher(this, configuration), initDelay, delay, - TimeUnit.SECONDS); - LOG.info("Started watchdog for timeline metrics store with initial " + - "delay = " + initDelay + ", delay = " + delay); + if (!configuration.isTimelineMetricsServiceWatcherDisabled()) { + int initDelay = configuration.getTimelineMetricsServiceWatcherInitDelay(); + int delay = configuration.getTimelineMetricsServiceWatcherDelay(); + // Start the watchdog + executorService.scheduleWithFixedDelay( + new TimelineMetricStoreWatcher(this, configuration), initDelay, delay, + TimeUnit.SECONDS); + LOG.info("Started watchdog for timeline metrics store with initial " + + "delay = " + initDelay + ", delay = " + delay); + } isInitialized = true; } http://git-wip-us.apache.org/repos/asf/ambari/blob/9120682e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java index 4dc1a95..60ac06f 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java @@ -244,14 +244,14 @@ public class PhoenixHBaseAccessor { String encoding = metricsConf.get(HBASE_ENCODING_SCHEME, DEFAULT_ENCODING); String compression = metricsConf.get(HBASE_COMPRESSION_SCHEME, DEFAULT_TABLE_COMPRESSION); - String precisionTtl = metricsConf.get(PRECISION_TABLE_TTL, "86400"); //1 day - String hostMinTtl = metricsConf.get(HOST_MINUTE_TABLE_TTL, "604800"); //7 days - String hostHourTtl = metricsConf.get(HOST_HOUR_TABLE_TTL, "2592000"); //30 days - String hostDailyTtl = metricsConf.get(HOST_DAILY_TABLE_TTL, "31536000"); //1 year - String clusterSecTtl = metricsConf.get(CLUSTER_SECOND_TABLE_TTL, "2592000"); //7 days - String clusterMinTtl = metricsConf.get(CLUSTER_MINUTE_TABLE_TTL, "7776000"); //30 days - String clusterHourTtl = metricsConf.get(CLUSTER_HOUR_TABLE_TTL, "31536000"); //1 year - String clusterDailyTtl = metricsConf.get(CLUSTER_DAILY_TABLE_TTL, "63072000"); //2 years + String precisionTtl = getDaysInSeconds(metricsConf.get(PRECISION_TABLE_TTL, "1")); //1 day + String hostMinTtl = getDaysInSeconds(metricsConf.get(HOST_MINUTE_TABLE_TTL, "7")); //7 days + String hostHourTtl = getDaysInSeconds(metricsConf.get(HOST_HOUR_TABLE_TTL, "30")); //30 days + String hostDailyTtl = getDaysInSeconds(metricsConf.get(HOST_DAILY_TABLE_TTL, "365")); //1 year + String clusterSecTtl = getDaysInSeconds(metricsConf.get(CLUSTER_SECOND_TABLE_TTL, "7")); //7 days + String clusterMinTtl = getDaysInSeconds(metricsConf.get(CLUSTER_MINUTE_TABLE_TTL, "30")); //30 days + String clusterHourTtl = getDaysInSeconds(metricsConf.get(CLUSTER_HOUR_TABLE_TTL, "365")); //1 year + String clusterDailyTtl = getDaysInSeconds(metricsConf.get(CLUSTER_DAILY_TABLE_TTL, "730")); //2 years try { LOG.info("Initializing metrics schema..."); @@ -1023,4 +1023,9 @@ public class PhoenixHBaseAccessor { "thread = " + Thread.currentThread().getName()); } } + + private String getDaysInSeconds(String daysString) { + double days = Double.valueOf(daysString.trim()); + return String.valueOf((int)(days*86400)); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/9120682e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index 6311f9f..c0ab6f1 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -181,6 +181,9 @@ public class TimelineMetricConfiguration { public static final String HANDLER_THREAD_COUNT = "timeline.metrics.service.handler.thread.count"; + public static final String WATCHER_DISABLED = + "timeline.metrics.service.watcher.disabled"; + public static final String WATCHER_INITIAL_DELAY = "timeline.metrics.service.watcher.initial.delay"; @@ -261,6 +264,13 @@ public class TimelineMetricConfiguration { return 20; } + public boolean isTimelineMetricsServiceWatcherDisabled() { + if (metricsConf != null) { + return Boolean.parseBoolean(metricsConf.get(WATCHER_DISABLED, "false")); + } + return false; + } + public int getTimelineMetricsServiceWatcherInitDelay() { if (metricsConf != null) { return Integer.parseInt(metricsConf.get(WATCHER_INITIAL_DELAY, "600")); http://git-wip-us.apache.org/repos/asf/ambari/blob/9120682e/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml index 8173743..5f66063 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml @@ -283,59 +283,65 @@ timeline.metrics.host.aggregator.daily.ttl - 31536000 + 365 Host based daily resolution data purge interval. Default is 1 year. timeline.metrics.host.aggregator.hourly.ttl - 2592000 + 30 Host based hourly resolution data purge interval. Default is 30 days. timeline.metrics.host.aggregator.minute.ttl - 604800 + 7 Host based minute resolution data purge interval. Default is 7 days. timeline.metrics.cluster.aggregator.second.ttl - 2592000 + 7 Cluster wide second resolution data purge interval. Default is 7 days. timeline.metrics.cluster.aggregator.minute.ttl - 7776000 + 30 Cluster wide minute resolution data purge interval. Default is 30 days. timeline.metrics.cluster.aggregator.hourly.ttl - 31536000 + 365 Cluster wide hourly resolution data purge interval. Default is 1 year. timeline.metrics.cluster.aggregator.daily.ttl - 63072000 + 730 Cluster wide daily resolution data purge interval. Default is 2 years. timeline.metrics.host.aggregator.ttl - 86400 + 1 - 1 minute resolution data purge interval. Default is 1 day. + 1 minute resolution data purge interval. Default is 1 day for embedded mode and 7 days for Distributed mode. + + + ams-site + timeline.metrics.service.operation.mode + + timeline.metrics.hbase.data.block.encoding @@ -492,6 +498,19 @@ + timeline.metrics.service.watcher.disabled + false + + Disable Timeline Metric Store watcher thread. Disabled by default in AMS distributed mode. + + + + ams-site + timeline.metrics.service.operation.mode + + + + timeline.metrics.service.watcher.initial.delay 600 http://git-wip-us.apache.org/repos/asf/ambari/blob/9120682e/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index e8df97b..07cc73b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -470,8 +470,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor): operatingMode = services["configurations"]["ams-site"]["properties"]["timeline.metrics.service.operation.mode"] if operatingMode == "distributed": + putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'true') + putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 7) putAmsHbaseSiteProperty("hbase.cluster.distributed", 'true') else: + putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'false') + putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 1) putAmsHbaseSiteProperty("hbase.cluster.distributed", 'false') rootDir = "file:///var/lib/ambari-metrics-collector/hbase" @@ -527,7 +531,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 134217728) putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", 0.35) putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.3) - putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 86400) if len(amsCollectorHosts) > 1: pass http://git-wip-us.apache.org/repos/asf/ambari/blob/9120682e/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 86d560a..6984e0e 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -2011,7 +2011,8 @@ class TestHDP22StackAdvisor(TestCase): "properties": { "timeline.metrics.cluster.aggregate.splitpoints": " ", "timeline.metrics.host.aggregate.splitpoints": " ", - "timeline.metrics.host.aggregator.ttl": "86400" + "timeline.metrics.host.aggregator.ttl": "1", + 'timeline.metrics.service.watcher.disabled': 'false' } } } @@ -2189,6 +2190,8 @@ class TestHDP22StackAdvisor(TestCase): expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512' expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102' expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '384' + expected['ams-site']['properties']['timeline.metrics.host.aggregator.ttl'] = '7' + expected['ams-site']['properties']['timeline.metrics.service.watcher.disabled'] = 'true' self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected)