ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-18098 stack_advisor should recommend AMS cache size and commit frequency (dsen)
Date Thu, 11 Aug 2016 16:51:18 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 3cc479e30 -> 6ff0d5767


AMBARI-18098 stack_advisor should recommend AMS cache size and commit frequency (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6ff0d576
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6ff0d576
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6ff0d576

Branch: refs/heads/trunk
Commit: 6ff0d5767d5ab67c2f1f906283fe520c32cdf401
Parents: 3cc479e
Author: Dmytro Sen <dsen@apache.org>
Authored: Thu Aug 11 19:51:11 2016 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Thu Aug 11 19:51:11 2016 +0300

----------------------------------------------------------------------
 .../stacks/HDP/2.0.6/services/stack_advisor.py  |  5 ++-
 .../stacks/2.2/common/test_stack_advisor.py     | 44 +++++++++++++++++++-
 2 files changed, 47 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6ff0d576/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 87618a8..71dd280 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -22,7 +22,7 @@ import os
 import sys
 import socket
 
-from math import ceil, floor
+from math import ceil, floor, log
 
 from resource_management.core.logger import Logger
 from resource_management.libraries.functions.mounted_dirs_helper import get_mounts_with_multiple_data_dirs
@@ -661,6 +661,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     putAmsEnvProperty("metrics_collector_heapsize", collector_heapsize)
 
+    putAmsSiteProperty("timeline.metrics.cache.size", max(100, int(log(total_sinks_count))
* 100))
+    putAmsSiteProperty("timeline.metrics.cache.commit.interval", min(10, max(12 - int(log(total_sinks_count)),
2)))
+
     # blockCache = 0.3, memstore = 0.35, phoenix-server = 0.15, phoenix-client = 0.25
     putAmsHbaseSiteProperty("hfile.block.cache.size", 0.3)
     putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 134217728)

http://git-wip-us.apache.org/repos/asf/ambari/blob/6ff0d576/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 3f41acb..75060f4 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2295,7 +2295,9 @@ class TestHDP22StackAdvisor(TestCase):
           "timeline.metrics.host.aggregator.ttl": "86400",
           "timeline.metrics.service.handler.thread.count": "20",
           'timeline.metrics.service.webapp.address': 'host1:6188',
-          'timeline.metrics.service.watcher.disabled': 'false'
+          'timeline.metrics.service.watcher.disabled': 'false',
+          'timeline.metrics.cache.size': '100',
+          'timeline.metrics.cache.commit.interval': '10'
         }
       }
     }
@@ -2331,6 +2333,9 @@ class TestHDP22StackAdvisor(TestCase):
         ]
       }
     ]
+
+    expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '500'
+    expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '7'
     expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1408'
     expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '320'
     expected["ams-env"]['properties']['metrics_collector_heapsize'] = '512'
@@ -2401,6 +2406,9 @@ class TestHDP22StackAdvisor(TestCase):
     ]
     expected["ams-site"]['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'master.Server.numDeadRegionServers'
     expected["ams-site"]['properties']['timeline.metrics.cluster.aggregate.splitpoints']
= 'master.Server.numDeadRegionServers'
+
+    expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '500'
+    expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '7'
     expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2432'
     expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '512'
     expected["ams-env"]['properties']['metrics_collector_heapsize'] = '640'
@@ -2489,6 +2497,40 @@ class TestHDP22StackAdvisor(TestCase):
     self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'].split(',')),
13)
     self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'].split(',')),
2)
 
+    # 2000-nodes cluster
+    for i in range(202, 2001):
+        hosts['items'].extend([{
+            "Hosts": {
+                "host_name": "host" + str(i)
+            }
+        }])
+
+    services['services'] = [
+        {
+            "StackServices": {
+                "service_name": "AMBARI_METRICS"
+            },
+            "components": [
+                {
+                    "StackServiceComponents": {
+                        "component_name": "METRICS_COLLECTOR",
+                        "hostnames": ["host1"]
+                    }
+                },
+                {
+                    "StackServiceComponents": {
+                        "component_name": "METRICS_MONITOR",
+                        "hostnames": ["host" + str(i) for i in range(1, 2001)]
+                    }
+                }
+            ]
+        }
+    ]
+
+    self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.size'],
'700')
+    self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.commit.interval'],
'5')
+
   def test_recommendHbaseConfigurations(self):
     servicesList = ["HBASE"]
     configurations = {}


Mime
View raw message