ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-20085 Confusing AMS collector heap size validation loop (dsen)
Date Thu, 23 Feb 2017 14:01:03 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 447fe8723 -> 4c96b84dc


AMBARI-20085 Confusing AMS collector heap size validation loop (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4c96b84d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4c96b84d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4c96b84d

Branch: refs/heads/branch-2.5
Commit: 4c96b84dc8a82441723f3e192a0f2b78ea989044
Parents: 447fe87
Author: Dmytro Sen <dsen@apache.org>
Authored: Thu Feb 23 16:00:53 2017 +0200
Committer: Dmytro Sen <dsen@apache.org>
Committed: Thu Feb 23 16:00:53 2017 +0200

----------------------------------------------------------------------
 .../stacks/BIGTOP/0.8/services/stack_advisor.py     |  2 +-
 .../stacks/HDP/2.0.6/services/stack_advisor.py      |  3 ++-
 .../stacks/HDP/2.2/services/stack_advisor.py        |  4 ++--
 .../stacks/HDP/2.5/services/stack_advisor.py        |  8 ++++++++
 .../stacks/HDPWIN/2.1/services/stack_advisor.py     |  2 +-
 .../stacks/HDPWIN/2.2/services/stack_advisor.py     |  4 ++--
 .../src/main/resources/stacks/stack_advisor.py      | 12 ++++++------
 .../python/stacks/2.5/common/test_stack_advisor.py  | 16 ++++++++++++++++
 .../src/test/resources/stacks/old_stack_advisor.py  | 12 ++++++------
 9 files changed, 44 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
index 701d0d4..1d18b44 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
@@ -284,7 +284,7 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
   def getNotPreferableOnServerComponents(self):
     return ['GANGLIA_SERVER']
 
-  def getCardinalitiesDict(self):
+  def getCardinalitiesDict(self, hosts):
     return {
       'ZOOKEEPER_SERVER': {"min": 3},
       'HBASE_MASTER': {"min": 1},

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index dad8106..5e04b53 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -1501,6 +1501,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
           if unusedMemory > 4*gb and collector_needs_increase:  # warn user, if more than
4GB RAM is unused
             recommended_collector_heapsize = int((unusedMemory - 4*gb)/5) + collector_heapsize
* mb
+            recommended_collector_heapsize = min(16*gb, recommended_collector_heapsize) #Make
sure heapsize <= 16GB
             recommended_collector_heapsize = round_to_n(recommended_collector_heapsize/mb,128)
# Round to 128m multiple
             if collector_heapsize < recommended_collector_heapsize:
               validation_msg = "Consider allocating {0} MB to metrics_collector_heapsize
in ams-env to use up some " \
@@ -1856,7 +1857,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
   def getNotPreferableOnServerComponents(self):
     return ['GANGLIA_SERVER', 'METRICS_COLLECTOR']
 
-  def getCardinalitiesDict(self):
+  def getCardinalitiesDict(self, hosts):
     return {
       'ZOOKEEPER_SERVER': {"min": 3},
       'HBASE_MASTER': {"min": 1},

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index f4583ba..d45cf96 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -1695,8 +1695,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     result.extend(['METRICS_MONITOR'])
     return result
 
-  def getCardinalitiesDict(self):
-    result = super(HDP22StackAdvisor, self).getCardinalitiesDict()
+  def getCardinalitiesDict(self, hosts):
+    result = super(HDP22StackAdvisor, self).getCardinalitiesDict(hosts)
     result['METRICS_COLLECTOR'] = {"min": 1}
     return result
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 8d7aaf7..45fae53 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -142,6 +142,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
     return self.toConfigurationValidationProblems(validationItems, "storm-site")
 
+  def getCardinalitiesDict(self, hosts):
+    result = super(HDP25StackAdvisor, self).getCardinalitiesDict(hosts)
+    min_val = 1
+    if len(hosts["items"]) > 999:
+      min_val = 2
+    result['METRICS_COLLECTOR'] = {"min": min_val}
+    return result
+
   def validateAtlasConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
     application_properties = self.getSiteProperties(configurations, "application-properties")
     validationItems = []

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
index cf52ade..e1df728 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
@@ -517,7 +517,7 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
   def getNotPreferableOnServerComponents(self):
     return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS']
 
-  def getCardinalitiesDict(self):
+  def getCardinalitiesDict(self, hosts):
     return {
       'ZOOKEEPER_SERVER': {"min": 3},
       'HBASE_MASTER': {"min": 1},

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
index 88cc6bb..833edf1 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
@@ -1092,8 +1092,8 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor):
     result.extend(['METRICS_COLLECTOR'])
     return result
 
-  def getCardinalitiesDict(self):
-    result = super(HDPWIN22StackAdvisor, self).getCardinalitiesDict()
+  def getCardinalitiesDict(self, hosts):
+    result = super(HDPWIN22StackAdvisor, self).getCardinalitiesDict(hosts)
     result['METRICS_COLLECTOR'] = {"min": 1}
     return result
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 2dce53a..e393b9c 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -478,7 +478,7 @@ class DefaultStackAdvisor(StackAdvisor):
       return component["StackServiceComponents"]["hostnames"]
 
     if len(hostsList) > 1 and self.isMasterComponentWithMultipleInstances(component):
-      hostsCount = self.getMinComponentCount(component)
+      hostsCount = self.getMinComponentCount(component, hosts)
       if hostsCount > 1: # get first 'hostsCount' available hosts
         hostsForComponent = []
         hostIndex = 0
@@ -823,13 +823,13 @@ class DefaultStackAdvisor(StackAdvisor):
     service = self.getNotValuableComponents()
     return componentName in service
 
-  def getMinComponentCount(self, component):
+  def getMinComponentCount(self, component, hosts):
     componentName = self.getComponentName(component)
-    return self.getComponentCardinality(componentName)["min"]
+    return self.getComponentCardinality(componentName, hosts)["min"]
 
   # Helper dictionaries
-  def getComponentCardinality(self, componentName):
-    return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1})
+  def getComponentCardinality(self, componentName, hosts):
+    return self.getCardinalitiesDict(hosts).get(componentName, {"min": 1, "max": 1})
 
   def getHostForComponent(self, component, hostsList):
     if len(hostsList) == 0:
@@ -863,7 +863,7 @@ class DefaultStackAdvisor(StackAdvisor):
   def getNotPreferableOnServerComponents(self):
     return []
 
-  def getCardinalitiesDict(self):
+  def getCardinalitiesDict(self, hosts):
     return {}
 
   def getComponentLayoutSchemes(self):

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 6890ef6..07d3941 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -404,6 +404,12 @@ class TestHDP25StackAdvisor(TestCase):
       data = json.load(f)
     return data
 
+  def prepareNHosts(self, host_count):
+    names = []
+    for i in range(0, host_count):
+      names.append("hostname" + str(i))
+    return self.prepareHosts(names)
+
   def prepareHosts(self, hostsNames):
     hosts = { "items": [] }
     for hostName in hostsNames:
@@ -435,6 +441,16 @@ class TestHDP25StackAdvisor(TestCase):
   def __getHosts(self, componentsList, componentName):
     return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"]
== componentName][0]
 
+  def test_getCardinalitiesDict(self):
+    hosts = self.prepareNHosts(5)
+    actual = self.stackAdvisor.getCardinalitiesDict(hosts)
+    expected = {'ZOOKEEPER_SERVER': {'min': 3}, 'HBASE_MASTER': {'min': 1}, 'METRICS_COLLECTOR':
{'min': 1}}
+    self.assertEquals(actual, expected)
+
+    hosts = self.prepareNHosts(1001)
+    actual = self.stackAdvisor.getCardinalitiesDict(hosts)
+    expected = {'ZOOKEEPER_SERVER': {'min': 3}, 'HBASE_MASTER': {'min': 1}, 'METRICS_COLLECTOR':
{'min': 2}}
+    self.assertEquals(actual, expected)
 
   def test_getComponentLayoutValidations_one_hsi_host(self):
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/test/resources/stacks/old_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/old_stack_advisor.py b/ambari-server/src/test/resources/stacks/old_stack_advisor.py
index 8a880e0..ee5674b 100644
--- a/ambari-server/src/test/resources/stacks/old_stack_advisor.py
+++ b/ambari-server/src/test/resources/stacks/old_stack_advisor.py
@@ -359,7 +359,7 @@ class DefaultStackAdvisor(StackAdvisor):
             availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)]
 
           if self.isMasterComponentWithMultipleInstances(component):
-            hostsCount = self.getMinComponentCount(component)
+            hostsCount = self.getMinComponentCount(component, hosts)
             if hostsCount > 1: # get first 'hostsCount' available hosts
               if len(availableHosts) < hostsCount:
                 hostsCount = len(availableHosts)
@@ -524,13 +524,13 @@ class DefaultStackAdvisor(StackAdvisor):
     service = self.getNotValuableComponents()
     return componentName in service
 
-  def getMinComponentCount(self, component):
+  def getMinComponentCount(self, component, hosts):
     componentName = self.getComponentName(component)
-    return self.getComponentCardinality(componentName)["min"]
+    return self.getComponentCardinality(componentName, hosts)["min"]
 
   # Helper dictionaries
-  def getComponentCardinality(self, componentName):
-    return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1})
+  def getComponentCardinality(self, componentName, hosts):
+    return self.getCardinalitiesDict(hosts).get(componentName, {"min": 1, "max": 1})
 
   def getHostForComponent(self, component, hostsList):
     componentName = self.getComponentName(component)
@@ -568,7 +568,7 @@ class DefaultStackAdvisor(StackAdvisor):
   def getNotPreferableOnServerComponents(self):
     return []
 
-  def getCardinalitiesDict(self):
+  def getCardinalitiesDict(self, hosts):
     return {}
 
   def getComponentLayoutSchemes(self):


Mime
View raw message