ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-20085 Confusing AMS collector heap size validation loop (dsen)
Date Thu, 23 Feb 2017 13:53:40 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 986e7a9be -> bb7b83f2b


AMBARI-20085 Confusing AMS collector heap size validation loop (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bb7b83f2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bb7b83f2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bb7b83f2

Branch: refs/heads/trunk
Commit: bb7b83f2bb1b6af2dad38e98ffdcddda13e1cb2b
Parents: 986e7a9
Author: Dmytro Sen <dsen@apache.org>
Authored: Thu Feb 23 15:52:02 2017 +0200
Committer: Dmytro Sen <dsen@apache.org>
Committed: Thu Feb 23 15:52:21 2017 +0200

----------------------------------------------------------------------
 .../stacks/HDP/2.0.6/services/stack_advisor.py      |  1 +
 .../stacks/HDP/2.5/services/stack_advisor.py        |  8 ++++++++
 .../src/main/resources/stacks/stack_advisor.py      | 12 ++++++------
 .../python/stacks/2.5/common/test_stack_advisor.py  | 16 ++++++++++++++++
 .../src/test/resources/stacks/old_stack_advisor.py  | 12 ++++++------
 .../stacks/HDF/2.0/services/stack_advisor.py        |  2 +-
 6 files changed, 38 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 5c68f15..61960bb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -1150,6 +1150,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
           if unusedMemory > 4*gb and collector_needs_increase:  # warn user, if more than
4GB RAM is unused
             recommended_collector_heapsize = int((unusedMemory - 4*gb)/5) + collector_heapsize
* mb
+            recommended_collector_heapsize = min(16*gb, recommended_collector_heapsize) #Make
sure heapsize <= 16GB
             recommended_collector_heapsize = round_to_n(recommended_collector_heapsize/mb,128)
# Round to 128m multiple
             if collector_heapsize < recommended_collector_heapsize:
               validation_msg = "Consider allocating {0} MB to metrics_collector_heapsize
in ams-env to use up some " \

http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 52ada52..6f3dfa7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -142,6 +142,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
     return self.toConfigurationValidationProblems(validationItems, "storm-site")
 
+  def getCardinalitiesDict(self, hosts):
+    result = super(HDP25StackAdvisor, self).getCardinalitiesDict(hosts)
+    min_val = 1
+    if len(hosts["items"]) > 999:
+      min_val = 2
+    result['METRICS_COLLECTOR'] = {"min": min_val}
+    return result
+
   def validateAtlasConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
     application_properties = self.getSiteProperties(configurations, "application-properties")
     validationItems = []

http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 04c6baf..0b81700 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -806,7 +806,7 @@ class DefaultStackAdvisor(StackAdvisor):
       return component["StackServiceComponents"]["hostnames"]
 
     if len(hostsList) > 1 and self.isMasterComponentWithMultipleInstances(component):
-      hostsCount = self.getMinComponentCount(component)
+      hostsCount = self.getMinComponentCount(component, hosts)
       if hostsCount > 1: # get first 'hostsCount' available hosts
         hostsForComponent = []
         hostIndex = 0
@@ -1460,13 +1460,13 @@ class DefaultStackAdvisor(StackAdvisor):
     service = self.getNotValuableComponents()
     return componentName in service
 
-  def getMinComponentCount(self, component):
+  def getMinComponentCount(self, component, hosts):
     componentName = self.getComponentName(component)
-    return self.getComponentCardinality(componentName)["min"]
+    return self.getComponentCardinality(componentName, hosts)["min"]
 
   # Helper dictionaries
-  def getComponentCardinality(self, componentName):
-    dict = self.getCardinalitiesDict()
+  def getComponentCardinality(self, componentName, hosts):
+    dict = self.getCardinalitiesDict(hosts)
     if componentName in dict:
       return dict[componentName]
     else:
@@ -1508,7 +1508,7 @@ class DefaultStackAdvisor(StackAdvisor):
   def getNotPreferableOnServerComponents(self):
     return self.notPreferableOnServerComponents
 
-  def getCardinalitiesDict(self):
+  def getCardinalitiesDict(self, hosts):
     return self.cardinalitiesDict
 
   def getComponentLayoutSchemes(self):

http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 6890ef6..cf1c0ee 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -404,6 +404,12 @@ class TestHDP25StackAdvisor(TestCase):
       data = json.load(f)
     return data
 
+  def prepareNHosts(self, host_count):
+    names = []
+    for i in range(0, host_count):
+      names.append("hostname" + str(i))
+    return self.prepareHosts(names)
+
   def prepareHosts(self, hostsNames):
     hosts = { "items": [] }
     for hostName in hostsNames:
@@ -435,6 +441,16 @@ class TestHDP25StackAdvisor(TestCase):
   def __getHosts(self, componentsList, componentName):
     return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"]
== componentName][0]
 
+  def test_getCardinalitiesDict(self):
+    hosts = self.prepareNHosts(5)
+    actual = self.stackAdvisor.getCardinalitiesDict(hosts)
+    expected = {'METRICS_COLLECTOR': {'min': 1}}
+    self.assertEquals(actual, expected)
+
+    hosts = self.prepareNHosts(1001)
+    actual = self.stackAdvisor.getCardinalitiesDict(hosts)
+    expected = {'METRICS_COLLECTOR': {'min': 2}}
+    self.assertEquals(actual, expected)
 
   def test_getComponentLayoutValidations_one_hsi_host(self):
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/test/resources/stacks/old_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/old_stack_advisor.py b/ambari-server/src/test/resources/stacks/old_stack_advisor.py
index 8a880e0..ee5674b 100644
--- a/ambari-server/src/test/resources/stacks/old_stack_advisor.py
+++ b/ambari-server/src/test/resources/stacks/old_stack_advisor.py
@@ -359,7 +359,7 @@ class DefaultStackAdvisor(StackAdvisor):
             availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)]
 
           if self.isMasterComponentWithMultipleInstances(component):
-            hostsCount = self.getMinComponentCount(component)
+            hostsCount = self.getMinComponentCount(component, hosts)
             if hostsCount > 1: # get first 'hostsCount' available hosts
               if len(availableHosts) < hostsCount:
                 hostsCount = len(availableHosts)
@@ -524,13 +524,13 @@ class DefaultStackAdvisor(StackAdvisor):
     service = self.getNotValuableComponents()
     return componentName in service
 
-  def getMinComponentCount(self, component):
+  def getMinComponentCount(self, component, hosts):
     componentName = self.getComponentName(component)
-    return self.getComponentCardinality(componentName)["min"]
+    return self.getComponentCardinality(componentName, hosts)["min"]
 
   # Helper dictionaries
-  def getComponentCardinality(self, componentName):
-    return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1})
+  def getComponentCardinality(self, componentName, hosts):
+    return self.getCardinalitiesDict(hosts).get(componentName, {"min": 1, "max": 1})
 
   def getHostForComponent(self, component, hostsList):
     componentName = self.getComponentName(component)
@@ -568,7 +568,7 @@ class DefaultStackAdvisor(StackAdvisor):
   def getNotPreferableOnServerComponents(self):
     return []
 
-  def getCardinalitiesDict(self):
+  def getCardinalitiesDict(self, hosts):
     return {}
 
   def getComponentLayoutSchemes(self):

http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
index 40cc847..da33b95 100644
--- a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
@@ -1615,7 +1615,7 @@ class HDF20StackAdvisor(DefaultStackAdvisor):
   def getNotPreferableOnServerComponents(self):
     return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'METRICS_COLLECTOR']
 
-  def getCardinalitiesDict(self):
+  def getCardinalitiesDict(self, hosts):
     return {
       'ZOOKEEPER_SERVER': {"min": 3},
       'METRICS_COLLECTOR': {"min": 1}


Mime
View raw message