ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smn...@apache.org
Subject ambari git commit: AMBARI-15700: Stack advisor - Component layout recommendation needs support for avoiding hosts in maintenance mode
Date Mon, 18 Apr 2016 18:30:05 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 58e7b1259 -> 10ae6cddf


AMBARI-15700: Stack advisor - Component layout recommendation needs support for avoiding hosts
in maintenance mode


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/10ae6cdd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/10ae6cdd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/10ae6cdd

Branch: refs/heads/trunk
Commit: 10ae6cddfbfe27cdd0ac15297159e4da3b30910f
Parents: 58e7b12
Author: Nahappan Somasundaram <nsomasundaram@hortonworks.com>
Authored: Mon Apr 11 10:34:25 2016 -0700
Committer: Nahappan Somasundaram <nsomasundaram@hortonworks.com>
Committed: Mon Apr 18 11:29:57 2016 -0700

----------------------------------------------------------------------
 .../stacks/BIGTOP/0.8/services/stack_advisor.py | 11 +--
 .../stacks/HDP/2.0.6/services/stack_advisor.py  | 12 +--
 .../stacks/HDPWIN/2.1/services/stack_advisor.py | 11 +--
 .../src/main/resources/stacks/stack_advisor.py  | 34 +++++++--
 .../src/test/python/TestStackAdvisor.py         | 80 ++++++++++++++++++++
 5 files changed, 127 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
index 53591cd..701d0d4 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
@@ -29,8 +29,8 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
     items = []
 
     # Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible
-    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
-    hostsCount = len(hostsList)
+    hostsSet = set(super(BaseBIGTOP08StackAdvisor, self).getActiveHosts([host["Hosts"] for
host in hosts["items"]]))
+    hostsCount = len(hostsSet)
 
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item for sublist in componentsListList for item in sublist]
@@ -42,9 +42,10 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
       if component["StackServiceComponents"]["cardinality"] is not None:
          componentName = component["StackServiceComponents"]["component_name"]
          componentDisplayName = component["StackServiceComponents"]["display_name"]
-         componentHostsCount = 0
+         componentHosts = []
          if component["StackServiceComponents"]["hostnames"] is not None:
-           componentHostsCount = len(component["StackServiceComponents"]["hostnames"])
+           componentHosts = [componentHost for componentHost in component["StackServiceComponents"]["hostnames"]
if componentHost in hostsSet]
+         componentHostsCount = len(componentHosts)
          cardinality = str(component["StackServiceComponents"]["cardinality"])
          # cardinality types: null, 1+, 1-2, 1, ALL
          message = None
@@ -71,7 +72,7 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
     # Validating host-usage
     usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if not self.isComponentNotValuable(component)]
     usedHostsList = [item for sublist in usedHostsListList for item in sublist]
-    nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
+    nonUsedHostsList = [item for item in hostsSet if item not in usedHostsList]
     for host in nonUsedHostsList:
       items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not
used', "host": str(host) } )
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 0130483..6dd1d84 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -32,8 +32,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     items = []
 
     # Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible
-    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
-    hostsCount = len(hostsList)
+    # Use a set for fast lookup
+    hostsSet =  set(super(HDP206StackAdvisor, self).getActiveHosts([host["Hosts"] for host
in hosts["items"]]))  #[host["Hosts"]["host_name"] for host in hosts["items"]]
+    hostsCount = len(hostsSet)
 
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item for sublist in componentsListList for item in sublist]
@@ -45,9 +46,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       if component["StackServiceComponents"]["cardinality"] is not None:
          componentName = component["StackServiceComponents"]["component_name"]
          componentDisplayName = component["StackServiceComponents"]["display_name"]
-         componentHostsCount = 0
+         componentHosts = []
          if component["StackServiceComponents"]["hostnames"] is not None:
-           componentHostsCount = len(component["StackServiceComponents"]["hostnames"])
+           componentHosts = [componentHost for componentHost in component["StackServiceComponents"]["hostnames"]
if componentHost in hostsSet]
+         componentHostsCount = len(componentHosts)
          cardinality = str(component["StackServiceComponents"]["cardinality"])
          # cardinality types: null, 1+, 1-2, 1, ALL
          message = None
@@ -74,7 +76,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     # Validating host-usage
     usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if not self.isComponentNotValuable(component)]
     usedHostsList = [item for sublist in usedHostsListList for item in sublist]
-    nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
+    nonUsedHostsList = [item for item in hostsSet if item not in usedHostsList]
     for host in nonUsedHostsList:
       items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not
used', "host": str(host) } )
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
index b99c484..cf52ade 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
@@ -31,8 +31,8 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
     items = []
 
     # Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible
-    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
-    hostsCount = len(hostsList)
+    hostsSet =  set(super(HDPWIN21StackAdvisor, self).getActiveHosts([host["Hosts"] for host
in hosts["items"]]))
+    hostsCount = len(hostsSet)
 
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item for sublist in componentsListList for item in sublist]
@@ -44,9 +44,10 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
       if component["StackServiceComponents"]["cardinality"] is not None:
          componentName = component["StackServiceComponents"]["component_name"]
          componentDisplayName = component["StackServiceComponents"]["display_name"]
-         componentHostsCount = 0
+         componentHosts = []
          if component["StackServiceComponents"]["hostnames"] is not None:
-           componentHostsCount = len(component["StackServiceComponents"]["hostnames"])
+           componentHosts = [componentHost for componentHost in component["StackServiceComponents"]["hostnames"]
if componentHost in hostsSet]
+         componentHostsCount = len(componentHosts)
          cardinality = str(component["StackServiceComponents"]["cardinality"])
          # cardinality types: null, 1+, 1-2, 1, ALL
          message = None
@@ -73,7 +74,7 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
     # Validating host-usage
     usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if not self.isComponentNotValuable(component)]
     usedHostsList = [item for sublist in usedHostsListList for item in sublist]
-    nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
+    nonUsedHostsList = [item for item in hostsSet if item not in usedHostsList]
     for host in nonUsedHostsList:
       items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not
used', "host": str(host) } )
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 539bd25..9979e7e 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -309,12 +309,25 @@ class DefaultStackAdvisor(StackAdvisor):
   implement
   """
 
+  """
+  Filters the list of specified hosts object and returns
+  a list of hosts which are not in maintenance mode.
+  """
+  def getActiveHosts(self, hosts):
+    hostsList = []
+
+    if (hosts is not None):
+      hostsList = [host['host_name'] for host in hosts
+                   if host.get('maintenance_state') is None or host.get('maintenance_state')
== "OFF"]
+
+    return hostsList
+
   def recommendComponentLayout(self, services, hosts):
     """Returns Services object with hostnames array populated for components"""
 
     stackName = services["Versions"]["stack_name"]
     stackVersion = services["Versions"]["stack_version"]
-    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    hostsList = self.getActiveHosts([host["Hosts"] for host in hosts["items"]])
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
 
     layoutRecommendations = self.createComponentLayoutRecommendations(services, hosts)
@@ -339,14 +352,17 @@ class DefaultStackAdvisor(StackAdvisor):
       }
     }
 
-    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    hostsList = self.getActiveHosts([host["Hosts"] for host in hosts["items"]])
+
+    # for fast lookup
+    hostsSet = set(hostsList)
 
     hostsComponentsMap = {}
     for hostName in hostsList:
       if hostName not in hostsComponentsMap:
         hostsComponentsMap[hostName] = []
 
-    #extend 'hostsComponentsMap' with MASTER components
+    #extend hostsComponentsMap' with MASTER components
     for service in services["services"]:
       masterComponents = [component for component in service["components"] if self.isMasterComponent(component)]
       for component in masterComponents:
@@ -373,7 +389,9 @@ class DefaultStackAdvisor(StackAdvisor):
 
         #extend 'hostsComponentsMap' with 'hostsForComponent'
         for hostName in hostsForComponent:
-          hostsComponentsMap[hostName].append( { "name":componentName } )
+          if hostName in hostsSet:
+            hostsComponentsMap[hostName].append( { "name":componentName } )
+
     #extend 'hostsComponentsMap' with Slave and Client Components
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item for sublist in componentsListList for item in sublist]
@@ -422,9 +440,10 @@ class DefaultStackAdvisor(StackAdvisor):
 
         #extend 'hostsComponentsMap' with 'hostsForComponent'
         for hostName in hostsForComponent:
-          if hostName not in hostsComponentsMap:
+          if hostName not in hostsComponentsMap and hostName in hostsSet:
             hostsComponentsMap[hostName] = []
-          hostsComponentsMap[hostName].append( { "name": componentName } )
+          if hostName in hostsSet:
+            hostsComponentsMap[hostName].append( { "name": componentName } )
 
     #prepare 'host-group's from 'hostsComponentsMap'
     host_groups = recommendations["blueprint"]["host_groups"]
@@ -621,6 +640,9 @@ class DefaultStackAdvisor(StackAdvisor):
     return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1})
 
   def getHostForComponent(self, component, hostsList):
+    if (len(hostsList) == 0):
+      return None
+
     componentName = self.getComponentName(component)
 
     if len(hostsList) != 1:

http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/test/python/TestStackAdvisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestStackAdvisor.py b/ambari-server/src/test/python/TestStackAdvisor.py
index 149ae1d..87d2d15 100644
--- a/ambari-server/src/test/python/TestStackAdvisor.py
+++ b/ambari-server/src/test/python/TestStackAdvisor.py
@@ -340,6 +340,86 @@ class TestStackAdvisorInitialization(TestCase):
     }
     self.assertEquals(actualRecommendLayoutResponse, expectedRecommendLayoutResponse)
 
+    # Test with maintenance_state. One host is in maintenance mode.
+    hosts= {
+      "items": [
+        {"Hosts": {"host_name": "host1", "maintenance_state":"OFF"}},
+        {"Hosts": {"host_name": "host2", "maintenance_state":"ON"}}
+      ]
+    }
+
+    actualRecommendLayoutResponse = default_stack_advisor.recommendComponentLayout(services,
hosts)
+    expectedRecommendLayoutResponse = {
+      "services": ["GANGLIA", "HBASE", "HDFS", "PIG", "TEZ", "ZOOKEEPER"],
+      "recommendations": {
+        "blueprint": {
+          "host_groups": [
+            {
+              "name": "host-group-1",
+              "components": [
+                {
+                  "name": "GANGLIA_SERVER"
+                },
+                {
+                  "name": "HBASE_MASTER"
+                },
+                {
+                  "name": "NAMENODE"
+                },
+                {
+                  "name": "SECONDARY_NAMENODE"
+                },
+                {
+                  "name": "ZOOKEEPER_SERVER"
+                },
+                {
+                  "name": "ZOOKEEPER_CLIENT"
+                }
+              ]
+            }
+          ]
+        },
+        "blueprint_cluster_binding":
+          {
+            "host_groups": [
+              {
+                "hosts": [{"fqdn": "host1"}],
+                "name": "host-group-1"
+              }
+            ]
+          }
+      },
+      "hosts": ["host1"],
+      "Versions": {"stack_name": "HDP1", "stack_version": "2.0.6"}
+    }
+    self.assertEquals(actualRecommendLayoutResponse, expectedRecommendLayoutResponse)
+
+    # Test with maintenance_state. Both hosts are in maintenance mode.
+    hosts= {
+      "items": [
+        {"Hosts": {"host_name": "host1", "maintenance_state":"ON"}},
+        {"Hosts": {"host_name": "host2", "maintenance_state":"ON"}}
+      ]
+    }
+
+    actualRecommendLayoutResponse = default_stack_advisor.recommendComponentLayout(services,
hosts)
+
+    expectedRecommendLayoutResponse = {
+      "Versions": {"stack_name": "HDP1", "stack_version": "2.0.6"},
+      "hosts": [],
+      "services": ['GANGLIA', 'HBASE', 'HDFS', 'PIG', 'TEZ', 'ZOOKEEPER'],
+      "recommendations": {
+        "blueprint": {
+          "host_groups": []
+        },
+        "blueprint_cluster_binding": {
+          "host_groups": []
+        }
+      }
+    }
+
+    self.assertEquals(actualRecommendLayoutResponse, expectedRecommendLayoutResponse)
+
     # Config groups support by default
     services["config-groups"] = [{
       "configurations": {


Mime
View raw message