ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject [20/25] ambari git commit: AMBARI-20122 - Stack advisor needs to recommend dependency for slaves and masters
Date Mon, 19 Jun 2017 20:16:42 GMT
AMBARI-20122 - Stack advisor needs to recommend dependency for slaves and masters


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/927d5c83
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/927d5c83
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/927d5c83

Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 927d5c8340e5a22cd2fb8b8a2d14956b19810b93
Parents: e8e9781
Author: Tim Thorpe <tthorpe@apache.org>
Authored: Mon Jun 19 09:00:20 2017 -0700
Committer: Tim Thorpe <tthorpe@apache.org>
Committed: Mon Jun 19 09:00:20 2017 -0700

----------------------------------------------------------------------
 .../src/main/resources/stacks/stack_advisor.py  | 116 +++++++++++++-
 .../stacks/2.0.6/common/test_stack_advisor.py   | 153 +++++++++++++++++++
 2 files changed, 264 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/927d5c83/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 4a81dc6..3a39a34 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -756,15 +756,19 @@ class DefaultStackAdvisor(StackAdvisor):
       if hostName not in hostsComponentsMap:
         hostsComponentsMap[hostName] = []
 
+    #Sort the services so that the dependent services will be processed before those that
depend on them.
+    sortedServices = self.getServicesSortedByDependencies(services)
     #extend hostsComponentsMap' with MASTER components
-    for service in services["services"]:
+    for service in sortedServices:
       masterComponents = [component for component in service["components"] if self.isMasterComponent(component)]
       serviceName = service["StackServices"]["service_name"]
       serviceAdvisor = self.getServiceAdvisor(serviceName)
       for component in masterComponents:
         componentName = component["StackServiceComponents"]["component_name"]
         advisor = serviceAdvisor if serviceAdvisor is not None else self
-        hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component,
hostsList)
+        #Filter the hosts such that only hosts that meet the dependencies are included (if
possible)
+        filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList,
hostsComponentsMap)
+        hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component,
filteredHosts)
 
         #extend 'hostsComponentsMap' with 'hostsForComponent'
         for hostName in hostsForComponent:
@@ -778,7 +782,7 @@ class DefaultStackAdvisor(StackAdvisor):
     utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
     freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
 
-    for service in services["services"]:
+    for service in sortedServices:
       slaveClientComponents = [component for component in service["components"]
                                if self.isSlaveComponent(component) or self.isClientComponent(component)]
       serviceName = service["StackServices"]["service_name"]
@@ -786,7 +790,10 @@ class DefaultStackAdvisor(StackAdvisor):
       for component in slaveClientComponents:
         componentName = component["StackServiceComponents"]["component_name"]
         advisor = serviceAdvisor if serviceAdvisor is not None else self
-        hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component,
hostsList, freeHosts)
+        #Filter the hosts and free hosts such that only hosts that meet the dependencies
are included (if possible)
+        filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList,
hostsComponentsMap)
+        filteredFreeHosts = self.filterList(freeHosts, filteredHosts)
+        hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component,
filteredHosts, filteredFreeHosts)
 
         #extend 'hostsComponentsMap' with 'hostsForComponent'
         for hostName in hostsForComponent:
@@ -796,7 +803,7 @@ class DefaultStackAdvisor(StackAdvisor):
             hostsComponentsMap[hostName].append( { "name": componentName } )
 
     #colocate custom services
-    for service in services["services"]:
+    for service in sortedServices:
       serviceName = service["StackServices"]["service_name"]
       serviceAdvisor = self.getServiceAdvisor(serviceName)
       if serviceAdvisor is not None:
@@ -866,6 +873,105 @@ class DefaultStackAdvisor(StackAdvisor):
 
     return hostsForComponent
 
+  def getServicesSortedByDependencies(self, services):
+    """
+    Sorts the services based on their dependencies.  This is limited to non-conditional host
scope dependencies.
+    Services with no dependencies will go first.  Services with dependencies will go after
the services they are dependent on.
+    If there are circular dependencies, the services will go in the order in which they were
processed.
+    """
+    processedServices = []
+    sortedServices = []
+
+    for service in services["services"]:
+      self.sortServicesByDependencies(services, service, processedServices, sortedServices)
+
+    return sortedServices
+
+  def sortServicesByDependencies(self, services, service, processedServices, sortedServices):
+    """
+    Sorts the services based on their dependencies.  This is limited to non-conditional host
scope dependencies.
+    Services with no dependencies will go first.  Services with dependencies will go after
the services they are dependent on.
+    If there are circular dependencies, the services will go in the order in which they were
processed.
+    """
+    if service is None or service in processedServices:
+      return
+
+    processedServices.append(service)
+
+    components = [] if "components" not in service else service["components"]
+    for component in components:
+      dependencies = [] if "dependencies" not in component else component['dependencies']
+      for dependency in dependencies:
+        # accounts only for dependencies that are not conditional
+        conditionsPresent =  "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"]
+        scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"]
+        if not conditionsPresent and scope == "host":
+          componentName = component["StackServiceComponents"]["component_name"]
+          requiredComponentName = dependency["Dependencies"]["component_name"]
+          requiredService = self.getServiceForComponentName(services, requiredComponentName)
+          self.sortServicesByDependencies(services, requiredService, processedServices, sortedServices)
+
+    sortedServices.append(service)
+
+  def getFilteredHostsBasedOnDependencies(self, services, component, hostsList, hostsComponentsMap):
+    """
+    Returns a list of hosts that only includes the ones which have all host scope dependencies
already assigned to them.
+    If an empty list would be returned, instead the full list of hosts are returned.
+    In that case, we can't possibly return a valid recommended layout so we will at least
return a fully filled layout.
+    """
+    removeHosts = []
+    dependencies = [] if "dependencies" not in component else component['dependencies']
+    for dependency in dependencies:
+      # accounts only for dependencies that are not conditional
+      conditionsPresent =  "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"]
+      if not conditionsPresent:
+        componentName = component["StackServiceComponents"]["component_name"]
+        requiredComponentName = dependency["Dependencies"]["component_name"]
+        requiredComponent = self.getRequiredComponent(services, requiredComponentName)
+
+        # We only deal with "host" scope.
+        if (requiredComponent is not None) and (requiredComponent["component_category"] !=
"CLIENT"):
+          scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"]
+          if scope == "host":
+            for host, hostComponents in hostsComponentsMap.iteritems():
+              isRequiredIncluded = False
+              for component in hostComponents:
+                currentComponentName = None if "name" not in component else component["name"]
+                if requiredComponentName == currentComponentName:
+                  isRequiredIncluded = True
+              if not isRequiredIncluded:
+                removeHosts.append(host)
+
+    filteredHostsList = []
+    for host in hostsList:
+      if host not in removeHosts:
+        filteredHostsList.append(host)
+    return filteredHostsList
+
+  def filterList(self, list, filter):
+    """
+    Returns the union of the two lists passed in (list and filter params).
+    """
+    filteredList = []
+    for item in list:
+      if item in filter:
+        filteredList.append(item)
+    return filteredList
+
+  def getServiceForComponentName(self, services, componentName):
+    """
+    Return service for component name
+
+    :type services dict
+    :type componentName str
+    """
+    for service in services["services"]:
+      for component in service["components"]:
+        if self.getComponentName(component) == componentName:
+          return service
+
+    return None
+
   def isComponentUsingCardinalityForLayout(self, componentName):
     return False
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/927d5c83/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 41c57f6..b6f1965 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -191,6 +191,159 @@ class TestHDP206StackAdvisor(TestCase):
     ]
     self.assertValidationResult(expectedItems, result)
 
+
+  def test_handleComponentDependencies(self):
+    services = {
+      "Versions":
+        {
+          "stack_name":"HDP",
+          "stack_version":"2.0.6"
+        },
+      "services" : [
+        {
+          "StackServices" : {
+            "service_name" : "HDFS",
+            "service_version" : "2.0.6",
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "stack_version": "2.0.6",
+                "stack_name": "HDP",
+                "component_category": "MASTER",
+                "is_client": False,
+                "is_master": True,
+                "service_name": "HDFS",
+                "cardinality": "1-2",
+                "hostnames": [],
+                "component_name": "NAMENODE",
+                "display_name": "NameNode"
+              },
+              "dependencies": [
+                {
+                  "Dependencies": {
+                    "stack_name": "HDP",
+                    "stack_version": "2.0.6",
+                    "scope": "cluster",
+                    "conditions": [
+                      {
+                        "configType": "hdfs-site",
+                        "property": "dfs.nameservices",
+                        "type": "PropertyExists",
+                      }
+                    ],
+                    "dependent_service_name": "HDFS",
+                    "dependent_component_name": "NAMENODE",
+                    "component_name": "ZOOKEEPER_SERVER"
+                  }
+                }
+              ]
+            }
+          ]
+        },
+        {
+          "StackServices" : {
+            "service_name" : "ZOOKEEPER",
+            "service_version" : "2.0.6",
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "stack_version": "2.0.6",
+                "stack_name": "HDP",
+                "component_category": "MASTER",
+                "is_client": False,
+                "is_master": True,
+                "service_name": "HDFS",
+                "cardinality": "1-2",
+                "hostnames": [],
+                "component_name": "ZOOKEEPER_SERVER",
+                "display_name": "ZooKeeper Server"
+              },
+              "dependencies": []
+            }
+          ]
+        }
+      ]
+    }
+
+    nameNodeDependencies = services["services"][0]["components"][0]["dependencies"][0]["Dependencies"]
+
+    # Tests for master component with dependencies
+
+    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org",
"c6404.ambari.apache.org"])
+    services["services"][1]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org",
"c6403.ambari.apache.org"]
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when there are conditions and cluster scope
+    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
+
+    nameNodeDependencies["scope"] = "host"
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when there are conditions (even for host scope)
+    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
+
+    nameNodeDependencies["scope"] = "cluster"
+    originalConditions = nameNodeDependencies["conditions"]
+    nameNodeDependencies["conditions"] = []
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when scope is cluster
+    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
+
+    nameNodeDependencies["scope"] = "host"
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are enforced for host scope without conditions
+    #self.assertEquals(recommendations, "")
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][0]['components']),
2)
+
+    services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = False
+    services["services"][1]["components"][0]["StackServiceComponents"]["component_category"]
= "CLIENT"
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when depending on client components
+    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
+
+    # Tests for slave component with dependencies
+    services["services"][0]["components"][0]["StackServiceComponents"]["component_category"]
= "SLAVE"
+    services["services"][0]["components"][0]["StackServiceComponents"]["is_master"] = False
+    services["services"][1]["components"][0]["StackServiceComponents"]["component_category"]
= "MASTER"
+    services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = True
+
+    nameNodeDependencies["scope"] = "cluster"
+    nameNodeDependencies["conditions"] = originalConditions
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when there are conditions and cluster scope
+    self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']),
1)
+    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
+
+    nameNodeDependencies["scope"] = "host"
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when there are conditions (even for host scope)
+    self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']),
1)
+    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
+
+    nameNodeDependencies["scope"] = "cluster"
+    nameNodeDependencies["conditions"] = []
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are ignored when scope is cluster
+    self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']),
1)
+    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
+
+    nameNodeDependencies["scope"] = "host"
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    # Assert that dependencies are enforced when host scope and no conditions
+    self.assertEquals(recommendations['blueprint']['host_groups'][1]['components'][1]['name'],
'NAMENODE')
+    self.assertEquals(len(recommendations['blueprint']['host_groups'][1]['components']),
2)
+
+
   def test_validateRequiredComponentsPresent(self):
     services = {
       "Versions":


Mime
View raw message