ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject [14/26] ambari git commit: Revert "AMBARI-20122 - Stack advisor needs to recommend dependency for slaves and masters"
Date Wed, 14 Jun 2017 21:19:45 GMT
Revert "AMBARI-20122 - Stack advisor needs to recommend dependency for slaves and masters"

This reverts commit 0b6d0dcef10ec60c83ad055567b487104573ae62.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/75152614
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/75152614
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/75152614

Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 75152614e84299d16f36a6c6d7387b77a172eb96
Parents: f1ca09c
Author: Jaimin Jetly <jaimin@hortonworks.com>
Authored: Sun Jun 11 22:32:46 2017 -0700
Committer: Jaimin Jetly <jaimin@hortonworks.com>
Committed: Mon Jun 12 15:46:51 2017 -0700

----------------------------------------------------------------------
 .../src/main/resources/stacks/stack_advisor.py  | 115 +-------------
 .../stacks/2.0.6/common/test_stack_advisor.py   | 153 -------------------
 2 files changed, 5 insertions(+), 263 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/75152614/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 6bc8fe4..4a81dc6 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -756,19 +756,15 @@ class DefaultStackAdvisor(StackAdvisor):
       if hostName not in hostsComponentsMap:
         hostsComponentsMap[hostName] = []
 
-    #Sort the services so that the dependent services will be processed before those that
depend on them.
-    sortedServices = self.getServicesSortedByDependencies(services)
     #extend hostsComponentsMap' with MASTER components
-    for service in sortedServices:
+    for service in services["services"]:
       masterComponents = [component for component in service["components"] if self.isMasterComponent(component)]
       serviceName = service["StackServices"]["service_name"]
       serviceAdvisor = self.getServiceAdvisor(serviceName)
       for component in masterComponents:
         componentName = component["StackServiceComponents"]["component_name"]
         advisor = serviceAdvisor if serviceAdvisor is not None else self
-        #Filter the hosts such that only hosts that meet the dependencies are included (if
possible)
-        filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList,
hostsComponentsMap)
-        hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component,
filteredHosts)
+        hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component,
hostsList)
 
         #extend 'hostsComponentsMap' with 'hostsForComponent'
         for hostName in hostsForComponent:
@@ -782,7 +778,7 @@ class DefaultStackAdvisor(StackAdvisor):
     utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
     freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
 
-    for service in sortedServices:
+    for service in services["services"]:
       slaveClientComponents = [component for component in service["components"]
                                if self.isSlaveComponent(component) or self.isClientComponent(component)]
       serviceName = service["StackServices"]["service_name"]
@@ -790,10 +786,7 @@ class DefaultStackAdvisor(StackAdvisor):
       for component in slaveClientComponents:
         componentName = component["StackServiceComponents"]["component_name"]
         advisor = serviceAdvisor if serviceAdvisor is not None else self
-        #Filter the hosts and free hosts such that only hosts that meet the dependencies
are included (if possible)
-        filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList,
hostsComponentsMap)
-        filteredFreeHosts = self.filterList(freeHosts, filteredHosts)
-        hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component,
filteredHosts, filteredFreeHosts)
+        hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component,
hostsList, freeHosts)
 
         #extend 'hostsComponentsMap' with 'hostsForComponent'
         for hostName in hostsForComponent:
@@ -803,7 +796,7 @@ class DefaultStackAdvisor(StackAdvisor):
             hostsComponentsMap[hostName].append( { "name": componentName } )
 
     #colocate custom services
-    for service in sortedServices:
+    for service in services["services"]:
       serviceName = service["StackServices"]["service_name"]
       serviceAdvisor = self.getServiceAdvisor(serviceName)
       if serviceAdvisor is not None:
@@ -873,104 +866,6 @@ class DefaultStackAdvisor(StackAdvisor):
 
     return hostsForComponent
 
-  def getServicesSortedByDependencies(self, services):
-    """
-    Sorts the services based on their dependencies.  This is limited to non-conditional host
scope dependencies.
-    Services with no dependencies will go first.  Services with dependencies will go after
the services they are dependent on.
-    If there are circular dependencies, the services will go in the order in which they were
processed.
-    """
-    processedServices = []
-    sortedServices = []
-
-    for service in services["services"]:
-      self.sortServicesByDependencies(services, service, processedServices, sortedServices)
-
-    return sortedServices
-
-  def sortServicesByDependencies(self, services, service, processedServices, sortedServices):
-    """
-    Sorts the services based on their dependencies.  This is limited to non-conditional host
scope dependencies.
-    Services with no dependencies will go first.  Services with dependencies will go after
the services they are dependent on.
-    If there are circular dependencies, the services will go in the order in which they were
processed.
-    """
-    if service in processedServices:
-      return
-
-    processedServices.append(service)
-
-    for component in service["components"]:
-      dependencies = [] if "dependencies" not in component else component['dependencies']
-      for dependency in dependencies:
-        # accounts only for dependencies that are not conditional
-        conditionsPresent =  "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"]
-        scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"]
-        if not conditionsPresent and scope == "host":
-          componentName = component["StackServiceComponents"]["component_name"]
-          requiredComponentName = dependency["Dependencies"]["component_name"]
-          requiredService = self.getServiceForComponentName(services, requiredComponentName)
-          self.sortServicesByDependencies(services, requiredService, processedServices, sortedServices)
-
-    sortedServices.append(service)
-
-  def getFilteredHostsBasedOnDependencies(self, services, component, hostsList, hostsComponentsMap):
-    """
-    Returns a list of hosts that only includes the ones which have all host scope dependencies
already assigned to them.
-    If an empty list would be returned, instead the full list of hosts are returned.
-    In that case, we can't possibly return a valid recommended layout so we will at least
return a fully filled layout.
-    """
-    removeHosts = []
-    dependencies = [] if "dependencies" not in component else component['dependencies']
-    for dependency in dependencies:
-      # accounts only for dependencies that are not conditional
-      conditionsPresent =  "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"]
-      if not conditionsPresent:
-        componentName = component["StackServiceComponents"]["component_name"]
-        requiredComponentName = dependency["Dependencies"]["component_name"]
-        requiredComponent = self.getRequiredComponent(services, requiredComponentName)
-
-        # We only deal with "host" scope.
-        if (requiredComponent is not None) and (requiredComponent["component_category"] !=
"CLIENT"):
-          scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"]
-          if scope == "host":
-            for host, hostComponents in hostsComponentsMap.iteritems():
-              isRequiredIncluded = False
-              for component in hostComponents:
-                currentComponentName = None if "name" not in component else component["name"]
-                if requiredComponentName == currentComponentName:
-                  isRequiredIncluded = True
-              if not isRequiredIncluded:
-                removeHosts.append(host)
-
-    filteredHostsList = []
-    for host in hostsList:
-      if host not in removeHosts:
-        filteredHostsList.append(host)
-    return filteredHostsList
-
-  def filterList(self, list, filter):
-    """
-    Returns the union of the two lists passed in (list and filter params).
-    """
-    filteredList = []
-    for item in list:
-      if item in filter:
-        filteredList.append(item)
-    return filteredList
-
-  def getServiceForComponentName(self, services, componentName):
-    """
-    Return service for component name
-
-    :type services dict
-    :type componentName str
-    """
-    for service in services["services"]:
-      for component in service["components"]:
-        if self.getComponentName(component) == componentName:
-          return service
-
-    return None
-
   def isComponentUsingCardinalityForLayout(self, componentName):
     return False
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/75152614/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index b6f1965..41c57f6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -191,159 +191,6 @@ class TestHDP206StackAdvisor(TestCase):
     ]
     self.assertValidationResult(expectedItems, result)
 
-
-  def test_handleComponentDependencies(self):
-    services = {
-      "Versions":
-        {
-          "stack_name":"HDP",
-          "stack_version":"2.0.6"
-        },
-      "services" : [
-        {
-          "StackServices" : {
-            "service_name" : "HDFS",
-            "service_version" : "2.0.6",
-          },
-          "components": [
-            {
-              "StackServiceComponents": {
-                "stack_version": "2.0.6",
-                "stack_name": "HDP",
-                "component_category": "MASTER",
-                "is_client": False,
-                "is_master": True,
-                "service_name": "HDFS",
-                "cardinality": "1-2",
-                "hostnames": [],
-                "component_name": "NAMENODE",
-                "display_name": "NameNode"
-              },
-              "dependencies": [
-                {
-                  "Dependencies": {
-                    "stack_name": "HDP",
-                    "stack_version": "2.0.6",
-                    "scope": "cluster",
-                    "conditions": [
-                      {
-                        "configType": "hdfs-site",
-                        "property": "dfs.nameservices",
-                        "type": "PropertyExists",
-                      }
-                    ],
-                    "dependent_service_name": "HDFS",
-                    "dependent_component_name": "NAMENODE",
-                    "component_name": "ZOOKEEPER_SERVER"
-                  }
-                }
-              ]
-            }
-          ]
-        },
-        {
-          "StackServices" : {
-            "service_name" : "ZOOKEEPER",
-            "service_version" : "2.0.6",
-          },
-          "components": [
-            {
-              "StackServiceComponents": {
-                "stack_version": "2.0.6",
-                "stack_name": "HDP",
-                "component_category": "MASTER",
-                "is_client": False,
-                "is_master": True,
-                "service_name": "HDFS",
-                "cardinality": "1-2",
-                "hostnames": [],
-                "component_name": "ZOOKEEPER_SERVER",
-                "display_name": "ZooKeeper Server"
-              },
-              "dependencies": []
-            }
-          ]
-        }
-      ]
-    }
-
-    nameNodeDependencies = services["services"][0]["components"][0]["dependencies"][0]["Dependencies"]
-
-    # Tests for master component with dependencies
-
-    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org",
"c6404.ambari.apache.org"])
-    services["services"][1]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org",
"c6403.ambari.apache.org"]
-    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-    # Assert that dependencies are ignored when there are conditions and cluster scope
-    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
-
-    nameNodeDependencies["scope"] = "host"
-    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-    # Assert that dependencies are ignored when there are conditions (even for host scope)
-    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
-
-    nameNodeDependencies["scope"] = "cluster"
-    originalConditions = nameNodeDependencies["conditions"]
-    nameNodeDependencies["conditions"] = []
-    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-    # Assert that dependencies are ignored when scope is cluster
-    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
-
-    nameNodeDependencies["scope"] = "host"
-    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-    # Assert that dependencies are enforced for host scope without conditions
-    #self.assertEquals(recommendations, "")
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][0]['components']),
2)
-
-    services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = False
-    services["services"][1]["components"][0]["StackServiceComponents"]["component_category"]
= "CLIENT"
-    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-    # Assert that dependencies are ignored when depending on client components
-    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
-
-    # Tests for slave component with dependencies
-    services["services"][0]["components"][0]["StackServiceComponents"]["component_category"]
= "SLAVE"
-    services["services"][0]["components"][0]["StackServiceComponents"]["is_master"] = False
-    services["services"][1]["components"][0]["StackServiceComponents"]["component_category"]
= "MASTER"
-    services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = True
-
-    nameNodeDependencies["scope"] = "cluster"
-    nameNodeDependencies["conditions"] = originalConditions
-    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-    # Assert that dependencies are ignored when there are conditions and cluster scope
-    self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']),
1)
-    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
-
-    nameNodeDependencies["scope"] = "host"
-    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-    # Assert that dependencies are ignored when there are conditions (even for host scope)
-    self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']),
1)
-    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
-
-    nameNodeDependencies["scope"] = "cluster"
-    nameNodeDependencies["conditions"] = []
-    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-    # Assert that dependencies are ignored when scope is cluster
-    self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']),
1)
-    self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']),
1)
-
-    nameNodeDependencies["scope"] = "host"
-    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
-    # Assert that dependencies are enforced when host scope and no conditions
-    self.assertEquals(recommendations['blueprint']['host_groups'][1]['components'][1]['name'],
'NAMENODE')
-    self.assertEquals(len(recommendations['blueprint']['host_groups'][1]['components']),
2)
-
-
   def test_validateRequiredComponentsPresent(self):
     services = {
       "Versions":


Mime
View raw message