ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: Revert "AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen)"
Date Fri, 22 Jul 2016 16:10:37 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk c879e9373 -> 40bdcb648


Revert "AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb'
after dependency change (dsen)"

This reverts commit b5b9493ff7da82f342a7b39c2c8d273d763c8eae.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/40bdcb64
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/40bdcb64
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/40bdcb64

Branch: refs/heads/trunk
Commit: 40bdcb6480abfe6393bac9be961ae53654c1cb2f
Parents: c879e93
Author: Dmytro Sen <dsen@apache.org>
Authored: Fri Jul 22 19:10:16 2016 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Fri Jul 22 19:10:16 2016 +0300

----------------------------------------------------------------------
 .../commands/StackAdvisorCommand.java           |  6 ----
 .../stacks/HDP/2.2/services/stack_advisor.py    | 34 ++++----------------
 .../src/main/resources/stacks/stack_advisor.py  | 16 +++------
 .../commands/StackAdvisorCommandTest.java       | 18 -----------
 .../stacks/2.2/common/test_stack_advisor.py     | 21 +++++++-----
 5 files changed, 24 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/40bdcb64/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index f93e180..6f612af 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -98,7 +98,6 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse>
extend
   private static final String CONFIGURATIONS_PROPERTY = "configurations";
   private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
   private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties";
-  private static final String STACK_ADVISOR_COMMAND_TYPE_PROPERTY = "stack-advisor-command-type";
 
   private File recommendationsDir;
   private String recommendationsArtifactsLifetime;
@@ -164,7 +163,6 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse>
extend
       populateConfigurations(root, request);
       populateConfigGroups(root, request);
       populateAmbariServerInfo(root);
-      populateCommandType(root);
       data.servicesJSON = mapper.writeValueAsString(root);
     } catch (Exception e) {
       // should not happen
@@ -185,10 +183,6 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse>
extend
     }
   }
 
-  protected void populateCommandType(ObjectNode root) throws StackAdvisorException {
-    root.put(STACK_ADVISOR_COMMAND_TYPE_PROPERTY, getCommandType().toString());
-  }
-
   private void populateConfigurations(ObjectNode root,
                                       StackAdvisorRequest request) {
     Map<String, Map<String, Map<String, String>>> configurations =

http://git-wip-us.apache.org/repos/asf/ambari/blob/40bdcb64/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 416447e..b77980a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -26,8 +26,6 @@ import socket
 import re
 import xml.etree.ElementTree as ET
 
-from stack_advisor import ActionType
-
 class HDP22StackAdvisor(HDP21StackAdvisor):
 
   def getServiceConfigurationRecommenderDict(self):
@@ -72,11 +70,14 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
 
       # Above is the default calculated 'maximum' values derived purely from hosts.
-      # However, there are 'maximum' and other attributes that actually should come from
services["configurations"] during validations.
+      # However, there are 'maximum' and other attributes that actually change based on the
values
+      #  of other configs. We need to update those values.
 
       # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
       # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
is always 5120 during first recommendation.
-      if (self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS and "yarn-site"
in services["configurations"]):
+      # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
+      # in this case (if 'changed-configurations' are empty).
+      if ('changed-configurations' in services.keys() and services['changed-configurations']
and "yarn-site" in services["configurations"]):
         if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]):
           putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
           putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
@@ -354,19 +355,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size,
yarnMaxAllocationSize)
 
     putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]),
container_size))
-
-    # "maximum" and "minimum" attributes should come from services["configurations"] during
validations.
-    # If one of dependencies are in changed-confogurations another can be absent. In this
case we need to take both values from 'services'.
-    if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations'
in services and services['changed-configurations'] \
-        and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb',
services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb',
services['changed-configurations']))):
-      if "yarn-site" in services["configurations"]:
-        if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
-          putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
-        if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
-          putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
-    else:
-      putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
-      putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+    putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
+    putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
 
     putHiveSiteProperty("hive.prewarm.enabled", "false")
     putHiveSiteProperty("hive.prewarm.numcontainers", "3")
@@ -1047,16 +1037,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site")
     yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
     yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),
int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
-
-    # "maximum" and "minimum" attributes should come from services["configurations"] during
validations.
-    # If one of dependencies are in changed-confogurations another can be absent. In this
case we need to take both values from 'services'.
-    if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations'
in services and services['changed-configurations'] \
-        and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb',
services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb',
services['changed-configurations']))):
-      if "yarn-site" in services["configurations"] and "yarn.scheduler.minimum-allocation-mb"
in services["configurations"]["yarn-site"]["properties"] \
-          and "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
-        yarnMinAllocationSize = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
-        yarnMaxAllocationSize = min(30 * int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),
int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
-
     putMapredPropertyAttribute("mapreduce.map.memory.mb", "maximum", yarnMaxAllocationSize)
     putMapredPropertyAttribute("mapreduce.map.memory.mb", "minimum", yarnMinAllocationSize)
     putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "maximum", yarnMaxAllocationSize)

http://git-wip-us.apache.org/repos/asf/ambari/blob/40bdcb64/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 380c8f4..437fe4f 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -298,12 +298,6 @@ class StackAdvisor(object):
     """
     pass
 
-class ActionType:
-  RECOMMEND_COMPONENT_LAYOUT_ACTION = 'recommend-component-layout'
-  VALIDATE_COMPONENT_LAYOUT_ACTION = 'validate-component-layout'
-  RECOMMEND_CONFIGURATIONS = 'recommend-configurations'
-  RECOMMEND_CONFIGURATION_DEPENDENCIES = 'recommend-configuration-dependencies'
-  VALIDATE_CONFIGURATIONS = 'validate-configurations'
 
 class DefaultStackAdvisor(StackAdvisor):
   """
@@ -319,6 +313,7 @@ class DefaultStackAdvisor(StackAdvisor):
     # Dictionary that maps serviceName or componentName to serviceAdvisor
     self.serviceAdvisorsDict = {}
 
+
   def getActiveHosts(self, hosts):
     """ Filters the list of specified hosts object and returns
         a list of hosts which are not in maintenance mode. """
@@ -992,13 +987,13 @@ class DefaultStackAdvisor(StackAdvisor):
       config[configType]["properties"] = {}
     def appendProperty(key, value):
       # If property exists in changedConfigs, do not override, use user defined property
-      if self.isPropertyInChangedConfigs(configType, key, changedConfigs):
+      if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
         config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
       else:
         config[configType]["properties"][key] = str(value)
     return appendProperty
 
-  def isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
+  def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
     for changedConfig in changedConfigs:
       if changedConfig['type']==configType and changedConfig['name']==propertyName:
         return True
@@ -1021,7 +1016,7 @@ class DefaultStackAdvisor(StackAdvisor):
 
     def updatePropertyWithCallback(key, value, callback):
       # If property exists in changedConfigs, do not override, use user defined property
-      if self.isPropertyInChangedConfigs(configType, key, changedConfigs):
+      if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
         config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
       else:
         # Give the callback an empty string if the mapping doesn't exist
@@ -1088,6 +1083,3 @@ class DefaultStackAdvisor(StackAdvisor):
 
   def getServiceNames(self, services):
     return [service["StackServices"]["service_name"] for service in services["services"]]
-
-  def getActionType(self, services):
-    return services.get("stack-advisor-command-type", ActionType.RECOMMEND_CONFIGURATIONS)

http://git-wip-us.apache.org/repos/asf/ambari/blob/40bdcb64/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
index a956789..25d634b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
@@ -241,24 +241,6 @@ public class StackAdvisorCommandTest {
   }
 
   @Test
-  public void testPopulateCommandType() throws Exception {
-    File file = mock(File.class);
-    String recommendationsArtifactsLifetime = "1w";
-    StackAdvisorRunner stackAdvisorRunner = mock(StackAdvisorRunner.class);
-    AmbariMetaInfo ambariMetaInfo = mock(AmbariMetaInfo.class);
-    StackAdvisorCommand<TestResource> cmd = new TestStackAdvisorCommand(file, recommendationsArtifactsLifetime,
"test", 1,
-      stackAdvisorRunner, ambariMetaInfo);
-    ObjectNode objectNode = (ObjectNode) cmd.mapper.readTree("{\"Versions\": " +
-      "{\"stack_name\": \"stack\", \"stack_version\":\"1.0.0\"}}");
-
-    cmd.populateCommandType(objectNode);
-
-    JsonNode commandType = objectNode.get("stack-advisor-command-type");
-    assertNotNull(commandType);
-    assertEquals(StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT.toString(), commandType.getTextValue());
-  }
-
-  @Test
   public void testPopulateStackHierarchy_noParents() throws Exception {
     File file = mock(File.class);
     String recommendationsArtifactsLifetime = "1w";

http://git-wip-us.apache.org/repos/asf/ambari/blob/40bdcb64/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 41d8409..5fe534a 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1136,19 +1136,24 @@ class TestHDP22StackAdvisor(TestCase):
     self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
     self.assertEquals(configurations, expected)
 
-    # During a recommendation request, we should get updated 'maximum's from recommendations
-    # During a validation request, update from services.
+    # With no 'changed-configurations', we should get updated 'maximum's from recommendations
+    # Else update from services.
     #
     # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
     # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
is always 5120 during first recommendation.
-
-    # Test validate-configurations:
-    services.pop("changed-configurations", None)
-    services["stack-advisor-command-type"] = 'validate-configurations'
+    # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
+    # in this case (if 'changed-configurations' are empty).
+    #
+    # Test not empty 'changed-configurations':
+    services["changed-configurations"] = [
+      {
+        "type": "yarn-env",
+        "name": "min_user_id"
+      }
+    ]
+    services.pop("configurations", None)
     services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb":
'4321', "yarn.nodemanager.resource.cpu-vcores": '9'}},
                                   "yarn-env": {"properties": {"min_user_id": "500"}}}
-
-    # Expected recommendations from 'servicess'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"]
= '9'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"]
= '9'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"]
= '4321'


Mime
View raw message