ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [23/34] ambari git commit: AMBARI-20628. Ambari doesn't set properties correctly (magyari_sandor)
Date Mon, 17 Apr 2017 20:18:17 GMT
AMBARI-20628. Ambari doesn't set properties correctly (magyari_sandor)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bf637950
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bf637950
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bf637950

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: bf637950901f2b0f213c1cc149f5e49b8a8d0968
Parents: c57300a
Author: Sandor Magyari <smagyari@hortonworks.com>
Authored: Thu Apr 13 17:04:14 2017 +0200
Committer: Sandor Magyari <smagyari@hortonworks.com>
Committed: Fri Apr 14 10:20:09 2017 +0200

----------------------------------------------------------------------
 .../StackAdvisorBlueprintProcessor.java         | 55 ++++----------------
 .../topology/ClusterConfigurationRequest.java   |  8 +--
 .../StackAdvisorBlueprintProcessorTest.java     |  2 -
 3 files changed, 16 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/bf637950/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
index 0abcc14..d306e25 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
@@ -32,10 +32,8 @@ import org.apache.ambari.server.controller.internal.ConfigurationTopologyExcepti
 import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.state.ValueAttributesInfo;
 import org.apache.ambari.server.topology.AdvisedConfiguration;
-import org.apache.ambari.server.topology.Blueprint;
 import org.apache.ambari.server.topology.ClusterTopology;
 import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
-import org.apache.ambari.server.topology.Configuration;
 import org.apache.ambari.server.topology.HostGroup;
 import org.apache.ambari.server.topology.HostGroupInfo;
 import org.slf4j.Logger;
@@ -75,13 +73,13 @@ public class StackAdvisorBlueprintProcessor {
   /**
    * Recommend configurations by the stack advisor, then store the results in cluster topology.
    * @param clusterTopology cluster topology instance
-   * @param existingConfigurations Existing configurations of cluster
+   * @param userProvidedConfigurations User configurations of cluster provided in Blueprint
+ Cluster template
    */
-  public void adviseConfiguration(ClusterTopology clusterTopology, Map<String, Map<String,
String>> existingConfigurations) throws ConfigurationTopologyException {
+  public void adviseConfiguration(ClusterTopology clusterTopology, Map<String, Map<String,
String>> userProvidedConfigurations) throws ConfigurationTopologyException {
     StackAdvisorRequest request = createStackAdvisorRequest(clusterTopology, StackAdvisorRequestType.CONFIGURATIONS);
     try {
       RecommendationResponse response = stackAdvisorHelper.recommend(request);
-      addAdvisedConfigurationsToTopology(response, clusterTopology, existingConfigurations);
+      addAdvisedConfigurationsToTopology(response, clusterTopology, userProvidedConfigurations);
     } catch (StackAdvisorException e) {
       throw new ConfigurationTopologyException(RECOMMENDATION_FAILED, e);
     } catch (IllegalArgumentException e) {
@@ -94,7 +92,7 @@ public class StackAdvisorBlueprintProcessor {
     Map<String, Set<String>> hgComponentsMap = gatherHostGroupComponents(clusterTopology);
     Map<String, Set<String>> hgHostsMap = gatherHostGroupBindings(clusterTopology);
     Map<String, Set<String>> componentHostsMap = gatherComponentsHostsMap(hgComponentsMap,
-      hgHostsMap);
+            hgHostsMap);
     return StackAdvisorRequest.StackAdvisorRequestBuilder
       .forStack(stack.getName(), stack.getVersion())
       .forServices(new ArrayList<>(clusterTopology.getBlueprint().getServices()))
@@ -167,7 +165,7 @@ public class StackAdvisorBlueprintProcessor {
   }
 
   private void addAdvisedConfigurationsToTopology(RecommendationResponse response,
-                                                  ClusterTopology topology, Map<String,
Map<String, String>> existingConfigurations) {
+                                                  ClusterTopology topology, Map<String,
Map<String, String>> userProvidedConfigurations) {
     Preconditions.checkArgument(response.getRecommendations() != null,
       "Recommendation response is empty.");
     Preconditions.checkArgument(response.getRecommendations().getBlueprint() != null,
@@ -175,67 +173,36 @@ public class StackAdvisorBlueprintProcessor {
     Preconditions.checkArgument(response.getRecommendations().getBlueprint().getConfigurations()
!= null,
       "Configurations are missing from the recommendation blueprint response.");
 
-    Map<String, Map<String, String>> userProvidedProperties = getUserProvidedProperties(topology,
existingConfigurations);
     Map<String, BlueprintConfigurations> recommendedConfigurations =
       response.getRecommendations().getBlueprint().getConfigurations();
     for (Map.Entry<String, BlueprintConfigurations> configEntry : recommendedConfigurations.entrySet())
{
       String configType = configEntry.getKey();
       BlueprintConfigurations blueprintConfig = filterBlueprintConfig(configType, configEntry.getValue(),
-        userProvidedProperties, topology);
+              userProvidedConfigurations, topology);
       topology.getAdvisedConfigurations().put(configType, new AdvisedConfiguration(
         blueprintConfig.getProperties(), blueprintConfig.getPropertyAttributes()));
     }
   }
 
   /**
-   * Gather user defined properties. (keep that only which is not included in the stack defaults
or it overrides the stack default value)
-   */
-  private Map<String, Map<String, String>> getUserProvidedProperties(ClusterTopology
topology, Map<String, Map<String, String>> existingConfigurations) {
-    Map<String, Map<String, String>> userProvidedProperties = Maps.newHashMap();
-    Blueprint blueprint = topology.getBlueprint();
-    Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices());
-    Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties();
-
-    for (Map.Entry<String, Map<String, String>> configGroup : existingConfigurations.entrySet())
{
-      String configType = configGroup.getKey();
-      Map<String, String> configsToAdd = Maps.newHashMap();
-      for (Map.Entry<String, String> configProp : configGroup.getValue().entrySet())
{
-        if (stackDefaultProps.containsKey(configType) && stackDefaultProps.get(configType).containsKey(configProp.getKey()))
{
-          String originalValue = stackDefaultProps.get(configType).get(configProp.getKey());
-          if (originalValue != null && !originalValue.equals(configProp.getValue()))
{
-            configsToAdd.put(configProp.getKey(), configProp.getValue());
-          }
-        } else {
-          configsToAdd.put(configProp.getKey(), configProp.getValue());
-        }
-      }
-      if (!configsToAdd.isEmpty()) {
-        userProvidedProperties.put(configGroup.getKey(), configsToAdd);
-      }
-    }
-
-    return userProvidedProperties;
-  }
-
-  /**
-   * Remove user defined properties from stack advisor output in case of ONLY_STACK_DEFAULTS_APPLY
or
+   * Remove user defined properties from Stack Advisor output in case of ONLY_STACK_DEFAULTS_APPLY
or
    * ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES.
    */
   private BlueprintConfigurations filterBlueprintConfig(String configType, BlueprintConfigurations
config,
-                                                        Map<String, Map<String, String>>
userProvidedProperties,
+                                                        Map<String, Map<String, String>>
userProvidedConfigurations,
                                                         ClusterTopology topology) {
     if (topology.getConfigRecommendationStrategy() == ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY
||
       topology.getConfigRecommendationStrategy() == ConfigRecommendationStrategy
         .ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES) {
-      if (userProvidedProperties.containsKey(configType)) {
+      if (userProvidedConfigurations.containsKey(configType)) {
         BlueprintConfigurations newConfig = new BlueprintConfigurations();
         Map<String, String> filteredProps = Maps.filterKeys(config.getProperties(),
-          Predicates.not(Predicates.in(userProvidedProperties.get(configType).keySet())));
+          Predicates.not(Predicates.in(userProvidedConfigurations.get(configType).keySet())));
         newConfig.setProperties(Maps.newHashMap(filteredProps));
 
         if (config.getPropertyAttributes() != null) {
           Map<String, ValueAttributesInfo> filteredAttributes = Maps.filterKeys(config.getPropertyAttributes(),
-            Predicates.not(Predicates.in(userProvidedProperties.get(configType).keySet())));
+            Predicates.not(Predicates.in(userProvidedConfigurations.get(configType).keySet())));
           newConfig.setPropertyAttributes(Maps.newHashMap(filteredAttributes));
         }
         return newConfig;

http://git-wip-us.apache.org/repos/asf/ambari/blob/bf637950/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 5913f4b..0e7d70b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -137,17 +137,19 @@ public class ClusterConfigurationRequest {
     // this will update the topo cluster config and all host group configs in the cluster
topology
     Set<String> updatedConfigTypes = new HashSet<>();
 
-    Configuration clusterConfiguration = clusterTopology.getConfiguration();
-    Map<String, Map<String, String>> existingConfigurations = clusterConfiguration.getFullProperties();
+    Map<String, Map<String, String>> userProvidedConfigurations = clusterTopology.getConfiguration().getFullProperties(1);
 
     try {
       if (configureSecurity) {
+        Configuration clusterConfiguration = clusterTopology.getConfiguration();
+        Map<String, Map<String, String>> existingConfigurations = clusterConfiguration.getFullProperties();
         updatedConfigTypes.addAll(configureKerberos(clusterConfiguration, existingConfigurations));
       }
 
       // obtain recommended configurations before config updates
       if (!ConfigRecommendationStrategy.NEVER_APPLY.equals(this.clusterTopology.getConfigRecommendationStrategy()))
{
-        stackAdvisorBlueprintProcessor.adviseConfiguration(this.clusterTopology, existingConfigurations);
+        // get merged properties form Blueprint & cluster template (this doesn't contains
stack default values)
+        stackAdvisorBlueprintProcessor.adviseConfiguration(this.clusterTopology, userProvidedConfigurations);
       }
 
       updatedConfigTypes.addAll(configurationProcessor.doUpdateForClusterCreate());

http://git-wip-us.apache.org/repos/asf/ambari/blob/bf637950/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
index 49f070a..3c5f8ed 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
@@ -165,9 +165,7 @@ public class StackAdvisorBlueprintProcessorTest {
     underTest.adviseConfiguration(clusterTopology, props);
     // THEN
     assertTrue(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey1"));
-    assertTrue(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey3"));
     assertTrue(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey2"));
-    assertTrue(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey3"));
     assertEquals("dummyValue", advisedConfigurations.get("core-site").getProperties().get("dummyKey1"));
     assertEquals(Boolean.toString(true), advisedConfigurations.get("core-site")
       .getPropertyValueAttributes().get("dummyKey2").getDelete());


Mime
View raw message