ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jspei...@apache.org
Subject [2/2] git commit: AMBARI-6762. Include configuration in exported blueprints.
Date Thu, 07 Aug 2014 02:33:11 GMT
AMBARI-6762.  Include configuration in exported blueprints.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8d464c2b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8d464c2b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8d464c2b

Branch: refs/heads/trunk
Commit: 8d464c2b286c522a47e2eb12f55d67e59ee61d37
Parents: 3e1ad2a
Author: John Speidel <jspeidel@hortonworks.com>
Authored: Wed Aug 6 17:56:02 2014 -0400
Committer: John Speidel <jspeidel@hortonworks.com>
Committed: Wed Aug 6 22:32:48 2014 -0400

----------------------------------------------------------------------
 .../query/render/ClusterBlueprintRenderer.java  | 400 +++++++-
 .../AmbariManagementControllerImpl.java         |  12 +-
 .../server/controller/ConfigurationRequest.java |  20 +
 .../internal/BaseBlueprintProcessor.java        |  82 +-
 .../BlueprintConfigurationProcessor.java        | 700 ++++++++++++++
 .../internal/ClusterResourceProvider.java       | 406 +-------
 .../internal/ConfigurationResourceProvider.java |  13 +-
 .../server/controller/internal/HostGroup.java   |  56 ++
 .../render/ClusterBlueprintRendererTest.java    |  32 +-
 .../BlueprintConfigurationProcessorTest.java    | 932 +++++++++++++++++++
 .../internal/ClusterResourceProviderTest.java   | 103 --
 11 files changed, 2182 insertions(+), 574 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8d464c2b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
index db3dff8..1afae39 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
@@ -19,6 +19,7 @@
 package org.apache.ambari.server.api.query.render;
 
 import org.apache.ambari.server.api.query.QueryInfo;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.api.services.Request;
 import org.apache.ambari.server.api.services.Result;
 import org.apache.ambari.server.api.services.ResultImpl;
@@ -26,15 +27,25 @@ import org.apache.ambari.server.api.services.ResultPostProcessor;
 import org.apache.ambari.server.api.services.ResultPostProcessorImpl;
 import org.apache.ambari.server.api.util.TreeNode;
 import org.apache.ambari.server.api.util.TreeNodeImpl;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.AmbariServer;
+import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor;
+import org.apache.ambari.server.controller.internal.HostGroup;
 import org.apache.ambari.server.controller.internal.ResourceImpl;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.HostConfig;
+import org.apache.ambari.server.state.PropertyInfo;
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -44,6 +55,18 @@ import java.util.Set;
  */
 public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
 
+  /**
+   * Management Controller used to get stack information.
+   */
+  private AmbariManagementController controller = AmbariServer.getController();
+
+  /**
+   * Map of configuration type to configuration properties which are required that a user
+   * input.  These properties will be stripped from the exported blueprint.
+   */
+  private Map<String, Collection<String>> propertiesToStrip = new HashMap<String, Collection<String>>();
+
+
   // ----- Renderer ----------------------------------------------------------
 
   @Override
@@ -55,6 +78,17 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
         null, properties, queryProperties.getName());
 
     copyPropertiesToResult(queryProperties, resultTree);
+
+    String configType = Resource.Type.Configuration.name();
+    if (resultTree.getChild(configType) == null) {
+      resultTree.addChild(new HashSet<String>(), configType);
+    }
+
+    String serviceType = Resource.Type.Service.name();
+    if (resultTree.getChild(serviceType) == null) {
+      resultTree.addChild(new HashSet<String>(), serviceType);
+    }
+
     String hostType = Resource.Type.Host.name();
     String hostComponentType = Resource.Type.HostComponent.name();
     TreeNode<Set<String>> hostComponentNode = resultTree.getChild(
@@ -67,6 +101,7 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
       }
       hostComponentNode = hostNode.addChild(new HashSet<String>(), hostComponentType);
     }
+    resultTree.getChild(configType).getObject().add("properties");
     hostComponentNode.getObject().add("HostRoles/component_name");
 
     return resultTree;
@@ -106,50 +141,141 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
   private Resource createBlueprintResource(TreeNode<Resource> clusterNode) {
     Resource clusterResource = clusterNode.getObject();
     Resource blueprintResource = new ResourceImpl(Resource.Type.Cluster);
-    String clusterName = (String) clusterResource.getPropertyValue(
-        PropertyHelper.getPropertyId("Clusters", "cluster_name"));
-    //todo: deal with name collision?
-    String blueprintName = "blueprint-" + clusterName;
+
     String[] stackTokens = ((String) clusterResource.getPropertyValue(
             PropertyHelper.getPropertyId("Clusters", "version"))).split("-");
 
-    blueprintResource.setProperty("Blueprints/blueprint_name", blueprintName);
     blueprintResource.setProperty("Blueprints/stack_name", stackTokens[0]);
     blueprintResource.setProperty("Blueprints/stack_version", stackTokens[1]);
-    blueprintResource.setProperty(
-        "host_groups", processHostGroups(clusterNode.getChild("hosts")));
+
+    Collection<HostGroupImpl> hostGroups =  processHostGroups(clusterNode.getChild("hosts"));
+
+    List<Map<String, Object>> groupList = formatGroupsAsList(hostGroups);
+    blueprintResource.setProperty("host_groups", groupList);
+
+    determinePropertiesToStrip(clusterNode.getChild("services"), stackTokens[0], stackTokens[1]);
+
+    blueprintResource.setProperty("configurations", processConfigurations(clusterNode, hostGroups));
 
     return blueprintResource;
   }
 
   /**
-   * Process host group information for all hosts.
+   * Determine which configuration properties need to be stripped from the configuration prior to exporting.
+   * Stripped properties are any property which are marked as required in the stack definition.  For example,
+   * all passwords are required properties and are therefore not exported.
+   *
+   * @param servicesNode  services node
+   * @param stackName     stack name
+   * @param stackVersion  stack version
+   */
+  private void determinePropertiesToStrip(TreeNode<Resource> servicesNode, String stackName, String stackVersion) {
+    AmbariMetaInfo stackInfo = getController().getAmbariMetaInfo();
+    for (TreeNode<Resource> service : servicesNode.getChildren()) {
+      String name = (String) service.getObject().getPropertyValue("ServiceInfo/service_name");
+      Map<String, PropertyInfo> requiredProperties = stackInfo.getRequiredProperties(stackName, stackVersion, name);
+      for (Map.Entry<String, PropertyInfo> entry : requiredProperties.entrySet()) {
+        String propertyName = entry.getKey();
+        PropertyInfo propertyInfo = entry.getValue();
+        String configCategory = propertyInfo.getFilename();
+        if (configCategory.endsWith(".xml")) {
+          configCategory = configCategory.substring(0, configCategory.indexOf(".xml"));
+        }
+        Collection<String> categoryProperties = propertiesToStrip.get(configCategory);
+        if (categoryProperties == null) {
+          categoryProperties = new ArrayList<String>();
+          propertiesToStrip.put(configCategory, categoryProperties);
+        }
+        categoryProperties.add(propertyName);
+      }
+    }
+  }
+
+  /**
+   * Process cluster scoped configurations.
    *
-   * @param hostNode a host node
+   * @param clusterNode  cluster node
+   * @param hostGroups   all host groups
    *
-   * @return list of host group property maps, one element for each host group
+   * @return cluster configuration
    */
-  private List<Map<String, Object>> processHostGroups(TreeNode<Resource> hostNode) {
-    Map<HostGroup, HostGroup> mapHostGroups = new HashMap<HostGroup, HostGroup>();
+  private List<Map<String, Map<String, String>>>  processConfigurations(TreeNode<Resource> clusterNode,
+                                                                        Collection<HostGroupImpl> hostGroups) {
+
+    List<Map<String, Map<String, String>>> configList = new ArrayList<Map<String, Map<String, String>>>();
+
+    Map<String, Object> desiredConfigMap = clusterNode.getObject().getPropertiesMap().get("Clusters/desired_configs");
+    TreeNode<Resource> configNode = clusterNode.getChild("configurations");
+    for (TreeNode<Resource> config : configNode.getChildren()) {
+      Configuration configuration = new Configuration(config);
+      DesiredConfig desiredConfig = (DesiredConfig) desiredConfigMap.get(configuration.getType());
+      if (desiredConfig != null && desiredConfig.getTag().equals(configuration.getTag())) {
+        Map<String, Map<String, String>> properties = Collections.singletonMap(
+            configuration.getType(), configuration.getProperties());
+
+        BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties);
+        properties = updater.doUpdateForBlueprintExport(hostGroups);
+        configList.add(properties);
+      }
+    }
+    return configList;
+  }
+
+  /**
+   * Process cluster host groups.
+   *
+   * @param hostNode  host node
+   *
+   * @return collection of host groups
+   */
+  private Collection<HostGroupImpl> processHostGroups(TreeNode<Resource> hostNode) {
+    Map<HostGroupImpl, HostGroupImpl> mapHostGroups = new HashMap<HostGroupImpl, HostGroupImpl>();
+    int count = 1;
     for (TreeNode<Resource> host : hostNode.getChildren()) {
-      HostGroup group = HostGroup.parse(host);
+      HostGroupImpl group = new HostGroupImpl(host);
+      String hostName = (String) host.getObject().getPropertyValue(
+          PropertyHelper.getPropertyId("Hosts", "host_name"));
+
       if (mapHostGroups.containsKey(group)) {
-        mapHostGroups.get(group).incrementCardinality();
+        HostGroupImpl hostGroup = mapHostGroups.get(group);
+        hostGroup.incrementCardinality();
+        hostGroup.addHost(hostName);
       } else {
         mapHostGroups.put(group, group);
+        group.setName("host_group_" + count++);
+        group.addHost(hostName);
       }
     }
+    return mapHostGroups.values();
+  }
 
-    int count = 1;
+
+  /**
+   * Process host group information for all hosts.
+   *
+   * @param hostGroups all host groups
+   *
+   * @return list of host group property maps, one element for each host group
+   */
+  private List<Map<String, Object>> formatGroupsAsList(Collection<HostGroupImpl> hostGroups) {
     List<Map<String, Object>> listHostGroups = new ArrayList<Map<String, Object>>();
-    for (HostGroup group : mapHostGroups.values()) {
-      String groupName = "host_group_" + count++;
+    for (HostGroupImpl group : hostGroups) {
       Map<String, Object> mapGroupProperties = new HashMap<String, Object>();
       listHostGroups.add(mapGroupProperties);
 
-      mapGroupProperties.put("name", groupName);
+      mapGroupProperties.put("name", group.getName());
       mapGroupProperties.put("cardinality", String.valueOf(group.getCardinality()));
       mapGroupProperties.put("components", processHostGroupComponents(group));
+      List<Map<String, Map<String, String>>> hostConfigurations = new ArrayList<Map<String, Map<String, String>>>();
+      for (Configuration configuration : group.getConfigurations()) {
+        Map<String, Map<String, String>> propertyMap = Collections.singletonMap(
+            configuration.getType(), configuration.properties);
+        BlueprintConfigurationProcessor configurationProcessor = new BlueprintConfigurationProcessor(propertyMap);
+        Map<String, Map<String, String>> updatedProps = configurationProcessor.doUpdateForBlueprintExport(hostGroups);
+        hostConfigurations.add(updatedProps);
+
+      }
+      mapGroupProperties.put("configurations", hostConfigurations);
     }
     return listHostGroups;
   }
@@ -161,7 +287,7 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
    *
    * @return list of component names for the host
    */
-  private List<Map<String, String>> processHostGroupComponents(HostGroup group) {
+  private List<Map<String, String>> processHostGroupComponents(HostGroupImpl group) {
     List<Map<String, String>> listHostGroupComponents = new ArrayList<Map<String, String>>();
     for (String component : group.getComponents()) {
       Map<String, String> mapComponentProperties = new HashMap<String, String>();
@@ -183,16 +309,37 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
     return isCollection != null && isCollection.equals("true");
   }
 
+  /**
+   * Get management controller instance.
+   *
+   * @return  management controller
+   */
+  protected AmbariManagementController getController() {
+    return controller;
+  }
+
   // ----- Host Group inner class --------------------------------------------
 
   /**
    * Host Group representation.
    */
-  private static class HostGroup {
+  private class HostGroupImpl implements HostGroup {
+
+    /**
+     * Host Group name.
+     *
+     */
+    private String name;
+
     /**
      * Associated components.
      */
-    private Set<String> m_components = new HashSet<String>();
+    private Set<String> components = new HashSet<String>();
+
+    /**
+     * Host group scoped configurations.
+     */
+    private Collection<Configuration> configurations = new HashSet<Configuration>();
 
     /**
      * Number of instances.
@@ -200,35 +347,107 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
     private int m_cardinality = 1;
 
     /**
-     * Factory method for obtaining a host group instance.
-     * Parses a host tree node for host related information.
-     *
-     * @param host  host tree node
-     *
-     * @return a new HostGroup instance
+     * Collection of associated hosts.
      */
-    public static HostGroup parse(TreeNode<Resource> host) {
-      HostGroup group = new HostGroup();
+    private Collection<String> hosts = new HashSet<String>();
 
+    /**
+     * Constructor.
+     *
+     * @param host  host node
+     */
+    public HostGroupImpl(TreeNode<Resource> host) {
       TreeNode<Resource> components = host.getChild("host_components");
       for (TreeNode<Resource> component : components.getChildren()) {
-        group.getComponents().add((String) component.getObject().getPropertyValue(
+        getComponents().add((String) component.getObject().getPropertyValue(
             "HostRoles/component_name"));
       }
-
-      group.addAmbariComponentIfLocalhost((String) host.getObject().getPropertyValue(
+      addAmbariComponentIfLocalhost((String) host.getObject().getPropertyValue(
           PropertyHelper.getPropertyId("Hosts", "host_name")));
 
-      return group;
+      processGroupConfiguration(host);
     }
 
-    /**                                                           `
-     * Obtain associated components.
+    /**
+     * Preocess host group configuration.
      *
-     * @return set of associated components
+     * @param host  host node
      */
+    private void processGroupConfiguration(TreeNode<Resource> host) {
+      Map<String, Object> desiredConfigMap = host.getObject().getPropertiesMap().get("Hosts/desired_configs");
+      if (desiredConfigMap != null) {
+        for (Map.Entry<String, Object> entry : desiredConfigMap.entrySet()) {
+          String type = entry.getKey();
+          HostConfig hostConfig = (HostConfig) entry.getValue();
+          Map<Long, String> overrides = hostConfig.getConfigGroupOverrides();
+
+          if (overrides != null && ! overrides.isEmpty()) {
+            Long version = Collections.max(overrides.keySet());
+            String tag = overrides.get(version);
+            TreeNode<Resource> clusterNode = host.getParent().getParent();
+            TreeNode<Resource> configNode = clusterNode.getChild("configurations");
+            for (TreeNode<Resource> config : configNode.getChildren()) {
+              Configuration configuration = new Configuration(config);
+              if (type.equals(configuration.getType()) && tag.equals(configuration.getTag())) {
+                getConfigurations().add(configuration);
+                break;
+              }
+            }
+          }
+        }
+      }
+    }
+
+    @Override
+    public String getName() {
+      return name;
+    }
+
+    @Override
     public Set<String> getComponents() {
-      return m_components;
+      return components;
+    }
+
+    @Override
+    public Collection<String> getHostInfo() {
+      return hosts;
+    }
+
+    @Override
+    public Map<String, Map<String, String>> getConfigurationProperties() {
+      Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+      for (Configuration configuration : configurations) {
+        properties.put(configuration.getType(), configuration.getProperties());
+      }
+
+      return properties;
+    }
+
+    /**
+     * Set the name.
+     *
+     * @param  name name of host group
+     */
+    public void setName(String name) {
+      this.name = name;
+    }
+
+    /**
+     * Add a host.
+     *
+     * @param host  host to add
+     */
+    public void addHost(String host) {
+      hosts.add(host);
+    }
+
+    /**
+     * Obtain associated host group scoped configurations.
+     *
+     * @return collection of host group scoped configurations
+     */
+    public Collection<Configuration> getConfigurations() {
+      return configurations;
     }
 
     /**
@@ -273,14 +492,115 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
       if (this == o) return true;
       if (o == null || getClass() != o.getClass()) return false;
 
-      HostGroup hostGroup = (HostGroup) o;
+      HostGroupImpl hostGroup = (HostGroupImpl) o;
 
-      return m_components.equals(hostGroup.m_components);
+      return components.equals(hostGroup.components) &&
+          configurations.equals(hostGroup.configurations);
     }
 
     @Override
     public int hashCode() {
-      return m_components.hashCode();
+      int result = components.hashCode();
+      result = 31 * result + configurations.hashCode();
+      return result;
+    }
+  }
+
+  /**
+   * Encapsulates a configuration.
+   */
+  private class Configuration {
+    /**
+     * Configuration type such as hdfs-site.
+     */
+    private String type;
+
+    /**
+     * Configuration tag.
+     */
+    private String tag;
+
+    /**
+     * Properties of the configuration.
+     */
+    private Map<String, String> properties = new HashMap<String, String>();
+
+    /**
+     * Constructor.
+     *
+     * @param configNode  configuration node
+     */
+    @SuppressWarnings("unchecked")
+    public Configuration(TreeNode<Resource> configNode) {
+      Resource configResource = configNode.getObject();
+      type = (String) configResource.getPropertyValue("type");
+      tag  = (String) configResource.getPropertyValue("tag");
+
+      // property map type is currently <String, Object>
+      properties = (Map) configNode.getObject().getPropertiesMap().get("properties");
+      stripRequiredProperties(properties);
+    }
+
+    /**
+     * Get configuration type.
+     *
+     * @return configuration type
+     */
+    public String getType() {
+      return type;
+    }
+
+    /**
+     * Get configuration tag.
+     *
+     * @return configuration tag
+     */
+    public String getTag() {
+      return tag;
+    }
+
+    /**
+     * Get configuration properties.
+     *
+     * @return map of properties and values
+     */
+    public Map<String, String> getProperties() {
+      return properties;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) return true;
+      if (o == null || getClass() != o.getClass()) return false;
+
+      Configuration that = (Configuration) o;
+      return tag.equals(that.tag) && type.equals(that.type) && properties.equals(that.properties);
+    }
+
+    @Override
+    public int hashCode() {
+      int result = type.hashCode();
+      result = 31 * result + tag.hashCode();
+      result = 31 * result + properties.hashCode();
+      return result;
+    }
+
+    /**
+     * Strip required properties from configuration.
+     *
+     * @param properties  property map
+     */
+    private void stripRequiredProperties(Map<String, String> properties) {
+      Iterator<Map.Entry<String, String>> iter = properties.entrySet().iterator();
+      while (iter.hasNext()) {
+        Map.Entry<String, String> entry = iter.next();
+        String property = entry.getKey();
+        String category = getType();
+        Collection<String> categoryProperties = propertiesToStrip.get(category);
+        if (categoryProperties != null && categoryProperties.contains(property)) {
+          iter.remove();
+        }
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8d464c2b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index eff3854..97c155a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -1080,16 +1080,19 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
       }
     }
     else {
+      boolean includeProps = request.includeProperties();
       if (null != request.getType()) {
         Map<String, Config> configs = cluster.getConfigsByType(
             request.getType());
 
         if (null != configs) {
           for (Entry<String, Config> entry : configs.entrySet()) {
+            Config config = entry.getValue();
             ConfigurationResponse response = new ConfigurationResponse(
                 cluster.getClusterName(), request.getType(),
-                entry.getValue().getTag(), entry.getValue().getVersion(), new HashMap<String, String>(),
-                new HashMap<String, Map<String,String>>());
+                config.getTag(), entry.getValue().getVersion(),
+                includeProps ? config.getProperties() : new HashMap<String, String>(),
+                includeProps ? config.getPropertiesAttributes() : new HashMap<String, Map<String,String>>());
             responses.add(response);
           }
         }
@@ -1099,8 +1102,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
 
         for (Config config : all) {
           ConfigurationResponse response = new ConfigurationResponse(
-             cluster.getClusterName(), config.getType(), config.getTag(), config.getVersion(),
-             new HashMap<String, String>(), new HashMap<String, Map<String,String>>());
+              cluster.getClusterName(), config.getType(), config.getTag(), config.getVersion(),
+              includeProps ? config.getProperties() : new HashMap<String, String>(),
+              includeProps ? config.getPropertiesAttributes() : new HashMap<String, Map<String,String>>());
 
           responses.add(response);
         }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8d464c2b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationRequest.java
index 0130de4..7c56eb5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationRequest.java
@@ -34,6 +34,7 @@ public class ConfigurationRequest {
   private Map<String, String> configs;
   private boolean selected = true;
   private Map<String, Map<String, String>> configsAttributes;
+  private boolean includeProperties;
 
   public ConfigurationRequest() {
     configs = new HashMap<String, String>();
@@ -52,6 +53,7 @@ public class ConfigurationRequest {
     this.tag = tag;
     this.configs = configs;
     this.configsAttributes = configsAttributes;
+    this.includeProperties = (type != null && tag != null);
   }
 
   /**
@@ -128,6 +130,24 @@ public class ConfigurationRequest {
   }
 
   /**
+   * Set whether properties should be included.
+   *
+   * @param includeProperties whether properties should be included
+   */
+  public void setIncludeProperties(boolean includeProperties) {
+    this.includeProperties = includeProperties;
+  }
+
+  /**
+   * Determine whether properties should be included.
+   *
+   * @return  true if properties should be included; false otherwise
+   */
+  public boolean includeProperties()  {
+    return this.includeProperties;
+  }
+
+  /**
    * @return Attributes of configs
    */
   public Map<String, Map<String, String>> getPropertiesAttributes() {

http://git-wip-us.apache.org/repos/asf/ambari/blob/8d464c2b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
index d723512..a4165d7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
@@ -84,9 +84,9 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
    *
    * @return collection of host groups which contain the specified component
    */
-  protected Collection<HostGroup> getHostGroupsForComponent(String component, Collection<HostGroup> hostGroups) {
-    Collection<HostGroup> resultGroups = new HashSet<HostGroup>();
-    for (HostGroup group : hostGroups ) {
+  protected Collection<HostGroupImpl> getHostGroupsForComponent(String component, Collection<HostGroupImpl> hostGroups) {
+    Collection<HostGroupImpl> resultGroups = new HashSet<HostGroupImpl>();
+    for (HostGroupImpl group : hostGroups ) {
       if (group.getComponents().contains(component)) {
         resultGroups.add(group);
       }
@@ -102,11 +102,11 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
    *
    * @return map of host group name to host group
    */
-  protected Map<String, HostGroup> parseBlueprintHostGroups(BlueprintEntity blueprint, Stack stack) {
-    Map<String, HostGroup> mapHostGroups = new HashMap<String, HostGroup>();
+  protected Map<String, HostGroupImpl> parseBlueprintHostGroups(BlueprintEntity blueprint, Stack stack) {
+    Map<String, HostGroupImpl> mapHostGroups = new HashMap<String, HostGroupImpl>();
 
     for (HostGroupEntity hostGroup : blueprint.getHostGroups()) {
-      mapHostGroups.put(hostGroup.getName(), new HostGroup(hostGroup, stack));
+      mapHostGroups.put(hostGroup.getName(), new HostGroupImpl(hostGroup, stack));
     }
     return mapHostGroups;
   }
@@ -149,14 +149,14 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
    */
   protected BlueprintEntity validateTopology(BlueprintEntity blueprint) throws AmbariException {
     Stack stack = new Stack(blueprint.getStackName(), blueprint.getStackVersion());
-    Map<String, HostGroup> hostGroupMap = parseBlueprintHostGroups(blueprint, stack);
-    Collection<HostGroup> hostGroups = hostGroupMap.values();
+    Map<String, HostGroupImpl> hostGroupMap = parseBlueprintHostGroups(blueprint, stack);
+    Collection<HostGroupImpl> hostGroups = hostGroupMap.values();
     Map<String, Map<String, String>> clusterConfig = processBlueprintConfigurations(blueprint, null);
     Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies =
         new HashMap<String, Map<String, Collection<DependencyInfo>>>();
 
     Collection<String> services = getTopologyServices(hostGroups);
-    for (HostGroup group : hostGroups) {
+    for (HostGroupImpl group : hostGroups) {
       Map<String, Collection<DependencyInfo>> missingGroupDependencies =
           group.validateTopology(hostGroups, services, clusterConfig);
       if (! missingGroupDependencies.isEmpty()) {
@@ -311,9 +311,9 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
    *
    * @return collections of all services provided by topology
    */
-  protected Collection<String> getTopologyServices(Collection<HostGroup> hostGroups) {
+  protected Collection<String> getTopologyServices(Collection<HostGroupImpl> hostGroups) {
     Collection<String> services = new HashSet<String>();
-    for (HostGroup group : hostGroups) {
+    for (HostGroupImpl group : hostGroups) {
       services.addAll(group.getServices());
     }
     return services;
@@ -359,7 +359,7 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
    * @return collection of missing component information
    */
   private Collection<String> verifyComponentCardinalityCount(BlueprintEntity blueprint,
-                                                             Collection<HostGroup> hostGroups,
+                                                             Collection<HostGroupImpl> hostGroups,
                                                              String component,
                                                              Cardinality cardinality,
                                                              AutoDeployInfo autoDeploy,
@@ -374,11 +374,11 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
       if (! validated && autoDeploy != null && autoDeploy.isEnabled() && cardinality.supportsAutoDeploy()) {
         String coLocateName = autoDeploy.getCoLocate();
         if (coLocateName != null && ! coLocateName.isEmpty()) {
-          Collection<HostGroup> coLocateHostGroups = getHostGroupsForComponent(
+          Collection<HostGroupImpl> coLocateHostGroups = getHostGroupsForComponent(
               coLocateName.split("/")[1], hostGroups);
           if (! coLocateHostGroups.isEmpty()) {
             validated = true;
-            HostGroup group = coLocateHostGroups.iterator().next();
+            HostGroupImpl group = coLocateHostGroups.iterator().next();
             if (group.addComponent(component)) {
               addComponentToBlueprint(blueprint, group.getEntity().getName(), component);
             }
@@ -405,7 +405,7 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
    * @return collection of missing component information
    */
   private Collection<String> verifyComponentInAllHostGroups(BlueprintEntity blueprint,
-                                                            Collection<HostGroup> hostGroups,
+                                                            Collection<HostGroupImpl> hostGroups,
                                                             String component,
                                                             AutoDeployInfo autoDeploy) {
 
@@ -413,7 +413,7 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
     int actualCount = getHostGroupsForComponent(component, hostGroups).size();
     if (actualCount != hostGroups.size()) {
       if (autoDeploy != null && autoDeploy.isEnabled()) {
-        for (HostGroup group : hostGroups) {
+        for (HostGroupImpl group : hostGroups) {
           if (group.addComponent(component)) {
             addComponentToBlueprint(blueprint, group.getEntity().getName(), component);
           }
@@ -846,7 +846,7 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
   /**
    * Host group representation.
    */
-  protected class HostGroup {
+  protected class HostGroupImpl implements HostGroup {
     /**
      * Host group entity
      */
@@ -885,13 +885,28 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
      * @param hostGroup  host group
      * @param stack      stack
      */
-    public HostGroup(HostGroupEntity hostGroup, Stack stack) {
+    public HostGroupImpl(HostGroupEntity hostGroup, Stack stack) {
       this.hostGroup = hostGroup;
       this.stack = stack;
       parseComponents();
       parseConfigurations();
     }
 
+    @Override
+    public String getName() {
+      return hostGroup.getName();
+    }
+
+    @Override
+    public Collection<String> getComponents() {
+      return this.components;
+    }
+
+    @Override
+    public Collection<String> getHostInfo() {
+      return this.hosts;
+    }
+
     /**
      * Associate a host with the host group.
      *
@@ -902,15 +917,6 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
     }
 
     /**
-     * Get associated host information.
-     *
-     * @return collection of hosts associated with the host group
-     */
-    public Collection<String> getHostInfo() {
-      return this.hosts;
-    }
-
-    /**
      * Get the services which are deployed to this host group.
      *
      * @return collection of services which have components in this host group
@@ -920,16 +926,6 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
     }
 
     /**
-     * Get the components associated with the host group.
-     *
-     * @return  collection of component names for the host group
-     */
-    public Collection<String> getComponents() {
-      return this.components;
-    }
-
-
-    /**
      * Add a component to the host group.
      *
      * @param component  component to add
@@ -969,7 +965,7 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
      *
      * @return map of configuration type to a map of properties
      */
-    public Map<String, Map<String, String>> getConfigurations() {
+    public Map<String, Map<String, String>> getConfigurationProperties() {
       return configurations;
     }
 
@@ -991,7 +987,7 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
      *
      * @return map of component to missing dependencies
      */
-    public Map<String, Collection<DependencyInfo>> validateTopology(Collection<HostGroup> hostGroups,
+    public Map<String, Collection<DependencyInfo>> validateTopology(Collection<HostGroupImpl> hostGroups,
                                                                     Collection<String> services,
                                                                     Map<String, Map<String, String>> clusterConfig) {
 
@@ -1059,8 +1055,12 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP
           typeProperties = new HashMap<String, String>();
           configurations.put(type, typeProperties);
         }
-        configurations.put(type, jsonSerializer.<Map<String, String>>fromJson(
-            configEntity.getConfigData(), Map.class));
+        Map<String, String> propertyMap =  jsonSerializer.<Map<String, String>>fromJson(
+            configEntity.getConfigData(), Map.class);
+
+        if (propertyMap != null) {
+          typeProperties.putAll(propertyMap);
+        }
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8d464c2b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
new file mode 100644
index 0000000..c7eef57
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -0,0 +1,700 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller.internal;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Updates configuration properties based on cluster topology.  This is done when exporting
+ * a blueprint and when a cluster is provisioned via a blueprint.
+ */
+public class BlueprintConfigurationProcessor {
+
+  /**
+   * Single host topology updaters
+   */
+  private static Map<String, Map<String, PropertyUpdater>> singleHostTopologyUpdaters =
+      new HashMap<String, Map<String, PropertyUpdater>>();
+
+  /**
+   * Multi host topology updaters
+   */
+  private static Map<String, Map<String, PropertyUpdater>> multiHostTopologyUpdaters =
+      new HashMap<String, Map<String, PropertyUpdater>>();
+
+  /**
+   * Database host topology updaters
+   */
+  private static Map<String, Map<String, PropertyUpdater>> dbHostTopologyUpdaters =
+      new HashMap<String, Map<String, PropertyUpdater>>();
+
+  /**
+   * Updaters for properties which need 'm' appended
+   */
+  private static Map<String, Map<String, PropertyUpdater>> mPropertyUpdaters =
+      new HashMap<String, Map<String, PropertyUpdater>>();
+
+  /**
+   * Collection of all updaters
+   */
+  private static Collection<Map<String, Map<String, PropertyUpdater>>> allUpdaters =
+      new ArrayList<Map<String, Map<String, PropertyUpdater>>>();
+
+  /**
+   * Compiled regex for hostgroup token.
+   */
+  private static Pattern HOSTGROUP_REGEX = Pattern.compile("%HOSTGROUP::(\\S+)%");
+
+  /**
+   * Compiled regex for hostgroup token with port information.
+   */
+  private static Pattern HOSTGROUP_PORT_REGEX = Pattern.compile("%HOSTGROUP::(\\w+|\\d+)%:?(\\d+)?");
+
+  /**
+   * Configuration properties to be updated
+   */
+  private Map<String, Map<String, String>> properties;
+
+
+  /**
+   * Constructor.
+   *
+   * @param properties  properties to update
+   */
+  public BlueprintConfigurationProcessor(Map<String, Map<String, String>> properties) {
+    this.properties = properties;
+  }
+
+  /**
+   * Update properties for cluster creation.  This involves updating topology related properties with
+   * concrete topology information.
+   *
+   * @param hostGroups  host groups of cluster to be deployed
+   *
+   * @return  updated properties
+   */
+  public Map<String, Map<String, String>> doUpdateForClusterCreate(Map<String, ? extends HostGroup> hostGroups) {
+    for (Map<String, Map<String, PropertyUpdater>> updaterMap : allUpdaters) {
+      for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaterMap.entrySet()) {
+        String type = entry.getKey();
+        for (Map.Entry<String, PropertyUpdater> updaterEntry : entry.getValue().entrySet()) {
+          String propertyName = updaterEntry.getKey();
+          PropertyUpdater updater = updaterEntry.getValue();
+
+          Map<String, String> typeMap = properties.get(type);
+          if (typeMap != null && typeMap.containsKey(propertyName)) {
+            typeMap.put(propertyName, updater.updateForClusterCreate(
+                hostGroups, typeMap.get(propertyName), properties));
+          }
+        }
+      }
+    }
+    return properties;
+  }
+
+  /**
+   * Update properties for blueprint export.
+   * This involves converting concrete topology information to host groups.
+   *
+   * @param hostGroups  cluster host groups
+   *
+   * @return  updated properties
+   */
+  public Map<String, Map<String, String>> doUpdateForBlueprintExport(Collection<? extends HostGroup> hostGroups) {
+    doSingleHostExportUpdate(hostGroups, singleHostTopologyUpdaters);
+    doSingleHostExportUpdate(hostGroups, dbHostTopologyUpdaters);
+    doMultiHostExportUpdate(hostGroups, multiHostTopologyUpdaters);
+
+    return properties;
+  }
+
+  /**
+   * Update single host topology configuration properties for blueprint export.
+   *
+   * @param hostGroups  cluster export
+   * @param updaters    registered updaters
+   */
+  private void doSingleHostExportUpdate(Collection<? extends HostGroup> hostGroups,
+                                        Map<String, Map<String, PropertyUpdater>> updaters) {
+
+    for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
+      String type = entry.getKey();
+      for (String propertyName : entry.getValue().keySet()) {
+        boolean matchedHost = false;
+
+        Map<String, String> typeProperties = properties.get(type);
+        if (typeProperties != null && typeProperties.containsKey(propertyName)) {
+          String propValue = typeProperties.get(propertyName);
+          for (HostGroup group : hostGroups) {
+            Collection<String> hosts = group.getHostInfo();
+            for (String host : hosts) {
+              if (propValue.contains(host)) {    //todo: need to use regular expression to avoid matching a host which is a superset.  Can this be fixed???
+                matchedHost = true;
+                typeProperties.put(propertyName, propValue.replace(
+                    host, "%HOSTGROUP::" + group.getName() + "%"));
+                break;
+              }
+            }
+            if (matchedHost) {
+              break;
+            }
+          }
+          if (! matchedHost) {
+            typeProperties.remove(propertyName);
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Update multi host topology configuration properties for blueprint export.
+   *
+   * @param hostGroups  cluster host groups
+   * @param updaters    registered updaters
+   */
+  private void doMultiHostExportUpdate(Collection<? extends HostGroup> hostGroups,
+                                       Map<String, Map<String, PropertyUpdater>> updaters) {
+
+    for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
+      String type = entry.getKey();
+      for (String propertyName : entry.getValue().keySet()) {
+        Map<String, String> typeProperties = properties.get(type);
+        if (typeProperties != null && typeProperties.containsKey(propertyName)) {
+          String propValue = typeProperties.get(propertyName);
+          for (HostGroup group : hostGroups) {
+            Collection<String> hosts = group.getHostInfo();
+            for (String host : hosts) {
+              propValue = propValue.replaceAll(host + "\\b", "%HOSTGROUP::" + group.getName() + "%");
+            }
+          }
+          Collection<String> addedGroups = new HashSet<String>();
+          String[] toks = propValue.split(",");
+          boolean inBrackets = propValue.startsWith("[");
+
+          StringBuilder sb = new StringBuilder();
+          if (inBrackets) {
+            sb.append('[');
+          }
+          boolean firstTok = true;
+          for (String tok : toks) {
+            tok = tok.replaceAll("[\\[\\]]", "");
+
+            if (addedGroups.add(tok)) {
+              if (! firstTok) {
+                sb.append(',');
+              }
+              sb.append(tok);
+            }
+            firstTok = false;
+          }
+
+          if (inBrackets) {
+            sb.append(']');
+          }
+          typeProperties.put(propertyName, sb.toString());
+        }
+      }
+    }
+  }
+
+  /**
+   * Get host groups which contain a component.
+   *
+   * @param component   component name
+   * @param hostGroups  collection of host groups to check
+   *
+   * @return collection of host groups which contain the specified component
+   */
+  private static Collection<HostGroup> getHostGroupsForComponent(String component,
+                                                                 Collection<? extends HostGroup> hostGroups) {
+
+    Collection<HostGroup> resultGroups = new HashSet<HostGroup>();
+    for (HostGroup group : hostGroups ) {
+      if (group.getComponents().contains(component)) {
+        resultGroups.add(group);
+      }
+    }
+    return resultGroups;
+  }
+
+  /**
+   * Convert a property value which includes a host group topology token to a physical host.
+   *
+   * @param hostGroups  cluster host groups
+   * @param val         value to be converted
+   *
+   * @return updated value with physical host name
+   */
+  private static Collection<String> getHostStrings(Map<String, ? extends HostGroup> hostGroups,
+                                                   String val) {
+
+    Collection<String> hosts = new HashSet<String>();
+    Matcher m = HOSTGROUP_PORT_REGEX.matcher(val);
+    while (m.find()) {
+      String groupName = m.group(1);
+      String port = m.group(2);
+
+
+      HostGroup hostGroup = hostGroups.get(groupName);
+      if (hostGroup == null) {
+        throw new IllegalArgumentException(
+            "Unable to match blueprint host group token to a host group: " + groupName);
+      }
+      for (String host : hostGroup.getHostInfo()) {
+        if (port != null) {
+          host += ":" + port;
+        }
+        hosts.add(host);
+      }
+    }
+    return hosts;
+  }
+
+  /**
+   * Provides functionality to update a property value.
+   */
+  public interface PropertyUpdater {
+    /**
+     * Update a property value.
+     *
+     *
+     * @param hostGroups  host groups
+     * @param origValue   original value of property
+     * @param properties  all properties
+     *
+     * @return new property value
+     */
+    public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
+                                         String origValue, Map<String, Map<String, String>> properties);
+  }
+
+  /**
+   * Topology based updater which replaces the original host name of a property with the host name
+   * which runs the associated (master) component in the new cluster.
+   */
+  private static class SingleHostTopologyUpdater implements PropertyUpdater {
+    /**
+     * Component name
+     */
+    private String component;
+
+    /**
+     * Constructor.
+     *
+     * @param component  component name associated with the property
+     */
+    public SingleHostTopologyUpdater(String component) {
+      this.component = component;
+    }
+
+    /**
+     * Update the property with the new host name which runs the associated component.
+     *
+     *
+     * @param hostGroups  host groups
+     * @param origValue   original value of property
+     * @param properties  all properties
+     *
+     * @return updated property value with old host name replaced by new host name
+     */
+    public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
+                                         String origValue,
+                                         Map<String, Map<String, String>> properties)  {
+
+      Matcher m = HOSTGROUP_REGEX.matcher(origValue);
+      if (m.find()) {
+        String hostGroupName = m.group(1);
+        HostGroup hostGroup = hostGroups.get(hostGroupName);
+        //todo: ensure > 0 hosts (is this necessary)
+        return origValue.replace(m.group(0), hostGroup.getHostInfo().iterator().next());
+      } else {
+        Collection<HostGroup> matchingGroups = getHostGroupsForComponent(component, hostGroups.values());
+        if (matchingGroups.size() == 1) {
+          return origValue.replace("localhost", matchingGroups.iterator().next().getHostInfo().iterator().next());
+        } else {
+          throw new IllegalArgumentException("Unable to update configuration property with topology information. " +
+              "Component '" + this.component + "' is not mapped to any host group or is mapped to multiple groups.");
+        }
+      }
+    }
+  }
+
+  /**
+   * Topology based updater which replaces the original host name of a database property with the host name
+   * where the DB is deployed in the new cluster.  If an existing database is specified, the original property
+   * value is returned.
+   */
+  private static class DBTopologyUpdater extends SingleHostTopologyUpdater {
+    /**
+     * Property type (global, core-site ...) for property which is used to determine if DB is external.
+     */
+    private final String configPropertyType;
+
+    /**
+     * Name of property which is used to determine if DB is new or existing (exernal).
+     */
+    private final String conditionalPropertyName;
+
+    /**
+     * Constructor.
+     *
+     * @param component                component to get hot name if new DB
+     * @param conditionalPropertyType  config type of property used to determine if DB is external
+     * @param conditionalPropertyName  name of property which is used to determine if DB is external
+     */
+    private DBTopologyUpdater(String component, String conditionalPropertyType,
+                              String conditionalPropertyName) {
+      super(component);
+      this.configPropertyType = conditionalPropertyType;
+      this.conditionalPropertyName = conditionalPropertyName;
+    }
+
+    /**
+     * If database is a new managed database, update the property with the new host name which
+     * runs the associated component.  If the database is external (non-managed), return the
+     * original value.
+     *
+     *
+     * @param hostGroups  host groups
+     * @param origValue   original value of property
+     * @param properties  all properties
+     *
+     * @return updated property value with old host name replaced by new host name or original value
+     *         if the database is external
+     */
+    @Override
+    public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
+                                         String origValue, Map<String, Map<String, String>> properties) {
+
+      if (isDatabaseManaged(properties)) {
+        return super.updateForClusterCreate(hostGroups, origValue, properties);
+      } else {
+        return origValue;
+      }
+    }
+
+    /**
+     * Determine if database is managed, meaning that it is a component in the cluster topology.
+     *
+     * @return true if the DB is managed; false otherwise
+     */
+    private boolean isDatabaseManaged(Map<String, Map<String, String>> properties) {
+      // conditional property should always exist since it is required to be specified in the stack
+      return properties.get(configPropertyType).
+          get(conditionalPropertyName).startsWith("New");
+    }
+  }
+
+  /**
+   * Topology based updater which replaces original host names (possibly more than one) contained in a property
+   * value with the host names which runs the associated component in the new cluster.
+   */
+  private static class MultipleHostTopologyUpdater implements PropertyUpdater {
+    /**
+     * Component name
+     */
+    private String component;
+
+    /**
+     * Separator for multiple property values
+     */
+    private Character separator = ',';
+
+    /**
+     * Constructor.
+     *
+     * @param component  component name associated with the property
+     */
+    public MultipleHostTopologyUpdater(String component) {
+      this.component = component;
+    }
+
+    /**
+     * Update all host names included in the original property value with new host names which run the associated
+     * component.
+     *
+     *
+     * @param hostGroups  host groups
+     * @param origValue   original value of property
+     * @param properties  all properties
+     *
+     * @return updated property value with old host names replaced by new host names
+     */
+    public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
+                                         String origValue,
+                                         Map<String, Map<String, String>> properties) {
+
+      Collection<String> hostStrings = getHostStrings(hostGroups, origValue);
+      if (hostStrings.isEmpty()) {
+        //default non-exported original value
+        String port = null;
+        if (origValue.contains(":")) {
+          //todo: currently assuming all hosts are using same port
+          port = origValue.substring(origValue.indexOf(":") + 1);
+        }
+        Collection<HostGroup> matchingGroups = getHostGroupsForComponent(component, hostGroups.values());
+        for (HostGroup group : matchingGroups) {
+          for (String host : group.getHostInfo()) {
+            if (port != null) {
+              host += ":" + port;
+            }
+            hostStrings.add(host);
+          }
+        }
+      }
+
+      StringBuilder sb = new StringBuilder();
+      boolean firstHost = true;
+      for (String host : hostStrings) {
+        if (!firstHost) {
+          sb.append(separator);
+        } else {
+          firstHost = false;
+        }
+        sb.append(host);
+      }
+
+      return sb.toString();
+    }
+  }
+
+  /**
+   * Updater which appends "m" to the original property value.
+   * For example, "1024" would be updated to "1024m".
+   */
+  private static class MPropertyUpdater implements PropertyUpdater {
+    /**
+     * Append 'm' to the original property value if it doesn't already exist.
+     *
+     *
+     * @param hostGroups  host groups
+     * @param origValue   original value of property
+     * @param properties  all properties
+     *
+     * @return property with 'm' appended
+     */
+    public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
+                                         String origValue, Map<String,
+        Map<String, String>> properties) {
+
+      return origValue.endsWith("m") ? origValue : origValue + 'm';
+    }
+  }
+
+  /**
+   * Class to facilitate special formatting needs of property values.
+   */
+  private abstract static class AbstractPropertyValueDecorator implements PropertyUpdater {
+    PropertyUpdater propertyUpdater;
+
+    /**
+     * Constructor.
+     *
+     * @param propertyUpdater  wrapped updater
+     */
+    public AbstractPropertyValueDecorator(PropertyUpdater propertyUpdater) {
+      this.propertyUpdater = propertyUpdater;
+    }
+
+    /**
+     * Return decorated form of the updated input property value.
+     *
+     * @param hostGroupMap  map of host group name to HostGroup
+     * @param origValue     original value of property
+     * @param properties    all properties
+     *
+     * @return Formatted output string
+     */
+    @Override
+    public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroupMap,
+                                         String origValue,
+                                         Map<String, Map<String, String>> properties) {
+
+      return doFormat(propertyUpdater.updateForClusterCreate(hostGroupMap, origValue, properties));
+    }
+
+    /**
+     * Transform input string to required output format.
+     *
+     * @param originalValue  original value of property
+     *
+     * @return formatted output string
+     */
+    public abstract String doFormat(String originalValue);
+  }
+
+  /**
+   * Return properties of the form ['value']
+   */
+  private static class YamlMultiValuePropertyDecorator extends AbstractPropertyValueDecorator {
+
+    public YamlMultiValuePropertyDecorator(PropertyUpdater propertyUpdater) {
+      super(propertyUpdater);
+    }
+
+    /**
+     * Format input String of the form, str1,str2 to ['str1','str2']
+     *
+     * @param origValue  input string
+     *
+     * @return formatted string
+     */
+    @Override
+    public String doFormat(String origValue) {
+      StringBuilder sb = new StringBuilder();
+      if (origValue != null) {
+        sb.append("[");
+        boolean isFirst = true;
+        for (String value : origValue.split(",")) {
+          if (!isFirst) {
+            sb.append(",");
+          } else {
+            isFirst = false;
+          }
+          sb.append("'");
+          sb.append(value);
+          sb.append("'");
+        }
+        sb.append("]");
+      }
+      return sb.toString();
+    }
+  }
+
+  /**
+   * Register updaters for configuration properties.
+   */
+  static {
+
+    allUpdaters.add(singleHostTopologyUpdaters);
+    allUpdaters.add(multiHostTopologyUpdaters);
+    allUpdaters.add(dbHostTopologyUpdaters);
+    allUpdaters.add(mPropertyUpdaters);
+
+    Map<String, PropertyUpdater> hdfsSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> mapredSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> coreSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> hbaseSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> yarnSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> hiveSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> oozieSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> stormSiteMap = new HashMap<String, PropertyUpdater>();
+
+    Map<String, PropertyUpdater> mapredEnvMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> hadoopEnvMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> hbaseEnvMap = new HashMap<String, PropertyUpdater>();
+
+    Map<String, PropertyUpdater> multiWebhcatSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> multiHbaseSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> multiStormSiteMap = new HashMap<String, PropertyUpdater>();
+
+    Map<String, PropertyUpdater> dbHiveSiteMap = new HashMap<String, PropertyUpdater>();
+
+
+    singleHostTopologyUpdaters.put("hdfs-site", hdfsSiteMap);
+    singleHostTopologyUpdaters.put("mapred-site", mapredSiteMap);
+    singleHostTopologyUpdaters.put("core-site", coreSiteMap);
+    singleHostTopologyUpdaters.put("hbase-site", hbaseSiteMap);
+    singleHostTopologyUpdaters.put("yarn-site", yarnSiteMap);
+    singleHostTopologyUpdaters.put("hive-site", hiveSiteMap);
+    singleHostTopologyUpdaters.put("oozie-site", oozieSiteMap);
+    singleHostTopologyUpdaters.put("storm-site", stormSiteMap);
+
+    mPropertyUpdaters.put("hadoop-env", hadoopEnvMap);
+    mPropertyUpdaters.put("hbase-env", hbaseEnvMap);
+    mPropertyUpdaters.put("mapred-env", mapredEnvMap);
+
+    multiHostTopologyUpdaters.put("webhcat-site", multiWebhcatSiteMap);
+    multiHostTopologyUpdaters.put("hbase-site", multiHbaseSiteMap);
+    multiHostTopologyUpdaters.put("storm-site", multiStormSiteMap);
+
+    dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap);
+
+    // NAMENODE
+    hdfsSiteMap.put("dfs.http.address", new SingleHostTopologyUpdater("NAMENODE"));
+    hdfsSiteMap.put("dfs.https.address", new SingleHostTopologyUpdater("NAMENODE"));
+    coreSiteMap.put("fs.default.name", new SingleHostTopologyUpdater("NAMENODE"));
+    hdfsSiteMap.put("dfs.namenode.http-address", new SingleHostTopologyUpdater("NAMENODE"));
+    hdfsSiteMap.put("dfs.namenode.https-address", new SingleHostTopologyUpdater("NAMENODE"));
+    coreSiteMap.put("fs.defaultFS", new SingleHostTopologyUpdater("NAMENODE"));
+    hbaseSiteMap.put("hbase.rootdir", new SingleHostTopologyUpdater("NAMENODE"));
+
+    // SECONDARY_NAMENODE
+    hdfsSiteMap.put("dfs.secondary.http.address", new SingleHostTopologyUpdater("SECONDARY_NAMENODE"));
+    hdfsSiteMap.put("dfs.namenode.secondary.http-address", new SingleHostTopologyUpdater("SECONDARY_NAMENODE"));
+
+    // JOBTRACKER
+    mapredSiteMap.put("mapred.job.tracker", new SingleHostTopologyUpdater("JOBTRACKER"));
+    mapredSiteMap.put("mapred.job.tracker.http.address", new SingleHostTopologyUpdater("JOBTRACKER"));
+    mapredSiteMap.put("mapreduce.history.server.http.address", new SingleHostTopologyUpdater("JOBTRACKER"));
+
+
+    // HISTORY_SERVER
+    yarnSiteMap.put("yarn.log.server.url", new SingleHostTopologyUpdater("HISTORYSERVER"));
+    mapredSiteMap.put("mapreduce.jobhistory.webapp.address", new SingleHostTopologyUpdater("HISTORYSERVER"));
+    mapredSiteMap.put("mapreduce.jobhistory.address", new SingleHostTopologyUpdater("HISTORYSERVER"));
+
+    // RESOURCEMANAGER
+    yarnSiteMap.put("yarn.resourcemanager.hostname", new SingleHostTopologyUpdater("RESOURCEMANAGER"));
+    yarnSiteMap.put("yarn.resourcemanager.resource-tracker.address", new SingleHostTopologyUpdater("RESOURCEMANAGER"));
+    yarnSiteMap.put("yarn.resourcemanager.webapp.address", new SingleHostTopologyUpdater("RESOURCEMANAGER"));
+    yarnSiteMap.put("yarn.resourcemanager.scheduler.address", new SingleHostTopologyUpdater("RESOURCEMANAGER"));
+    yarnSiteMap.put("yarn.resourcemanager.address", new SingleHostTopologyUpdater("RESOURCEMANAGER"));
+    yarnSiteMap.put("yarn.resourcemanager.admin.address", new SingleHostTopologyUpdater("RESOURCEMANAGER"));
+
+    // HIVE_SERVER
+    hiveSiteMap.put("hive.metastore.uris", new SingleHostTopologyUpdater("HIVE_SERVER"));
+    dbHiveSiteMap.put("javax.jdo.option.ConnectionURL",
+        new DBTopologyUpdater("MYSQL_SERVER", "hive-env", "hive_database"));
+
+    // OOZIE_SERVER
+    oozieSiteMap.put("oozie.base.url", new SingleHostTopologyUpdater("OOZIE_SERVER"));
+
+    // ZOOKEEPER_SERVER
+    multiHbaseSiteMap.put("hbase.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
+    multiWebhcatSiteMap.put("templeton.zookeeper.hosts", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
+
+    // STORM
+    stormSiteMap.put("nimbus.host", new SingleHostTopologyUpdater("NIMBUS"));
+    stormSiteMap.put("worker.childopts", new SingleHostTopologyUpdater("GANGLIA_SERVER"));
+    stormSiteMap.put("supervisor.childopts", new SingleHostTopologyUpdater("GANGLIA_SERVER"));
+    stormSiteMap.put("nimbus.childopts", new SingleHostTopologyUpdater("GANGLIA_SERVER"));
+    multiStormSiteMap.put("storm.zookeeper.servers",
+        new YamlMultiValuePropertyDecorator(new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")));
+
+    // Required due to AMBARI-4933.  These no longer seem to be required as the default values in the stack
+    // are now correct but are left here in case an existing blueprint still contains an old value.
+    hadoopEnvMap.put("namenode_heapsize", new MPropertyUpdater());
+    hadoopEnvMap.put("namenode_opt_newsize", new MPropertyUpdater());
+    hadoopEnvMap.put("namenode_opt_maxnewsize", new MPropertyUpdater());
+    hadoopEnvMap.put("dtnode_heapsize", new MPropertyUpdater());
+    mapredEnvMap.put("jtnode_opt_newsize", new MPropertyUpdater());
+    mapredEnvMap.put("jtnode_opt_maxnewsize", new MPropertyUpdater());
+    mapredEnvMap.put("jtnode_heapsize", new MPropertyUpdater());
+    hbaseEnvMap.put("hbase_master_heapsize", new MPropertyUpdater());
+    hbaseEnvMap.put("hbase_regionserver_heapsize", new MPropertyUpdater());
+  }
+}
\ No newline at end of file


Mime
View raw message