ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rnettle...@apache.org
Subject ambari git commit: AMBARI-14786. Localhost configuration value for multiple host properties does not work properly. (Daniel Gergely via rnettleton)
Date Thu, 04 Feb 2016 19:07:13 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 8ff0b5ed8 -> 7616e0f5d


AMBARI-14786. Localhost configuration value for multiple host properties does not work properly.
(Daniel Gergely via rnettleton)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7616e0f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7616e0f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7616e0f5

Branch: refs/heads/trunk
Commit: 7616e0f5d6939509d1914857dc03150eacc9e3b9
Parents: 8ff0b5e
Author: Bob Nettleton <rnettleton@hortonworks.com>
Authored: Thu Feb 4 14:06:34 2016 -0500
Committer: Bob Nettleton <rnettleton@hortonworks.com>
Committed: Thu Feb 4 14:07:02 2016 -0500

----------------------------------------------------------------------
 .../BlueprintConfigurationProcessor.java        | 206 ++++++++++------
 .../BlueprintConfigurationProcessorTest.java    | 243 ++++++++++++++++++-
 2 files changed, 370 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/7616e0f5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 9f42901..de31a0d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -126,6 +126,11 @@ public class BlueprintConfigurationProcessor {
   private static Pattern HOSTGROUP_PORT_REGEX = Pattern.compile("%HOSTGROUP::(\\S+?)%:?(\\d+)?");
 
   /**
+   * Compiled regex for hostgroup token with port information.
+   */
+  private static Pattern LOCALHOST_PORT_REGEX = Pattern.compile("localhost:?(\\d+)?");
+
+  /**
    * Statically-defined set of properties that can support HA using a nameservice name
    *   in the configuration, rather than just a host name.
    *   This set also contains other HA properties that will be exported if the
@@ -1583,7 +1588,9 @@ public class BlueprintConfigurationProcessor {
      */
     private final boolean usePrefixForEachHost;
 
-    private final Set<String> setOfKnownURLSchemes = Collections.singleton("thrift://");
+    private final boolean useSuffixForEachHost;
+
+    private final boolean usePortForEachHost;
 
     /**
      * Constructor.
@@ -1591,7 +1598,7 @@ public class BlueprintConfigurationProcessor {
      * @param component  component name associated with the property
      */
     public MultipleHostTopologyUpdater(String component) {
-      this(component, DEFAULT_SEPARATOR, false);
+      this(component, DEFAULT_SEPARATOR, false, false, true);
     }
 
     /**
@@ -1601,10 +1608,12 @@ public class BlueprintConfigurationProcessor {
      * @param separator the separator character to use when multiple hosts
      *                  are specified in a property or URL
      */
-    public MultipleHostTopologyUpdater(String component, Character separator, boolean userPrefixForEachHost)
{
+    public MultipleHostTopologyUpdater(String component, Character separator, boolean usePrefixForEachHost,
boolean useSuffixForEachHost, boolean usePortForEachHost) {
       this.component = component;
       this.separator = separator;
-      this.usePrefixForEachHost = userPrefixForEachHost;
+      this.usePrefixForEachHost = usePrefixForEachHost;
+      this.useSuffixForEachHost = useSuffixForEachHost;
+      this.usePortForEachHost = usePortForEachHost;
     }
 
     /**
@@ -1625,36 +1634,101 @@ public class BlueprintConfigurationProcessor {
 
       StringBuilder sb = new StringBuilder();
 
-      if (!origValue.contains("%HOSTGROUP") &&
-          (!origValue.contains("localhost"))) {
+      if (!origValue.contains("%HOSTGROUP") && (!origValue.contains("localhost")))
{
         // this property must contain FQDNs specified directly by the user
         // of the Blueprint, so the processor should not attempt to update them
         return origValue;
       }
 
-      if (origValue.contains("localhost") && topology.getHostGroupsForComponent(component).size()
== 1) {
-        return origValue.replace("localhost", topology.getHostAssignmentsForComponent(component).iterator().next());
+      Collection<String> hostStrings = getHostStrings(origValue, topology);
+      hostStrings.addAll(getHostStringsFromLocalhost(origValue, topology));
+
+      return resolveHostGroupPlaceholder(origValue, hostStrings);
+    }
+
+    /**
+     * Gets the prefix for hosts
+     * @param value property value
+     * @return prefix
+     */
+    private String getPrefix(String value) {
+      Matcher localhostMatcher = LOCALHOST_PORT_REGEX.matcher(value);
+      Matcher hostGroupMatcher = HOSTGROUP_PORT_REGEX.matcher(value);
+      String prefixCandidate = null;
+
+      if(localhostMatcher.find()) {
+        prefixCandidate = value.substring(0,localhostMatcher.start());
+      } else if(hostGroupMatcher.find()) {
+        prefixCandidate = value.substring(0,hostGroupMatcher.start());
+      } else {
+        return prefixCandidate;
+      }
+
+      // remove YAML array notation
+      if(prefixCandidate.startsWith("[")) {
+        prefixCandidate = prefixCandidate.substring(1);
+      }
+      // remove YAML string notation
+      if(prefixCandidate.startsWith("'")) {
+        prefixCandidate = prefixCandidate.substring(1);
       }
 
-      String prefix = null;
-      Collection<String> hostStrings = getHostStrings(origValue, topology);
-      if (hostStrings.isEmpty()) {
-        //default non-exported original value
-        String port;
-        for (String urlScheme : setOfKnownURLSchemes) {
-          if (origValue.startsWith(urlScheme)) {
-            prefix = urlScheme;
-          }
-        }
+      return prefixCandidate;
+    }
 
-        if (prefix != null) {
-          String valueWithoutPrefix = origValue.substring(prefix.length());
-          port = calculatePort(valueWithoutPrefix);
-          sb.append(prefix);
-        } else {
-          port = calculatePort(origValue);
-        }
+    /**
+     * Gets the suffix for hosts
+     * @param value property value
+     * @return suffix
+     */
+    private String getSuffix(String value) {
+      Matcher localhostMatcher = LOCALHOST_PORT_REGEX.matcher(value);
+      Matcher hostGroupMatcher = HOSTGROUP_PORT_REGEX.matcher(value);
+
+
+      Matcher activeMatcher = null;
 
+      if(localhostMatcher.find()) {
+        activeMatcher = localhostMatcher;
+      } else if(hostGroupMatcher.find()) {
+        activeMatcher = hostGroupMatcher;
+      } else {
+        return null;
+      }
+
+      String suffixCandidate = null;
+      int indexOfEnd;
+      do {
+        indexOfEnd = activeMatcher.end();
+      } while (activeMatcher.find());
+      suffixCandidate = value.substring(indexOfEnd);
+
+      // remove YAML array notation
+      if(suffixCandidate.endsWith("]")) {
+        suffixCandidate = suffixCandidate.substring(0, suffixCandidate.length()-1);
+      }
+      // remove YAML string notation
+      if(suffixCandidate.endsWith("'")) {
+        suffixCandidate = suffixCandidate.substring(0, suffixCandidate.length()-1);
+      }
+
+      return suffixCandidate;
+    }
+
+    /**
+     * Resolves localhost value to "host:port" elements (port is optional)
+     * @param origValue property value
+     * @param topology cluster topology
+     * @return list of hosts that have the given components
+     */
+    private Collection<String> getHostStringsFromLocalhost(String origValue, ClusterTopology
topology) {
+      Set<String> hostStrings = new HashSet<String>();
+      if(origValue.contains("localhost")) {
+        Matcher localhostMatcher = LOCALHOST_PORT_REGEX.matcher(origValue);
+        String port = null;
+        if(localhostMatcher.find()) {
+          port = calculatePort(localhostMatcher.group());
+        }
         for (String host : topology.getHostAssignmentsForComponent(component)) {
           if (port != null) {
             host += ":" + port;
@@ -1662,62 +1736,52 @@ public class BlueprintConfigurationProcessor {
           hostStrings.add(host);
         }
       }
-
-      return sb.append(resolveHostGroupPlaceholder(origValue, prefix, hostStrings)).toString();
+      return hostStrings;
     }
 
     /**
      * Resolves the host group place holders in the passed in original value.
      * @param originalValue The original value containing the place holders to be resolved.
-     * @param prefix The prefix to be added to the returned value.
      * @param hostStrings The collection of host names that are mapped to the host groups
to be resolved
      * @return The new value with place holders resolved.
      */
-    protected String resolveHostGroupPlaceholder(String originalValue, String prefix, Collection<String>
hostStrings) {
-      String suffix = null;
-      StringBuilder sb = new StringBuilder();
+    protected String resolveHostGroupPlaceholder(String originalValue, Collection<String>
hostStrings) {
+      String prefix = getPrefix(originalValue);
+      String suffix = getSuffix(originalValue);
+      String port = removePorts(hostStrings);
 
-      // parse out prefix if one exists
-      Matcher matcher = HOSTGROUP_PORT_REGEX.matcher(originalValue);
-      if (matcher.find()) {
-        int indexOfStart = matcher.start();
-        // handle the case of a YAML config property
-        if ((indexOfStart > 0) && (!originalValue.substring(0, indexOfStart).equals("['"))
&& (!originalValue.substring(0, indexOfStart).equals("[")) ) {
-          // append prefix before adding host names
-          prefix = originalValue.substring(0, indexOfStart);
-          sb.append(prefix);
-        }
+      String sep = (useSuffixForEachHost ? suffix : "") + separator + (usePrefixForEachHost
? prefix : "");
+      String combinedHosts = (usePrefixForEachHost ? prefix : "") + StringUtils.join(hostStrings,
sep);
 
-        // parse out suffix if one exists
-        int indexOfEnd;
-        do {
-          indexOfEnd = matcher.end();
-        } while (matcher.find());
+      return (usePrefixForEachHost ? "" : prefix) + combinedHosts + (usePortForEachHost ||
port == null ? "" : ":" + port) + suffix;
+    }
 
-        if (indexOfEnd < (originalValue.length())) {
-          suffix = originalValue.substring(indexOfEnd);
-        }
-      }
+    /**
+     * Removes "port" part of the hosts and returns it
+     * @param hostStrings list of "host:port" strings (port is optional)
+     * @return the port
+     */
+    private String removePorts(Collection<String> hostStrings) {
+      String port = null;
+      if(!usePortForEachHost && !hostStrings.isEmpty()) {
+        Set<String> temp = new HashSet<String>();
 
-      // add hosts to property, using the specified separator
-      boolean firstHost = true;
-      for (String host : hostStrings) {
-        if (!firstHost) {
-          sb.append(separator);
-          // support config properties that use a list of full URIs
-          if (usePrefixForEachHost && (prefix != null)) {
-            sb.append(prefix);
+        // extract port
+        Iterator<String> i = hostStrings.iterator();
+        do {
+          port = calculatePort(i.next());
+        } while (i.hasNext() && port == null);
+
+        // update hosts
+        if(port != null) {
+          for(String host : hostStrings) {
+            temp.add(host.replace(":"+port,""));
           }
-        } else {
-          firstHost = false;
         }
-        sb.append(host);
+        hostStrings.clear();
+        hostStrings.addAll(temp);
       }
-
-      if ((suffix != null) && (!suffix.equals("']")) && (!suffix.equals("]"))
) {
-        sb.append(suffix);
-      }
-      return sb.toString();
+      return port;
     }
 
     private static String calculatePort(String origValue) {
@@ -1988,7 +2052,7 @@ public class BlueprintConfigurationProcessor {
     TempletonHivePropertyUpdater() {
       // the only known property that requires hostname substitution is hive.metastore.uris,
       // but this updater should be flexible enough for other properties in the future.
-      mapOfKeysToUpdaters.put("hive.metastore.uris", new MultipleHostTopologyUpdater("HIVE_METASTORE",
',', true));
+      mapOfKeysToUpdaters.put("hive.metastore.uris", new MultipleHostTopologyUpdater("HIVE_METASTORE",
',', true, false, true));
     }
 
     @Override
@@ -2180,8 +2244,8 @@ public class BlueprintConfigurationProcessor {
     hbaseSiteMap.put("hbase.rootdir", new SingleHostTopologyUpdater("NAMENODE"));
     accumuloSiteMap.put("instance.volumes", new SingleHostTopologyUpdater("NAMENODE"));
     // HDFS shared.edits JournalNode Quorum URL uses semi-colons as separators
-    multiHdfsSiteMap.put("dfs.namenode.shared.edits.dir", new MultipleHostTopologyUpdater("JOURNALNODE",
';', false));
-    multiHdfsSiteMap.put("dfs.encryption.key.provider.uri", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER",
';', false));
+    multiHdfsSiteMap.put("dfs.namenode.shared.edits.dir", new MultipleHostTopologyUpdater("JOURNALNODE",
';', false, false, true));
+    multiHdfsSiteMap.put("dfs.encryption.key.provider.uri", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER",
';', false, false, false));
 
     // SECONDARY_NAMENODE
     hdfsSiteMap.put("dfs.secondary.http.address", new SingleHostTopologyUpdater("SECONDARY_NAMENODE"));
@@ -2213,14 +2277,14 @@ public class BlueprintConfigurationProcessor {
     yarnSiteMap.put("yarn.timeline-service.webapp.https.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER"));
 
     // HIVE_SERVER
-    multiHiveSiteMap.put("hive.metastore.uris", new MultipleHostTopologyUpdater("HIVE_METASTORE",
',', true));
+    multiHiveSiteMap.put("hive.metastore.uris", new MultipleHostTopologyUpdater("HIVE_METASTORE",
',', true, true, true));
     dbHiveSiteMap.put("javax.jdo.option.ConnectionURL",
         new DBTopologyUpdater("MYSQL_SERVER", "hive-env", "hive_database"));
     multiCoreSiteMap.put("hadoop.proxyuser.hive.hosts", new MultipleHostTopologyUpdater("HIVE_SERVER"));
     multiCoreSiteMap.put("hadoop.proxyuser.HTTP.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
     multiCoreSiteMap.put("hadoop.proxyuser.hcat.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
     multiCoreSiteMap.put("hadoop.proxyuser.yarn.hosts", new MultipleHostTopologyUpdater("RESOURCEMANAGER"));
-    multiCoreSiteMap.put("hadoop.security.key.provider.path", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER",
';', false));
+    multiCoreSiteMap.put("hadoop.security.key.provider.path", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER",
';', false, false, true));
     multiWebhcatSiteMap.put("templeton.hive.properties", new TempletonHivePropertyUpdater());
     multiWebhcatSiteMap.put("templeton.kerberos.principal", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
     hiveEnvMap.put("hive_hostname", new SingleHostTopologyUpdater("HIVE_SERVER"));

http://git-wip-us.apache.org/repos/asf/ambari/blob/7616e0f5/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 1afd6a8..7a77a25 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -41,6 +41,7 @@ import org.apache.ambari.server.topology.HostGroupInfo;
 import org.apache.ambari.server.topology.InvalidTopologyException;
 import org.apache.ambari.server.utils.CollectionPresentationUtils;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -2354,6 +2355,116 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
+  public void testMultipleHostTopologyUpdater__localhost__singleHost() throws Exception {
+
+    final String typeName = "hbase-site";
+    final String propertyName = "hbase.zookeeper.quorum";
+    final String originalValue = "localhost";
+    final String component1 = "ZOOKEEPER_SERVER";
+    final String component2 = "ZOOKEEPER_CLIENT";
+
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> typeProps = new HashMap<String, String>();
+    typeProps.put(propertyName, originalValue);
+    properties.put(typeName, typeProps);
+
+    Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String,
Map<String, String>>>emptyMap());
+
+    Collection<String> hgComponents = new HashSet<String>();
+    hgComponents.add(component1);
+    Set<String> hosts1 = new HashSet<String>();
+    hosts1.add("testhost1a");
+    TestHostGroup group1 = new TestHostGroup("group1", hgComponents, hosts1);
+
+    Collection<String> hgComponents2 = new HashSet<String>();
+    hgComponents2.add(component2);
+    Set<String> hosts2 = new HashSet<String>();
+    hosts2.add("testhost2");
+    TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2);
+
+    Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+    hostGroups.add(group1);
+    hostGroups.add(group2);
+
+    ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+
+    BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1);
+    String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties,
topology);
+
+    assertEquals("testhost1a", newValue);
+  }
+
+  @Test
+  public void testMultipleHostTopologyUpdater__localhost__singleHostGroup() throws Exception
{
+
+    final String typeName = "hbase-site";
+    final String propertyName = "hbase.zookeeper.quorum";
+    final String originalValue = "localhost";
+    final String component1 = "ZOOKEEPER_SERVER";
+
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> typeProps = new HashMap<String, String>();
+    typeProps.put(propertyName, originalValue);
+    properties.put(typeName, typeProps);
+
+    Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String,
Map<String, String>>>emptyMap());
+
+    Collection<String> hgComponents = new HashSet<String>();
+    hgComponents.add(component1);
+    Set<String> hosts1 = new HashSet<String>();
+    hosts1.add("testhost1a");
+    hosts1.add("testhost1b");
+    hosts1.add("testhost1c");
+    TestHostGroup group1 = new TestHostGroup("group1", hgComponents, hosts1);
+
+    Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+    hostGroups.add(group1);
+
+    ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+
+    BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1);
+    String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties,
topology);
+
+    List<String> hostArray = Arrays.asList(newValue.split(","));
+    Assert.assertTrue(hostArray.containsAll(hosts1) && hosts1.containsAll(hostArray));
+  }
+
+  @Test
+  public void testMultipleHostTopologyUpdater__hostgroup__singleHostGroup() throws Exception
{
+
+    final String typeName = "hbase-site";
+    final String propertyName = "hbase.zookeeper.quorum";
+    final String originalValue = "%HOSTGROUP::group1%";
+    final String component1 = "ZOOKEEPER_SERVER";
+
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> typeProps = new HashMap<String, String>();
+    typeProps.put(propertyName, originalValue);
+    properties.put(typeName, typeProps);
+
+    Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String,
Map<String, String>>>emptyMap());
+
+    Collection<String> hgComponents = new HashSet<String>();
+    hgComponents.add(component1);
+    Set<String> hosts1 = new HashSet<String>();
+    hosts1.add("testhost1a");
+    hosts1.add("testhost1b");
+    hosts1.add("testhost1c");
+    TestHostGroup group1 = new TestHostGroup("group1", hgComponents, hosts1);
+
+    Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+    hostGroups.add(group1);
+
+    ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+
+    BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1);
+    String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties,
topology);
+
+    List<String> hostArray = Arrays.asList(newValue.split(","));
+    Assert.assertTrue(hostArray.containsAll(hosts1) && hosts1.containsAll(hostArray));
+  }
+
+  @Test
   public void testDoUpdateForClusterVerifyRetrySettingsDefault() throws Exception {
     Map<String, Map<String, String>> configProperties =
       new HashMap<String, Map<String, String>>();
@@ -5887,8 +5998,16 @@ public class BlueprintConfigurationProcessorTest {
     // When
     configProcessor.doUpdateForClusterCreate();
 
+    String updatedVal = clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri");
+    Assert.assertTrue(updatedVal.startsWith("kms://http@"));
+    Assert.assertTrue(updatedVal.endsWith(":9292/kms"));
+    String hostsString = updatedVal.substring(11,updatedVal.length()-9);
+
+    List<String> hostArray = Arrays.asList(hostsString.split(";"));
+    List<String> expected = Arrays.asList("host1","host2");
+
     // Then
-    assertEquals("kms://http@host1;host2:9292/kms", clusterConfig.getPropertyValue(configType,
"dfs.encryption.key.provider.uri"));
+    Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray));
   }
 
 
@@ -5982,6 +6101,114 @@ public class BlueprintConfigurationProcessorTest {
     assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType,
"dfs.encryption.key.provider.uri"));
   }
 
+  @Test
+  public void testHdfsWithRangerKmsServer__multiple_hosts__localhost() throws Exception {
+    // Given
+    final String configType = "hdfs-site";
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> configProperties = new HashMap<String, String>();
+
+    properties.put(configType, configProperties);
+    configProperties.put("dfs.encryption.key.provider.uri", "kms://http@localhost:9292/kms");
+
+
+    Map<String, Map<String, String>> parentProperties = new HashMap<String,
Map<String, String>>();
+    Configuration parentClusterConfig = new Configuration(parentProperties,
+      Collections.<String, Map<String, Map<String, String>>>emptyMap());
+    Configuration clusterConfig = new Configuration(properties,
+      Collections.<String, Map<String, Map<String, String>>>emptyMap(),
parentClusterConfig);
+
+
+    Collection<String> kmsServerComponents = new HashSet<String>();
+    kmsServerComponents.add("RANGER_KMS_SERVER");
+
+    Collection<String> hdfsComponents = new HashSet<String>();
+    hdfsComponents.add("NAMENODE");
+    hdfsComponents.add("DATANODE");
+
+    Collection<String> hosts = new HashSet<String>();
+    hosts.add("host1");
+    hosts.add("host2");
+
+    TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, hosts);
+    group1.components.add("DATANODE");
+
+    TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host3"));
+
+    Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+    ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+    BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+    // When
+    configProcessor.doUpdateForClusterCreate();
+
+    String updatedVal = clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri");
+    Assert.assertTrue(updatedVal.startsWith("kms://http@"));
+    Assert.assertTrue(updatedVal.endsWith(":9292/kms"));
+    String hostsString = updatedVal.substring(11,updatedVal.length()-9);
+
+    List<String> hostArray = Arrays.asList(hostsString.split(";"));
+    List<String> expected = Arrays.asList("host1","host2");
+
+    // Then
+    Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray));
+  }
+
+  @Test
+  public void testHdfsWithRangerKmsServer__multiple_hosts__hostgroup() throws Exception {
+    // Given
+    final String configType = "hdfs-site";
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> configProperties = new HashMap<String, String>();
+
+    properties.put(configType, configProperties);
+    configProperties.put("dfs.encryption.key.provider.uri", "kms://http@%HOSTGROUP::group1%:9292/kms");
+
+
+    Map<String, Map<String, String>> parentProperties = new HashMap<String,
Map<String, String>>();
+    Configuration parentClusterConfig = new Configuration(parentProperties,
+      Collections.<String, Map<String, Map<String, String>>>emptyMap());
+    Configuration clusterConfig = new Configuration(properties,
+      Collections.<String, Map<String, Map<String, String>>>emptyMap(),
parentClusterConfig);
+
+
+    Collection<String> kmsServerComponents = new HashSet<String>();
+    kmsServerComponents.add("RANGER_KMS_SERVER");
+
+    Collection<String> hdfsComponents = new HashSet<String>();
+    hdfsComponents.add("NAMENODE");
+    hdfsComponents.add("DATANODE");
+
+    Collection<String> hosts = new HashSet<String>();
+    hosts.add("host1");
+    hosts.add("host2");
+
+    TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, hosts);
+    group1.components.add("DATANODE");
+
+    TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host3"));
+
+    Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+    ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+    BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+    // When
+    configProcessor.doUpdateForClusterCreate();
+
+    String updatedVal = clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri");
+    Assert.assertTrue(updatedVal.startsWith("kms://http@"));
+    Assert.assertTrue(updatedVal.endsWith(":9292/kms"));
+    String hostsString = updatedVal.substring(11,updatedVal.length()-9);
+
+    List<String> hostArray = Arrays.asList(hostsString.split(";"));
+    List<String> expected = Arrays.asList("host1","host2");
+
+    // Then
+    Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray));
+  }
+
 
   @Test
   public void testHadoopWithRangerKmsServer() throws Exception {
@@ -6248,8 +6475,8 @@ public class BlueprintConfigurationProcessorTest {
     String propertyOriginalValue2 = "[%HOSTGROUP::group_1%]";
 
     // When
-    String updatedValue1 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue1, null,
ImmutableList.<String>of("host1:100"));
-    String updatedValue2 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue2, null,
ImmutableList.<String>of("host1:100"));
+    String updatedValue1 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue1, ImmutableList.<String>of("host1:100"));
+    String updatedValue2 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue2, ImmutableList.<String>of("host1:100"));
 
     // Then
     assertEquals("host1:100", updatedValue1);
@@ -6269,8 +6496,8 @@ public class BlueprintConfigurationProcessorTest {
     String propertyOriginalValue2 = "[%HOSTGROUP::group_1%, %HOSTGROUP::group_2%]";
 
     // When
-    String updatedValue1 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue1, null,
ImmutableList.<String>of("host1:100", "host2:200"));
-    String updatedValue2 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue2, null,
ImmutableList.<String>of("host1:100", "host2:200"));
+    String updatedValue1 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue1, ImmutableList.<String>of("host1:100",
"host2:200"));
+    String updatedValue2 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue2, ImmutableList.<String>of("host1:100",
"host2:200"));
 
     // Then
     assertEquals("host1:100,host2:200", updatedValue1);
@@ -6288,7 +6515,7 @@ public class BlueprintConfigurationProcessorTest {
     String propertyOriginalValue = "http://%HOSTGROUP::group_1%#";
 
     // When
-    String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, null, ImmutableList.<String>of("host1:100"));
+    String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, ImmutableList.<String>of("host1:100"));
 
     // Then
     assertEquals("http://host1:100#", updatedValue);
@@ -6303,7 +6530,7 @@ public class BlueprintConfigurationProcessorTest {
     String propertyOriginalValue = "http://%HOSTGROUP::group_1,HOSTGROUP::group_2%/resource";
 
     // When
-    String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, null, ImmutableList.<String>of("host1:100",
"host2:200"));
+    String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, ImmutableList.<String>of("host1:100",
"host2:200"));
 
     // Then
     assertEquals("http://host1:100,host2:200/resource", updatedValue);
@@ -6318,7 +6545,7 @@ public class BlueprintConfigurationProcessorTest {
     String propertyOriginalValue = "%HOSTGROUP::group_1%:11,%HOSTGROUP::group_2%:11";
 
     // When
-    String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, null, ImmutableList.<String>of("host1:100",
"host2:200"));
+    String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, ImmutableList.<String>of("host1:100",
"host2:200"));
 
     // Then
     assertEquals("host1:100,host2:200", updatedValue);


Mime
View raw message