ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rnettle...@apache.org
Subject ambari git commit: AMBARI-10388. Blueprint not properly handling host group names that include "-" characters. (rnettleton)
Date Tue, 07 Apr 2015 22:50:11 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk a1cd22f99 -> 90da27509


AMBARI-10388. Blueprint not properly handling host group names that include "-" characters.
(rnettleton)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/90da2750
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/90da2750
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/90da2750

Branch: refs/heads/trunk
Commit: 90da27509c93825eee1db57a5e962ab31f0cc550
Parents: a1cd22f
Author: Bob Nettleton <rnettleton@hortonworks.com>
Authored: Tue Apr 7 18:49:29 2015 -0400
Committer: Bob Nettleton <rnettleton@hortonworks.com>
Committed: Tue Apr 7 18:49:29 2015 -0400

----------------------------------------------------------------------
 .../BlueprintConfigurationProcessor.java        |   3 +-
 .../BlueprintConfigurationProcessorTest.java    | 162 +++++++++++++++++++
 2 files changed, 164 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/90da2750/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 04027af..12768f7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -85,7 +85,7 @@ public class BlueprintConfigurationProcessor {
   /**
    * Compiled regex for hostgroup token with port information.
    */
-  private static Pattern HOSTGROUP_PORT_REGEX = Pattern.compile("%HOSTGROUP::(\\w+|\\d+)%:?(\\d+)?");
+  private static Pattern HOSTGROUP_PORT_REGEX = Pattern.compile("%HOSTGROUP::(\\S+?)%:?(\\d+)?");
 
   /**
    * Statically-defined set of properties that can support using a nameservice name
@@ -95,6 +95,7 @@ public class BlueprintConfigurationProcessor {
     new HashSet<String>(Arrays.asList("fs.defaultFS", "hbase.rootdir"));
 
 
+
   /**
    * Configuration properties to be updated
    */

http://git-wip-us.apache.org/repos/asf/ambari/blob/90da2750/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index d8794ac..b886956 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -788,6 +788,62 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
+  public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_UsingMinusSymbolInHostGroupName()
{
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> typeProps = new HashMap<String, String>();
+    typeProps.put("yarn.resourcemanager.hostname", "%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-6%");
+    properties.put("yarn-site", typeProps);
+
+    Collection<String> hgComponents = new HashSet<String>();
+    hgComponents.add("NAMENODE");
+    hgComponents.add("SECONDARY_NAMENODE");
+    hgComponents.add("RESOURCEMANAGER");
+    HostGroup group1 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-6", Collections.singleton("testhost"),
hgComponents);
+
+    Collection<String> hgComponents2 = new HashSet<String>();
+    hgComponents2.add("DATANODE");
+    hgComponents2.add("HDFS_CLIENT");
+    HostGroup group2 = new TestHostGroup("group2", Collections.singleton("testhost2"), hgComponents2);
+
+    Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>();
+    hostGroups.put(group1.getName(), group1);
+    hostGroups.put(group2.getName(), group2);
+
+    BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties);
+    Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups,
null);
+    String updatedVal = updatedProperties.get("yarn-site").get("yarn.resourcemanager.hostname");
+    assertEquals("testhost", updatedVal);
+  }
+
+  @Test
+  public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_WithPort_UsingMinusSymbolInHostGroupName()
{
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> typeProps = new HashMap<String, String>();
+    typeProps.put("yarn.resourcemanager.hostname", "%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-6%:2180");
+    properties.put("yarn-site", typeProps);
+
+    Collection<String> hgComponents = new HashSet<String>();
+    hgComponents.add("NAMENODE");
+    hgComponents.add("SECONDARY_NAMENODE");
+    hgComponents.add("RESOURCEMANAGER");
+    HostGroup group1 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-6", Collections.singleton("testhost"),
hgComponents);
+
+    Collection<String> hgComponents2 = new HashSet<String>();
+    hgComponents2.add("DATANODE");
+    hgComponents2.add("HDFS_CLIENT");
+    HostGroup group2 = new TestHostGroup("group2", Collections.singleton("testhost2"), hgComponents2);
+
+    Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>();
+    hostGroups.put(group1.getName(), group1);
+    hostGroups.put(group2.getName(), group2);
+
+    BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties);
+    Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups,
null);
+    String updatedVal = updatedProperties.get("yarn-site").get("yarn.resourcemanager.hostname");
+    assertEquals("testhost:2180", updatedVal);
+  }
+
+  @Test
   public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue__WithPort()
{
     Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
     Map<String, String> typeProps = new HashMap<String, String>();
@@ -924,6 +980,62 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
+  public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___withPorts_UsingMinusSymbolInHostGroupName()
{
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> typeProps = new HashMap<String, String>();
+    typeProps.put("ha.zookeeper.quorum", "%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-6%:2181,%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-5%:2181,%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-7%:2181");
+    properties.put("core-site", typeProps);
+
+    Collection<String> hgComponents = new HashSet<String>();
+    hgComponents.add("NAMENODE");
+    hgComponents.add("SECONDARY_NAMENODE");
+    hgComponents.add("ZOOKEEPER_SERVER");
+    HostGroup group1 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-6", Collections.singleton("testhost"),
hgComponents);
+
+    Collection<String> hgComponents2 = new HashSet<String>();
+    hgComponents2.add("DATANODE");
+    hgComponents2.add("HDFS_CLIENT");
+    hgComponents2.add("ZOOKEEPER_SERVER");
+    Set<String> hosts2 = new HashSet<String>();
+    hosts2.add("testhost2");
+    hosts2.add("testhost2a");
+    hosts2.add("testhost2b");
+    HostGroup group2 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-5", hosts2,
hgComponents2);
+
+    Collection<String> hgComponents3 = new HashSet<String>();
+    hgComponents2.add("HDFS_CLIENT");
+    hgComponents2.add("ZOOKEEPER_CLIENT");
+    Set<String> hosts3 = new HashSet<String>();
+    hosts3.add("testhost3");
+    hosts3.add("testhost3a");
+    HostGroup group3 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-7", hosts3,
hgComponents3);
+
+    Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>();
+    hostGroups.put(group1.getName(), group1);
+    hostGroups.put(group2.getName(), group2);
+    hostGroups.put(group3.getName(), group3);
+
+    BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties);
+    Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups,
null);
+    String updatedVal = updatedProperties.get("core-site").get("ha.zookeeper.quorum");
+    String[] hosts = updatedVal.split(",");
+
+    Collection<String> expectedHosts = new HashSet<String>();
+    expectedHosts.add("testhost:2181");
+    expectedHosts.add("testhost2:2181");
+    expectedHosts.add("testhost2a:2181");
+    expectedHosts.add("testhost2b:2181");
+    expectedHosts.add("testhost3:2181");
+    expectedHosts.add("testhost3a:2181");
+
+    assertEquals(6, hosts.length);
+    for (String host : hosts) {
+      assertTrue("Expected host :" + host + "was not included in the multi-server list in
this property.", expectedHosts.contains(host));
+      expectedHosts.remove(host);
+    }
+  }
+
+  @Test
   public void testDoUpdateForClusterCreate_MultiHostProperty_exportedValues_withPorts_singleHostValue()
{
     Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
     Map<String, String> yarnSiteConfig = new HashMap<String, String>();
@@ -2307,6 +2419,56 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
+  public void testHDFSConfigClusterUpdateQuorumJournalURL_UsingMinusSymbolInHostName() throws
Exception {
+    final String expectedHostNameOne = "c6401.apache.ambari.org";
+    final String expectedHostNameTwo = "c6402.apache.ambari.org";
+    final String expectedPortNum = "808080";
+    final String expectedHostGroupName = "host-group-1";
+    final String expectedHostGroupNameTwo = "host-group-2";
+
+    EasyMockSupport mockSupport = new EasyMockSupport();
+
+    HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+    HostGroup mockHostGroupTwo = mockSupport.createMock(HostGroup.class);
+
+    expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostNameOne)).atLeastOnce();
+    expect(mockHostGroupTwo.getHostInfo()).andReturn(Arrays.asList(expectedHostNameTwo)).atLeastOnce();
+
+    mockSupport.replayAll();
+
+    Map<String, Map<String, String>> configProperties =
+      new HashMap<String, Map<String, String>>();
+
+    Map<String, String> hdfsSiteProperties =
+      new HashMap<String, String>();
+
+    configProperties.put("hdfs-site", hdfsSiteProperties);
+
+    // setup properties that include host information
+    // setup shared edit property, that includes a qjournal URL scheme
+    hdfsSiteProperties.put("dfs.namenode.shared.edits.dir", "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum, expectedHostGroupNameTwo)
+ "/mycluster");
+
+    BlueprintConfigurationProcessor configProcessor =
+      new BlueprintConfigurationProcessor(configProperties);
+
+    Map<String, HostGroup> mapOfHostGroups =
+      new HashMap<String, HostGroup>();
+    mapOfHostGroups.put(expectedHostGroupName, mockHostGroupOne);
+    mapOfHostGroups.put(expectedHostGroupNameTwo, mockHostGroupTwo);
+
+    // call top-level export method
+    configProcessor.doUpdateForClusterCreate(mapOfHostGroups, null);
+
+    // expect that all servers are included in the updated config, and that the qjournal
URL format is preserved
+    assertEquals("HDFS HA shared edits directory property not properly updated for cluster
create.",
+      "qjournal://" + createHostAddress(expectedHostNameOne, expectedPortNum) + ";" + createHostAddress(expectedHostNameTwo,
expectedPortNum) + "/mycluster",
+      hdfsSiteProperties.get("dfs.namenode.shared.edits.dir"));
+
+    mockSupport.verifyAll();
+
+  }
+
+  @Test
   public void testHiveConfigExported() throws Exception {
     final String expectedHostName = "c6401.apache.ambari.org";
     final String expectedHostNameTwo = "c6402.ambari.apache.org";


Mime
View raw message