Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B25441159F for ; Tue, 23 Sep 2014 16:02:57 +0000 (UTC) Received: (qmail 48728 invoked by uid 500); 23 Sep 2014 16:02:57 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 48618 invoked by uid 500); 23 Sep 2014 16:02:57 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 48315 invoked by uid 99); 23 Sep 2014 16:02:57 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 23 Sep 2014 16:02:57 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 070AF93746F; Tue, 23 Sep 2014 16:02:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jonathanhurley@apache.org To: commits@ambari.apache.org Date: Tue, 23 Sep 2014 16:03:06 -0000 Message-Id: <62e394ed16824206863b2c50a7c149cd@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [11/12] git commit: Fixing blueprint export problems with undisguised hostnames Fixing blueprint export problems with undisguised hostnames This patch addresses AMBARI-7368. The Blueprint configuration processor was not properly masking the hostnames in some properties after a blueprint export from a running cluster. Most of these properties were related to HA or Kerberos, which are not supported by the Blueprints processor directly. This patch addresses this problem by registering PropertyUpdater instances for each of the properties that includes hostname information, but is not currently masked during a blueprint export. This patch implements support for tagging the HA Namenode configuration properties as candidates for hostname masking. The HA properties are dynamic with respect to naming, since the properties include the nameserivces and name nodes defined during HA setup in the Ambari UI. This patch implements a method to inspect the HA configuration, if present, and register the HA properties for each NameService and NameNode in order to remove the hostname information during the exporting process. This patch also implements support for the Namenode HA configuration properties to be properly handled with hostname resolution during cluster creation. This patch implements several new unit test cases to verify this change, and also updates some existing test cases to verify this change as well. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/efa0fa6a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/efa0fa6a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/efa0fa6a Branch: refs/heads/branch-alerts-dev Commit: efa0fa6aba000ef90aca3b04fec059cfa465f38f Parents: 4365e44 Author: Bob Nettleton Authored: Mon Sep 15 12:48:21 2014 -0400 Committer: John Speidel Committed: Tue Sep 23 11:29:09 2014 -0400 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 184 +++- .../BlueprintConfigurationProcessorTest.java | 906 ++++++++++++++++++- 2 files changed, 1069 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/efa0fa6a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index c31a869..b3cc098 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -22,6 +22,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -96,7 +98,7 @@ public class BlueprintConfigurationProcessor { * @return updated properties */ public Map> doUpdateForClusterCreate(Map hostGroups) { - for (Map> updaterMap : allUpdaters) { + for (Map> updaterMap : createCollectionOfUpdaters()) { for (Map.Entry> entry : updaterMap.entrySet()) { String type = entry.getKey(); for (Map.Entry updaterEntry : entry.getValue().entrySet()) { @@ -115,6 +117,43 @@ public class BlueprintConfigurationProcessor { } /** + * Creates a Collection of PropertyUpdater maps that will handle the configuration + * update for this cluster. If NameNode HA is enabled, then updater + * instances will be added to the collection, in addition to the default list + * of Updaters that are statically defined. + * + * @return Collection of PropertyUpdater maps used to handle cluster config update + */ + private Collection>> createCollectionOfUpdaters() { + return (isNameNodeHAEnabled()) ? addHAUpdaters(allUpdaters) : allUpdaters; + } + + /** + * Creates a Collection of PropertyUpdater maps that include the NameNode HA properties, and + * adds these to the list of updaters used to process the cluster configuration. The HA + * properties are based on the names of the HA namservices and name nodes, and so must + * be registered at runtime, rather than in the static list. This new Collection includes + * the statically-defined updaters, in addition to the HA-related updaters. + * + * @param updaters a Collection of updater maps to be included in the list of updaters for + * this cluster config update + * @return A Collection of PropertyUpdater maps to handle the cluster config update + */ + private Collection>> addHAUpdaters(Collection>> updaters) { + Collection>> highAvailabilityUpdaters = + new LinkedList>>(); + + // always add the statically-defined list of updaters to the list to use + // in processing cluster configuration + highAvailabilityUpdaters.addAll(updaters); + + // add the updaters for the dynamic HA properties, based on the HA config in hdfs-site + highAvailabilityUpdaters.add(createMapOfHAUpdaters()); + + return highAvailabilityUpdaters; + } + + /** * Update properties for blueprint export. * This involves converting concrete topology information to host groups. * @@ -125,12 +164,101 @@ public class BlueprintConfigurationProcessor { public Map> doUpdateForBlueprintExport(Collection hostGroups) { doSingleHostExportUpdate(hostGroups, singleHostTopologyUpdaters); doSingleHostExportUpdate(hostGroups, dbHostTopologyUpdaters); + + if (isNameNodeHAEnabled()) { + doNameNodeHAUpdate(hostGroups); + } + doMultiHostExportUpdate(hostGroups, multiHostTopologyUpdaters); return properties; } /** + * Perform export update processing for HA configuration for NameNodes. The HA NameNode property + * names are based on the nameservices defined when HA is enabled via the Ambari UI, so this method + * dynamically determines the property names, and registers PropertyUpdaters to handle the masking of + * host names in these configuration items. + * + * @param hostGroups cluster host groups + */ + public void doNameNodeHAUpdate(Collection hostGroups) { + Map> highAvailabilityUpdaters = createMapOfHAUpdaters(); + + // perform a single host update on these dynamically generated property names + if (highAvailabilityUpdaters.get("hdfs-site").size() > 0) { + doSingleHostExportUpdate(hostGroups, highAvailabilityUpdaters); + } + } + + /** + * Creates map of PropertyUpdater instances that are associated with + * NameNode High Availability (HA). The HA configuration property + * names are dynamic, and based on other HA config elements in + * hdfs-site. This method registers updaters for the required + * properties associated with each nameservice and namenode. + * + * @return a Map of registered PropertyUpdaters for handling HA properties in hdfs-site + */ + private Map> createMapOfHAUpdaters() { + Map> highAvailabilityUpdaters = new HashMap>(); + Map hdfsSiteUpdatersForAvailability = new HashMap(); + highAvailabilityUpdaters.put("hdfs-site", hdfsSiteUpdatersForAvailability); + + Map hdfsSiteConfig = properties.get("hdfs-site"); + // generate the property names based on the current HA config for the NameNode deployments + for (String nameService : parseNameServices(hdfsSiteConfig)) { + for (String nameNode : parseNameNodes(nameService, hdfsSiteConfig)) { + final String httpsPropertyName = "dfs.namenode.https-address." + nameService + "." + nameNode; + hdfsSiteUpdatersForAvailability.put(httpsPropertyName, new SingleHostTopologyUpdater("NAMENODE")); + final String httpPropertyName = "dfs.namenode.http-address." + nameService + "." + nameNode; + hdfsSiteUpdatersForAvailability.put(httpPropertyName, new SingleHostTopologyUpdater("NAMENODE")); + final String rpcPropertyName = "dfs.namenode.rpc-address." + nameService + "." + nameNode; + hdfsSiteUpdatersForAvailability.put(rpcPropertyName, new SingleHostTopologyUpdater("NAMENODE")); + } + } + return highAvailabilityUpdaters; + } + + /** + * Convenience function to determine if NameNode HA is enabled. + * + * @return true if NameNode HA is enabled + * false if NameNode HA is not enabled + */ + boolean isNameNodeHAEnabled() { + return properties.containsKey("hdfs-site") && properties.get("hdfs-site").containsKey("dfs.nameservices"); + } + + + /** + * Parses out the list of nameservices associated with this HDFS configuration. + * + * @param properties config properties for this cluster + * + * @return array of Strings that indicate the nameservices for this cluster + */ + static String[] parseNameServices(Map properties) { + final String nameServices = properties.get("dfs.nameservices"); + return splitAndTrimStrings(nameServices); + } + + /** + * Parses out the list of name nodes associated with a given HDFS + * NameService, based on a given HDFS configuration. + * + * @param nameService the nameservice used for this parsing + * @param properties config properties for this cluster + * + * @return array of Strings that indicate the name nodes associated + * with this nameservice + */ + static String[] parseNameNodes(String nameService, Map properties) { + final String nameNodes = properties.get("dfs.ha.namenodes." + nameService); + return splitAndTrimStrings(nameNodes); + } + + /** * Update single host topology configuration properties for blueprint export. * * @param hostGroups cluster export @@ -274,15 +402,22 @@ public class BlueprintConfigurationProcessor { return hosts; } - /** - * Provides package-level access to the map of single host topology updaters. - * This is useful for facilitating unit-testing of this class. + * Convenience method for splitting out the HA-related properties, while + * also removing leading/trailing whitespace. + * + * @param propertyName property name to parse * - * @return the map of single host topology updaters + * @return an array of Strings that represent the comma-separated + * elements in this property */ - static Map> getSingleHostTopologyUpdaters() { - return singleHostTopologyUpdaters; + private static String[] splitAndTrimStrings(String propertyName) { + List namesWithoutWhitespace = new LinkedList(); + for (String service : propertyName.split(",")) { + namesWithoutWhitespace.add(service.trim()); + } + + return namesWithoutWhitespace.toArray(new String[namesWithoutWhitespace.size()]); } /** @@ -630,13 +765,19 @@ public class BlueprintConfigurationProcessor { Map mapredEnvMap = new HashMap(); Map hadoopEnvMap = new HashMap(); Map hbaseEnvMap = new HashMap(); + Map hiveEnvMap = new HashMap(); + Map oozieEnvMap = new HashMap(); Map multiWebhcatSiteMap = new HashMap(); Map multiHbaseSiteMap = new HashMap(); Map multiStormSiteMap = new HashMap(); + Map multiCoreSiteMap = new HashMap(); + Map multiHdfsSiteMap = new HashMap(); Map dbHiveSiteMap = new HashMap(); + Map nagiosEnvMap = new HashMap(); + singleHostTopologyUpdaters.put("hdfs-site", hdfsSiteMap); singleHostTopologyUpdaters.put("mapred-site", mapredSiteMap); @@ -647,6 +788,9 @@ public class BlueprintConfigurationProcessor { singleHostTopologyUpdaters.put("oozie-site", oozieSiteMap); singleHostTopologyUpdaters.put("storm-site", stormSiteMap); singleHostTopologyUpdaters.put("falcon-startup.properties", falconStartupPropertiesMap); + singleHostTopologyUpdaters.put("nagios-env", nagiosEnvMap); + singleHostTopologyUpdaters.put("hive-env", hiveEnvMap); + singleHostTopologyUpdaters.put("oozie-env", oozieEnvMap); mPropertyUpdaters.put("hadoop-env", hadoopEnvMap); mPropertyUpdaters.put("hbase-env", hbaseEnvMap); @@ -655,6 +799,8 @@ public class BlueprintConfigurationProcessor { multiHostTopologyUpdaters.put("webhcat-site", multiWebhcatSiteMap); multiHostTopologyUpdaters.put("hbase-site", multiHbaseSiteMap); multiHostTopologyUpdaters.put("storm-site", multiStormSiteMap); + multiHostTopologyUpdaters.put("core-site", multiCoreSiteMap); + multiHostTopologyUpdaters.put("hdfs-site", multiHdfsSiteMap); dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap); @@ -666,6 +812,7 @@ public class BlueprintConfigurationProcessor { hdfsSiteMap.put("dfs.namenode.https-address", new SingleHostTopologyUpdater("NAMENODE")); coreSiteMap.put("fs.defaultFS", new SingleHostTopologyUpdater("NAMENODE")); hbaseSiteMap.put("hbase.rootdir", new SingleHostTopologyUpdater("NAMENODE")); + multiHdfsSiteMap.put("dfs.namenode.shared.edits.dir", new MultipleHostTopologyUpdater("JOURNALNODE")); // SECONDARY_NAMENODE hdfsSiteMap.put("dfs.secondary.http.address", new SingleHostTopologyUpdater("SECONDARY_NAMENODE")); @@ -690,17 +837,34 @@ public class BlueprintConfigurationProcessor { yarnSiteMap.put("yarn.resourcemanager.address", new SingleHostTopologyUpdater("RESOURCEMANAGER")); yarnSiteMap.put("yarn.resourcemanager.admin.address", new SingleHostTopologyUpdater("RESOURCEMANAGER")); + // APP_TIMELINE_SERVER + yarnSiteMap.put("yarn.timeline-service.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER")); + yarnSiteMap.put("yarn.timeline-service.webapp.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER")); + yarnSiteMap.put("yarn.timeline-service.webapp.https.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER")); + + // HIVE_SERVER hiveSiteMap.put("hive.metastore.uris", new SingleHostTopologyUpdater("HIVE_SERVER")); dbHiveSiteMap.put("javax.jdo.option.ConnectionURL", new DBTopologyUpdater("MYSQL_SERVER", "hive-env", "hive_database")); + multiCoreSiteMap.put("hadoop.proxyuser.hive.hosts", new MultipleHostTopologyUpdater("HIVE_SERVER")); + multiCoreSiteMap.put("hadoop.proxyuser.HTTP.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER")); + multiCoreSiteMap.put("hadoop.proxyuser.hcat.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER")); + multiWebhcatSiteMap.put("templeton.hive.properties", new MultipleHostTopologyUpdater("HIVE_SERVER")); + multiWebhcatSiteMap.put("templeton.kerberos.principal", new MultipleHostTopologyUpdater("WEBHCAT_SERVER")); + hiveEnvMap.put("hive_hostname", new SingleHostTopologyUpdater("HIVE_SERVER")); // OOZIE_SERVER oozieSiteMap.put("oozie.base.url", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieSiteMap.put("oozie.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieSiteMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieEnvMap.put("oozie_hostname", new SingleHostTopologyUpdater("OOZIE_SERVER")); + multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER")); // ZOOKEEPER_SERVER multiHbaseSiteMap.put("hbase.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); multiWebhcatSiteMap.put("templeton.zookeeper.hosts", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); + multiCoreSiteMap.put("ha.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); // STORM stormSiteMap.put("nimbus.host", new SingleHostTopologyUpdater("NIMBUS")); @@ -712,6 +876,12 @@ public class BlueprintConfigurationProcessor { // FALCON falconStartupPropertiesMap.put("*.broker.url", new SingleHostTopologyUpdater("FALCON_SERVER")); + falconStartupPropertiesMap.put("*.falcon.service.authentication.kerberos.principal", new SingleHostTopologyUpdater("FALCON_SERVER")); + falconStartupPropertiesMap.put("*.falcon.http.authentication.kerberos.principal", new SingleHostTopologyUpdater("FALCON_SERVER")); + + + // NAGIOS + nagiosEnvMap.put("nagios_principal_name", new SingleHostTopologyUpdater("NAGIOS_SERVER")); // Required due to AMBARI-4933. These no longer seem to be required as the default values in the stack http://git-wip-us.apache.org/repos/asf/ambari/blob/efa0fa6a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 3213b20..f906092 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -18,8 +18,10 @@ package org.apache.ambari.server.controller.internal; +import org.easymock.EasyMockSupport; import org.junit.Test; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -29,8 +31,9 @@ import java.util.Set; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertNotNull; import static junit.framework.Assert.assertTrue; - +import static org.easymock.EasyMock.expect; /** * BlueprintConfigurationProcessor unit tests. @@ -897,24 +900,899 @@ public class BlueprintConfigurationProcessorTest { } @Test - public void testFalconConfigPropertyUpdaterAdded() throws Exception { - Map> singleHostUpdaters = - BlueprintConfigurationProcessor.getSingleHostTopologyUpdaters(); + public void testFalconConfigExport() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedPortNum = "808080"; + final String expectedHostGroupName = "host_group_1"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map falconStartupProperties = + new HashMap(); + + configProperties.put("falcon-startup.properties", falconStartupProperties); + + // setup properties that include host information + falconStartupProperties.put("*.broker.url", expectedHostName + ":" + expectedPortNum); + falconStartupProperties.put("*.falcon.service.authentication.kerberos.principal", "falcon/" + expectedHostName + "@EXAMPLE.COM"); + falconStartupProperties.put("*.falcon.http.authentication.kerberos.principal", "HTTP/" + expectedHostName + "@EXAMPLE.COM"); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + // call top-level export method + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne)); + + assertEquals("Falcon Broker URL property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), falconStartupProperties.get("*.broker.url")); + + assertEquals("Falcon Kerberos Principal property not properly exported", + "falcon/" + "%HOSTGROUP::" + expectedHostGroupName + "%" + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal")); + + assertEquals("Falcon Kerberos HTTP Principal property not properly exported", + "HTTP/" + "%HOSTGROUP::" + expectedHostGroupName + "%" + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal")); + + mockSupport.verifyAll(); + + } + + @Test + public void testFalconConfigClusterUpdate() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedPortNum = "808080"; + final String expectedHostGroupName = "host_group_1"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map falconStartupProperties = + new HashMap(); + + configProperties.put("falcon-startup.properties", falconStartupProperties); + + // setup properties that include host information + falconStartupProperties.put("*.broker.url", createExportedAddress(expectedPortNum, expectedHostGroupName)); + falconStartupProperties.put("*.falcon.service.authentication.kerberos.principal", "falcon/" + createExportedHostName(expectedHostGroupName) + "@EXAMPLE.COM"); + falconStartupProperties.put("*.falcon.http.authentication.kerberos.principal", "HTTP/" + createExportedHostName(expectedHostGroupName) + "@EXAMPLE.COM"); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + Map mapOfHostGroups = + new HashMap(); + mapOfHostGroups.put(expectedHostGroupName, mockHostGroupOne); + + // call top-level export method + configProcessor.doUpdateForClusterCreate(mapOfHostGroups); + + assertEquals("Falcon Broker URL property not properly exported", + expectedHostName + ":" + expectedPortNum, falconStartupProperties.get("*.broker.url")); + + assertEquals("Falcon Kerberos Principal property not properly exported", + "falcon/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal")); + + assertEquals("Falcon Kerberos HTTP Principal property not properly exported", + "HTTP/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal")); + + mockSupport.verifyAll(); + + } + + @Test + public void testFalconConfigClusterUpdateDefaultConfig() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedPortNum = "808080"; + final String expectedHostGroupName = "host_group_1"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + expect(mockHostGroupOne.getComponents()).andReturn(Arrays.asList("FALCON_SERVER")).atLeastOnce(); + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map falconStartupProperties = + new HashMap(); + + configProperties.put("falcon-startup.properties", falconStartupProperties); + + // setup properties that include host information + falconStartupProperties.put("*.broker.url", "localhost:" + expectedPortNum); + falconStartupProperties.put("*.falcon.service.authentication.kerberos.principal", "falcon/" + "localhost" + "@EXAMPLE.COM"); + falconStartupProperties.put("*.falcon.http.authentication.kerberos.principal", "HTTP/" + "localhost" + "@EXAMPLE.COM"); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + Map mapOfHostGroups = + new HashMap(); + mapOfHostGroups.put(expectedHostGroupName, mockHostGroupOne); + + // call top-level export method + configProcessor.doUpdateForClusterCreate(mapOfHostGroups); + + assertEquals("Falcon Broker URL property not properly exported", + expectedHostName + ":" + expectedPortNum, falconStartupProperties.get("*.broker.url")); + + assertEquals("Falcon Kerberos Principal property not properly exported", + "falcon/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal")); + + assertEquals("Falcon Kerberos HTTP Principal property not properly exported", + "HTTP/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal")); + + mockSupport.verifyAll(); + + } + + @Test + public void testDoUpdateForClusterWithNameNodeHAEnabled() throws Exception { + final String expectedNameService = "mynameservice"; + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedPortNum = "808080"; + final String expectedNodeOne = "nn1"; + final String expectedNodeTwo = "nn2"; + final String expectedHostGroupName = "host_group_1"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map hdfsSiteProperties = + new HashMap(); + + configProperties.put("hdfs-site", hdfsSiteProperties); + + // setup hdfs HA config for test + hdfsSiteProperties.put("dfs.nameservices", expectedNameService); + hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne + ", " + expectedNodeTwo); + + + // setup properties that include exported host group information + hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName)); + hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName)); + hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName)); + hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName)); + hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName)); + hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName)); + + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + Map mapOfHostGroups = new HashMap(); + mapOfHostGroups.put(expectedHostGroupName,mockHostGroupOne); + + configProcessor.doUpdateForClusterCreate(mapOfHostGroups); + + // verify that the expected hostname was substitued for the host group name in the config + assertEquals("HTTPS address HA property not properly exported", + expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo)); + + assertEquals("HTTPS address HA property not properly exported", + expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo)); + + assertEquals("HTTPS address HA property not properly exported", + expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo)); + + mockSupport.verifyAll(); + } + + @Test + public void testDoNameNodeHighAvailabilityUpdateWithHAEnabled() throws Exception { + final String expectedNameService = "mynameservice"; + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedPortNum = "808080"; + final String expectedNodeOne = "nn1"; + final String expectedNodeTwo = "nn2"; + final String expectedHostGroupName = "host_group_1"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map hdfsSiteProperties = + new HashMap(); + + configProperties.put("hdfs-site", hdfsSiteProperties); + + // setup hdfs config for test + + hdfsSiteProperties.put("dfs.nameservices", expectedNameService); + hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne + ", " + expectedNodeTwo); + + + // setup properties that include host information + hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); + + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + // call top-level export method, which will call the HA-specific method if HA is enabled + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne)); + + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo)); + + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo)); + + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo)); + + mockSupport.verifyAll(); + + } + + @Test + public void testDoNameNodeHighAvailabilityUpdateWithHANotEnabled() throws Exception { + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map hdfsSiteProperties = + new HashMap(); + + configProperties.put("hdfs-site", hdfsSiteProperties); + + // hdfs-site config for this test will not include an HA values + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + assertEquals("Incorrect initial state for hdfs-site config", + 0, hdfsSiteProperties.size()); + + // call top-level export method + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne)); + + assertEquals("Incorrect state for hdsf-site config after HA call in non-HA environment, should be zero", + 0, hdfsSiteProperties.size()); + + mockSupport.verifyAll(); + + } + + @Test + public void testDoNameNodeHighAvailabilityUpdateWithHAEnabledMultipleServices() throws Exception { + final String expectedNameServiceOne = "mynameserviceOne"; + final String expectedNameServiceTwo = "mynameserviceTwo"; + final String expectedHostNameOne = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.apache.ambari.org"; + + final String expectedPortNum = "808080"; + final String expectedNodeOne = "nn1"; + final String expectedNodeTwo = "nn2"; + final String expectedHostGroupName = "host_group_1"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostNameOne, expectedHostNameTwo, "serverTwo")).atLeastOnce(); + expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map hdfsSiteProperties = + new HashMap(); + + configProperties.put("hdfs-site", hdfsSiteProperties); + + // setup hdfs config for test + + hdfsSiteProperties.put("dfs.nameservices", expectedNameServiceOne + "," + expectedNameServiceTwo); + hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceOne, expectedNodeOne + ", " + expectedNodeTwo); + hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceTwo, expectedNodeOne + ", " + expectedNodeTwo); + + + // setup properties that include host information for nameservice one + hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeOne, expectedHostNameOne + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeTwo, expectedHostNameOne + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeOne, expectedHostNameOne + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeTwo, expectedHostNameOne + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeOne, expectedHostNameOne + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeTwo, expectedHostNameOne + ":" + expectedPortNum); + + // setup properties that include host information for nameservice two + hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeOne, expectedHostNameTwo + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeTwo, expectedHostNameTwo + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeOne, expectedHostNameTwo + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeTwo, expectedHostNameTwo + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeOne, expectedHostNameTwo + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeTwo, expectedHostNameTwo + ":" + expectedPortNum); + - assertTrue("Falcon startup.properties map was not added to the list of updater maps", - singleHostUpdaters.containsKey("falcon-startup.properties")); + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); - Map fieldsToUpdaters = - singleHostUpdaters.get("falcon-startup.properties"); + // call top-level export method, which will call the HA-specific method if HA is enabled + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne)); - assertTrue("Expected Falcon config property was not present in updater map", - fieldsToUpdaters.containsKey("*.broker.url")); + // verify results for name service one + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeTwo)); - assertTrue("PropertyUpdater was not of the expected type for Falcon config property", - fieldsToUpdaters.get("*.broker.url") instanceof BlueprintConfigurationProcessor.SingleHostTopologyUpdater); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeTwo)); + + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeTwo)); + + + // verify results for name service two + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeTwo)); + + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeTwo)); + + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeOne)); + assertEquals("HTTPS address HA property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeTwo)); + + mockSupport.verifyAll(); + + } + + @Test + public void testIsNameNodeHAEnabled() throws Exception { + Map> configProperties = + new HashMap>(); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + assertFalse("Incorrect HA detection, hdfs-site not available", + configProcessor.isNameNodeHAEnabled()); + + Map hdfsSiteMap = new HashMap(); + configProperties.put("hdfs-site", hdfsSiteMap); + + assertFalse("Incorrect HA detection, HA flag not enabled", + configProcessor.isNameNodeHAEnabled()); + + hdfsSiteMap.put("dfs.nameservices", "myTestNameService"); + + assertTrue("Incorrect HA detection, HA was enabled", + configProcessor.isNameNodeHAEnabled()); + + } + + @Test + public void testParseNameServices() throws Exception { + Map hdfsSiteConfigMap = + new HashMap(); + hdfsSiteConfigMap.put("dfs.nameservices", "serviceOne"); + + // verify that a single service is parsed correctly + String[] result = BlueprintConfigurationProcessor.parseNameServices(hdfsSiteConfigMap); + + assertNotNull("Resulting array was null", + result); + assertEquals("Incorrect array size", + 1, result.length); + assertEquals("Incorrect value for returned name service", + "serviceOne", result[0]); + + // verify that multiple services are parsed correctly + hdfsSiteConfigMap.put("dfs.nameservices", " serviceTwo, serviceThree, serviceFour"); + + String[] resultTwo = BlueprintConfigurationProcessor.parseNameServices(hdfsSiteConfigMap); + + assertNotNull("Resulting array was null", + resultTwo); + assertEquals("Incorrect array size", + 3, resultTwo.length); + assertEquals("Incorrect value for returned name service", + "serviceTwo", resultTwo[0]); + assertEquals("Incorrect value for returned name service", + "serviceThree", resultTwo[1]); + assertEquals("Incorrect value for returned name service", + "serviceFour", resultTwo[2]); + } + + @Test + public void testParseNameNodes() throws Exception { + final String expectedServiceName = "serviceOne"; + Map hdfsSiteConfigMap = + new HashMap(); + hdfsSiteConfigMap.put("dfs.ha.namenodes." + expectedServiceName, "node1"); + + // verify that a single name node is parsed correctly + String[] result = + BlueprintConfigurationProcessor.parseNameNodes(expectedServiceName, hdfsSiteConfigMap); + + assertNotNull("Resulting array was null", + result); + assertEquals("Incorrect array size", + 1, result.length); + assertEquals("Incorrect value for returned name nodes", + "node1", result[0]); + + // verify that multiple name nodes are parsed correctly + hdfsSiteConfigMap.put("dfs.ha.namenodes." + expectedServiceName, " nodeSeven, nodeEight, nodeNine"); + + String[] resultTwo = + BlueprintConfigurationProcessor.parseNameNodes(expectedServiceName, hdfsSiteConfigMap); + + assertNotNull("Resulting array was null", + resultTwo); + assertEquals("Incorrect array size", + 3, resultTwo.length); + assertEquals("Incorrect value for returned name node", + "nodeSeven", resultTwo[0]); + assertEquals("Incorrect value for returned name node", + "nodeEight", resultTwo[1]); + assertEquals("Incorrect value for returned name node", + "nodeNine", resultTwo[2]); + + } + + @Test + public void testYarnConfigExported() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedPortNum = "808080"; + final String expectedHostGroupName = "host_group_1"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map yarnSiteProperties = + new HashMap(); + + configProperties.put("yarn-site", yarnSiteProperties); + + // setup properties that include host information + yarnSiteProperties.put("yarn.log.server.url", "http://" + expectedHostName +":19888/jobhistory/logs"); + yarnSiteProperties.put("yarn.resourcemanager.hostname", expectedHostName); + yarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", expectedHostName + ":" + expectedPortNum); + yarnSiteProperties.put("yarn.resourcemanager.webapp.address", expectedHostName + ":" + expectedPortNum); + yarnSiteProperties.put("yarn.resourcemanager.scheduler.address", expectedHostName + ":" + expectedPortNum); + yarnSiteProperties.put("yarn.resourcemanager.address", expectedHostName + ":" + expectedPortNum); + yarnSiteProperties.put("yarn.resourcemanager.admin.address", expectedHostName + ":" + expectedPortNum); + yarnSiteProperties.put("yarn.timeline-service.address", expectedHostName + ":" + expectedPortNum); + yarnSiteProperties.put("yarn.timeline-service.webapp.address", expectedHostName + ":" + expectedPortNum); + yarnSiteProperties.put("yarn.timeline-service.webapp.https.address", expectedHostName + ":" + expectedPortNum); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + // call top-level export method + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne)); + + assertEquals("Yarn Log Server URL was incorrectly exported", + "http://" + "%HOSTGROUP::" + expectedHostGroupName + "%" +":19888/jobhistory/logs", yarnSiteProperties.get("yarn.log.server.url")); + assertEquals("Yarn ResourceManager hostname was incorrectly exported", + createExportedHostName(expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.hostname")); + assertEquals("Yarn ResourceManager tracker address was incorrectly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.resource-tracker.address")); + assertEquals("Yarn ResourceManager webapp address was incorrectly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.webapp.address")); + assertEquals("Yarn ResourceManager scheduler address was incorrectly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.scheduler.address")); + assertEquals("Yarn ResourceManager address was incorrectly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.address")); + assertEquals("Yarn ResourceManager admin address was incorrectly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.admin.address")); + assertEquals("Yarn ResourceManager timeline-service address was incorrectly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.address")); + assertEquals("Yarn ResourceManager timeline webapp address was incorrectly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.webapp.address")); + assertEquals("Yarn ResourceManager timeline webapp HTTPS address was incorrectly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.webapp.https.address")); + + mockSupport.verifyAll(); + + } + + @Test + public void testHDFSConfigExported() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedPortNum = "808080"; + final String expectedHostGroupName = "host_group_1"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map hdfsSiteProperties = + new HashMap(); + + Map coreSiteProperties = + new HashMap(); + + Map hbaseSiteProperties = + new HashMap(); + + configProperties.put("hdfs-site", hdfsSiteProperties); + configProperties.put("core-site", coreSiteProperties); + configProperties.put("hbase-site", hbaseSiteProperties); + + // setup properties that include host information + hdfsSiteProperties.put("dfs.http.address", expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.https.address", expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.http-address", expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.https-address", expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.secondary.http.address", expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.secondary.http-address", expectedHostName + ":" + expectedPortNum); + hdfsSiteProperties.put("dfs.namenode.shared.edits.dir", expectedHostName + ":" + expectedPortNum); + + coreSiteProperties.put("fs.default.name", expectedHostName + ":" + expectedPortNum); + coreSiteProperties.put("fs.defaultFS", "hdfs://" + expectedHostName + ":" + expectedPortNum); + + hbaseSiteProperties.put("hbase.rootdir", "hdfs://" + expectedHostName + ":" + expectedPortNum + "/apps/hbase/data"); + + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + // call top-level export method + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne)); + + assertEquals("hdfs config property not exported properly", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.http.address")); + assertEquals("hdfs config property not exported properly", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.https.address")); + assertEquals("hdfs config property not exported properly", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address")); + assertEquals("hdfs config property not exported properly", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address")); + assertEquals("hdfs config property not exported properly", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.secondary.http.address")); + assertEquals("hdfs config property not exported properly", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.secondary.http-address")); + assertEquals("hdfs config property not exported properly", + createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.shared.edits.dir")); + + assertEquals("hdfs config in core-site not exported properly", + createExportedAddress(expectedPortNum, expectedHostGroupName), coreSiteProperties.get("fs.default.name")); + assertEquals("hdfs config in core-site not exported properly", + "hdfs://" + createExportedAddress(expectedPortNum, expectedHostGroupName), coreSiteProperties.get("fs.defaultFS")); + + assertEquals("hdfs config in hbase-site not exported properly", + "hdfs://" + createExportedAddress(expectedPortNum, expectedHostGroupName) + "/apps/hbase/data", hbaseSiteProperties.get("hbase.rootdir")); + + mockSupport.verifyAll(); + + } + + @Test + public void testHiveConfigExported() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.ambari.apache.org"; + final String expectedPortNum = "808080"; + final String expectedHostGroupName = "host_group_1"; + final String expectedHostGroupNameTwo = "host_group_2"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + HostGroup mockHostGroupTwo = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + expect(mockHostGroupTwo.getHostInfo()).andReturn(Arrays.asList(expectedHostNameTwo, "serverTwo")).atLeastOnce(); + expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce(); + expect(mockHostGroupTwo.getName()).andReturn(expectedHostGroupNameTwo).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map hiveSiteProperties = + new HashMap(); + Map hiveEnvProperties = + new HashMap(); + Map webHCatSiteProperties = + new HashMap(); + Map coreSiteProperties = + new HashMap(); + + configProperties.put("hive-site", hiveSiteProperties); + configProperties.put("hive-env", hiveEnvProperties); + configProperties.put("webhcat-site", webHCatSiteProperties); + configProperties.put("core-site", coreSiteProperties); + + + // setup properties that include host information + hiveSiteProperties.put("hive.metastore.uris", expectedHostName + ":" + expectedPortNum); + hiveSiteProperties.put("javax.jdo.option.ConnectionURL", expectedHostName + ":" + expectedPortNum); + hiveEnvProperties.put("hive_hostname", expectedHostName); + + webHCatSiteProperties.put("templeton.hive.properties", expectedHostName + "," + expectedHostNameTwo); + webHCatSiteProperties.put("templeton.kerberos.principal", expectedHostName); + + coreSiteProperties.put("hadoop.proxyuser.hive.hosts", expectedHostName + "," + expectedHostNameTwo); + coreSiteProperties.put("hadoop.proxyuser.HTTP.hosts", expectedHostName + "," + expectedHostNameTwo); + coreSiteProperties.put("hadoop.proxyuser.hcat.hosts", expectedHostName + "," + expectedHostNameTwo); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + // call top-level export method + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne, mockHostGroupTwo)); + + assertEquals("hive property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("hive.metastore.uris")); + assertEquals("hive property not properly exported", + createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("javax.jdo.option.ConnectionURL")); + assertEquals("hive property not properly exported", + createExportedHostName(expectedHostGroupName), hiveEnvProperties.get("hive_hostname")); + + assertEquals("hive property not properly exported", + createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), + webHCatSiteProperties.get("templeton.hive.properties")); + assertEquals("hive property not properly exported", + createExportedHostName(expectedHostGroupName), webHCatSiteProperties.get("templeton.kerberos.principal")); + + assertEquals("hive property not properly exported", + createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hive.hosts")); + + assertEquals("hive property not properly exported", + createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.HTTP.hosts")); + + assertEquals("hive property not properly exported", + createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hcat.hosts")); + + mockSupport.verifyAll(); + } + + + @Test + public void testOozieConfigExported() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.ambari.apache.org"; + final String expectedHostGroupName = "host_group_1"; + final String expectedHostGroupNameTwo = "host_group_2"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + HostGroup mockHostGroupTwo = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + expect(mockHostGroupTwo.getHostInfo()).andReturn(Arrays.asList(expectedHostNameTwo, "serverTwo")).atLeastOnce(); + expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce(); + expect(mockHostGroupTwo.getName()).andReturn(expectedHostGroupNameTwo).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map oozieSiteProperties = + new HashMap(); + Map oozieEnvProperties = + new HashMap(); + Map coreSiteProperties = + new HashMap(); + + configProperties.put("oozie-site", oozieSiteProperties); + configProperties.put("oozie-env", oozieEnvProperties); + configProperties.put("hive-env", oozieEnvProperties); + configProperties.put("core-site", coreSiteProperties); + + oozieSiteProperties.put("oozie.base.url", expectedHostName); + oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName); + oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName); + + oozieEnvProperties.put("oozie_hostname", expectedHostName); + + coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + // call top-level export method + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne, mockHostGroupTwo)); + + assertEquals("oozie property not exported correctly", + createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.base.url")); + assertEquals("oozie property not exported correctly", + createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.authentication.kerberos.principal")); + assertEquals("oozie property not exported correctly", + createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal")); + assertEquals("oozie property not exported correctly", + createExportedHostName(expectedHostGroupName), oozieEnvProperties.get("oozie_hostname")); + assertEquals("oozie property not exported correctly", + createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.oozie.hosts")); + + mockSupport.verifyAll(); + + } + + @Test + public void testZookeeperConfigExported() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.ambari.apache.org"; + final String expectedHostGroupName = "host_group_1"; + final String expectedHostGroupNameTwo = "host_group_2"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + HostGroup mockHostGroupTwo = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + expect(mockHostGroupTwo.getHostInfo()).andReturn(Arrays.asList(expectedHostNameTwo, "serverTwo")).atLeastOnce(); + expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce(); + expect(mockHostGroupTwo.getName()).andReturn(expectedHostGroupNameTwo).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map coreSiteProperties = + new HashMap(); + Map hbaseSiteProperties = + new HashMap(); + Map webHCatSiteProperties = + new HashMap(); + + configProperties.put("core-site", coreSiteProperties); + configProperties.put("hbase-site", hbaseSiteProperties); + configProperties.put("webhcat-site", webHCatSiteProperties); + + coreSiteProperties.put("ha.zookeeper.quorum", expectedHostName + "," + expectedHostNameTwo); + hbaseSiteProperties.put("hbase.zookeeper.quorum", expectedHostName + "," + expectedHostNameTwo); + webHCatSiteProperties.put("templeton.zookeeper.hosts", expectedHostName + "," + expectedHostNameTwo); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + // call top-level export method + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne, mockHostGroupTwo)); + + assertEquals("zookeeper config not properly exported", + createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), + coreSiteProperties.get("ha.zookeeper.quorum")); + assertEquals("zookeeper config not properly exported", + createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), + hbaseSiteProperties.get("hbase.zookeeper.quorum")); + assertEquals("zookeeper config not properly exported", + createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), + webHCatSiteProperties.get("templeton.zookeeper.hosts")); + + mockSupport.verifyAll(); + + } + + @Test + public void testNagiosConfigExported() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedHostGroupName = "host_group_1"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map nagiosEnvProperties = + new HashMap(); + + configProperties.put("nagios-env", nagiosEnvProperties); + + nagiosEnvProperties.put("nagios_principal_name", expectedHostName); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + // call top-level export method + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne)); + + assertEquals("nagios config not properly exported", + createExportedHostName(expectedHostGroupName), + nagiosEnvProperties.get("nagios_principal_name")); + + mockSupport.verifyAll(); + + } + + private static String createExportedAddress(String expectedPortNum, String expectedHostGroupName) { + return createExportedHostName(expectedHostGroupName) + ":" + expectedPortNum; + } - assertEquals("PropertyUpdater was not associated with the expected component name", - "FALCON_SERVER", ((BlueprintConfigurationProcessor.SingleHostTopologyUpdater)fieldsToUpdaters.get("*.broker.url")).getComponentName()); + private static String createExportedHostName(String expectedHostGroupName) { + return "%HOSTGROUP::" + expectedHostGroupName + "%"; } private class TestHostGroup implements HostGroup {