Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6698811A5C for ; Wed, 24 Sep 2014 20:53:02 +0000 (UTC) Received: (qmail 93901 invoked by uid 500); 24 Sep 2014 20:53:02 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 93839 invoked by uid 500); 24 Sep 2014 20:53:02 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 92652 invoked by uid 99); 24 Sep 2014 20:53:01 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Sep 2014 20:53:01 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 559779A3BF3; Wed, 24 Sep 2014 20:53:01 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jonathanhurley@apache.org To: commits@ambari.apache.org Date: Wed, 24 Sep 2014 20:53:28 -0000 Message-Id: <292a6fb999804e7cb0a8fe8f0fb0b0e6@git.apache.org> In-Reply-To: <998119e1ae8c4d4598070a58f013b4f6@git.apache.org> References: <998119e1ae8c4d4598070a58f013b4f6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [29/37] git commit: Fix Blueprint export handling of properties containing nameservice references Fix Blueprint export handling of properties containing nameservice references This patch implements a fix for AMBARI-7458. This patch updates the BlueprintConfigurationProcessor such that properties that can contain a namservice reference (there are only two known properties at this point) are not removed by the BlueprintConfigurationProcessor during the export process. Previously, these properties were removed, since they didn't contain hostname information. In a NameNode HA scenario, these particular properties will reference nameservices, so this problem would only occur when NameNode HA is enabled. This patch addresses the immediate problem, but in the longer-term the stack definitions should be updated to include HA-related metadata, so that the Blueprint processor can determine which properties are modified in an HA scenario. This patch also implements a new unit test to verify this change. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6c6e4fc2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6c6e4fc2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6c6e4fc2 Branch: refs/heads/branch-alerts-dev Commit: 6c6e4fc2eabd1af1c8bd89aa6aebf4068bdcdefb Parents: d8af15e Author: Bob Nettleton Authored: Tue Sep 23 14:08:38 2014 -0400 Committer: John Speidel Committed: Wed Sep 24 13:20:09 2014 -0400 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 28 +++++++++- .../BlueprintConfigurationProcessorTest.java | 54 +++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6c6e4fc2/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index b3cc098..3e1bcf0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -19,12 +19,14 @@ package org.apache.ambari.server.controller.internal; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -75,6 +77,14 @@ public class BlueprintConfigurationProcessor { private static Pattern HOSTGROUP_PORT_REGEX = Pattern.compile("%HOSTGROUP::(\\w+|\\d+)%:?(\\d+)?"); /** + * Statically-defined set of properties that can support using a nameservice name + * in the configuration, rather than just a host name. + */ + private static Set configPropertiesWithHASupport = + new HashSet(Arrays.asList("fs.defaultFS", "hbase.rootdir")); + + + /** * Configuration properties to be updated */ private Map> properties; @@ -290,7 +300,10 @@ public class BlueprintConfigurationProcessor { break; } } - if (! matchedHost) { + // remove properties that do not contain hostnames, + // except in the case of HA-related properties, that + // can contain nameservice references instead of hostnames (Fix for Bug AMBARI-7458). + if (! matchedHost && ! isNameServiceProperty(propertyName)) { typeProperties.remove(propertyName); } } @@ -299,6 +312,19 @@ public class BlueprintConfigurationProcessor { } /** + * Determines if a given property name's value can include + * nameservice references instead of host names. + * + * @param propertyName name of the property + * + * @return true if this property can support using nameservice names + * false if this property cannot support using nameservice names + */ + private static boolean isNameServiceProperty(String propertyName) { + return configPropertiesWithHASupport.contains(propertyName); + } + + /** * Update multi host topology configuration properties for blueprint export. * * @param hostGroups cluster host groups http://git-wip-us.apache.org/repos/asf/ambari/blob/6c6e4fc2/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index f906092..590b9c2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -1135,8 +1135,15 @@ public class BlueprintConfigurationProcessorTest { Map hdfsSiteProperties = new HashMap(); + Map coreSiteProperties = + new HashMap(); + Map hbaseSiteProperties = + new HashMap(); + configProperties.put("hdfs-site", hdfsSiteProperties); + configProperties.put("core-site", coreSiteProperties); + configProperties.put("hbase-site", hbaseSiteProperties); // setup hdfs config for test @@ -1152,7 +1159,6 @@ public class BlueprintConfigurationProcessorTest { hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); - BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(configProperties); @@ -1179,6 +1185,52 @@ public class BlueprintConfigurationProcessorTest { } @Test + public void testDoNameNodeHighAvailabilityUpdateWithHAEnabledNameServicePropertiesIncluded() throws Exception { + final String expectedNameService = "mynameservice"; + final String expectedHostName = "c6401.apache.ambari.org"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce(); + + mockSupport.replayAll(); + + Map> configProperties = + new HashMap>(); + + Map coreSiteProperties = + new HashMap(); + Map hbaseSiteProperties = + new HashMap(); + + + configProperties.put("core-site", coreSiteProperties); + configProperties.put("hbase-site", hbaseSiteProperties); + + // configure fs.defaultFS to include a nameservice name, rather than a host name + coreSiteProperties.put("fs.defaultFS", "hdfs://" + expectedNameService); + // configure hbase.rootdir to include a nameservice name, rather than a host name + hbaseSiteProperties.put("hbase.rootdir", "hdfs://" + expectedNameService + "/apps/hbase/data"); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + // call top-level export method, which will call the HA-specific method if HA is enabled + configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne)); + + // verify that any properties that include nameservices are not removed from the exported blueprint's configuration + assertEquals("Property containing an HA nameservice (fs.defaultFS), was not correctly exported by the processor", + "hdfs://" + expectedNameService, coreSiteProperties.get("fs.defaultFS")); + assertEquals("Property containing an HA nameservice (hbase.rootdir), was not correctly exported by the processor", + "hdfs://" + expectedNameService + "/apps/hbase/data", hbaseSiteProperties.get("hbase.rootdir")); + + mockSupport.verifyAll(); + + } + + @Test public void testDoNameNodeHighAvailabilityUpdateWithHANotEnabled() throws Exception { EasyMockSupport mockSupport = new EasyMockSupport();