ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jspei...@apache.org
Subject git commit: Remove Oozie external DB properties from Blueprint Export
Date Fri, 26 Sep 2014 15:42:21 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 2e3082af1 -> 20f112acd


Remove Oozie external DB properties from Blueprint Export

This patch resolves bug AMBARI-7507.

Some Oozie properties related to using an external MySQL
  Database (oozie_existing_mysql_host, oozie.service.JPAService.jdbc.url)
  are currently appearing in an exported Blueprint with
  hostname information included.  This is incorrect, since
  the hostname information should be completely removed or
  masked in an exported Blueprint.  Having a reference to a
  given hostname in this scenario is not very useful, since the
  Blueprint may be applied to a completely different set of
  machines.

These Oozie properties generally will only be set to
  either a default, in-memory DB, or to an external
  MYSQL server instance.  This means that the normal
  PropertyUpdater processing won't really apply to
  these two properties.

This patch fixes this problem by creating a new
  PropertyUpdater type, which is only responsible
  for tracking these two property types, and marking
  them as properties that will not be included
  in an exported Blueprint.

This patch also adds a processing method to
  handle any properties that use this new
  PropertyUpdater, and this processing method
  will remove these registered properties
  during the export process.

This patch also adds a unit test to verify this change.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/20f112ac
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/20f112ac
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/20f112ac

Branch: refs/heads/trunk
Commit: 20f112acd619f58aaad85a5fa65ff82c2e4cc498
Parents: 2e3082a
Author: Bob Nettleton <rnettleton@hortonworks.com>
Authored: Thu Sep 25 21:37:42 2014 -0400
Committer: John Speidel <jspeidel@hortonworks.com>
Committed: Fri Sep 26 11:42:07 2014 -0400

----------------------------------------------------------------------
 .../BlueprintConfigurationProcessor.java        | 60 ++++++++++++++++++++
 .../BlueprintConfigurationProcessorTest.java    |  9 +++
 2 files changed, 69 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/20f112ac/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 3e1bcf0..d33ae65 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -61,6 +61,15 @@ public class BlueprintConfigurationProcessor {
       new HashMap<String, Map<String, PropertyUpdater>>();
 
   /**
+   * Updaters that preserve the original property value, functions
+   *   as a placeholder for DB-related properties that need to be
+   *   removed from export, but do not require an update during
+   *   cluster creation
+   */
+  private static Map<String, Map<String, PropertyUpdater>> removePropertyUpdaters
=
+    new HashMap<String, Map<String, PropertyUpdater>>();
+
+  /**
    * Collection of all updaters
    */
   private static Collection<Map<String, Map<String, PropertyUpdater>>>
allUpdaters =
@@ -181,10 +190,37 @@ public class BlueprintConfigurationProcessor {
 
     doMultiHostExportUpdate(hostGroups, multiHostTopologyUpdaters);
 
+    doRemovePropertyExport(removePropertyUpdaters);
+
     return properties;
   }
 
   /**
+   * Performs export update for the set of properties that do not
+   * require update during cluster setup, but should be removed
+   * during a Blueprint export.
+   *
+   * In the case of a service referring to an external DB, any
+   * properties that contain external host information should
+   * be removed from the configuration that will be available in
+   * the exported Blueprint.
+   *
+   * @param updaters set of updaters for properties that should
+   *                 always be removed during a Blueprint export
+   */
+  private void doRemovePropertyExport(Map<String, Map<String, PropertyUpdater>>
updaters) {
+    for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet())
{
+      String type = entry.getKey();
+      for (String propertyName : entry.getValue().keySet()) {
+        Map<String, String> typeProperties = properties.get(type);
+        if ( (typeProperties != null) && (typeProperties.containsKey(propertyName))
) {
+          typeProperties.remove(propertyName);
+        }
+      }
+    }
+  }
+
+  /**
    * Perform export update processing for HA configuration for NameNodes.  The HA NameNode
property
    *   names are based on the nameservices defined when HA is enabled via the Ambari UI,
so this method
    *   dynamically determines the property names, and registers PropertyUpdaters to handle
the masking of
@@ -768,6 +804,21 @@ public class BlueprintConfigurationProcessor {
   }
 
   /**
+   * PropertyUpdater implementation that will always return the original
+   *   value for the updateForClusterCreate() method.
+   *   This updater type should only be used in cases where a given
+   *   property requires no updates, but may need to be considered
+   *   during the Blueprint export process.
+   */
+  private static class OriginalValuePropertyUpdater implements PropertyUpdater {
+    @Override
+    public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
String origValue, Map<String, Map<String, String>> properties) {
+      // always return the original value, since these properties do not require update handling
+      return origValue;
+    }
+  }
+
+  /**
    * Register updaters for configuration properties.
    */
   static {
@@ -783,6 +834,7 @@ public class BlueprintConfigurationProcessor {
     Map<String, PropertyUpdater> hbaseSiteMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> yarnSiteMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> hiveSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> oozieSiteOriginalValueMap = new HashMap<String,
PropertyUpdater>();
     Map<String, PropertyUpdater> oozieSiteMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> stormSiteMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> falconStartupPropertiesMap = new HashMap<String,
PropertyUpdater>();
@@ -793,6 +845,8 @@ public class BlueprintConfigurationProcessor {
     Map<String, PropertyUpdater> hbaseEnvMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> hiveEnvMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> oozieEnvMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> oozieEnvOriginalValueMap = new HashMap<String,
PropertyUpdater>();
+
 
     Map<String, PropertyUpdater> multiWebhcatSiteMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> multiHbaseSiteMap = new HashMap<String, PropertyUpdater>();
@@ -830,6 +884,9 @@ public class BlueprintConfigurationProcessor {
 
     dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap);
 
+    removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap);
+    removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap);
+
     // NAMENODE
     hdfsSiteMap.put("dfs.http.address", new SingleHostTopologyUpdater("NAMENODE"));
     hdfsSiteMap.put("dfs.https.address", new SingleHostTopologyUpdater("NAMENODE"));
@@ -886,6 +943,9 @@ public class BlueprintConfigurationProcessor {
     oozieSiteMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER"));
     oozieEnvMap.put("oozie_hostname", new SingleHostTopologyUpdater("OOZIE_SERVER"));
     multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER"));
+    // register updaters for Oozie properties that may point to an external DB
+    oozieEnvOriginalValueMap.put("oozie_existing_mysql_host", new OriginalValuePropertyUpdater());
+    oozieSiteOriginalValueMap.put("oozie.service.JPAService.jdbc.url", new OriginalValuePropertyUpdater());
 
     // ZOOKEEPER_SERVER
     multiHbaseSiteMap.put("hbase.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));

http://git-wip-us.apache.org/repos/asf/ambari/blob/20f112ac/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 590b9c2..90f83bc 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -1685,6 +1685,7 @@ public class BlueprintConfigurationProcessorTest {
   public void testOozieConfigExported() throws Exception {
     final String expectedHostName = "c6401.apache.ambari.org";
     final String expectedHostNameTwo = "c6402.ambari.apache.org";
+    final String expectedExternalHost = "c6408.ambari.apache.org";
     final String expectedHostGroupName = "host_group_1";
     final String expectedHostGroupNameTwo = "host_group_2";
 
@@ -1718,8 +1719,10 @@ public class BlueprintConfigurationProcessorTest {
     oozieSiteProperties.put("oozie.base.url", expectedHostName);
     oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName);
     oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName);
+    oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + expectedExternalHost
+ "/ooziedb");
 
     oozieEnvProperties.put("oozie_hostname", expectedHostName);
+    oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost);
 
     coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo);
 
@@ -1740,6 +1743,12 @@ public class BlueprintConfigurationProcessorTest {
     assertEquals("oozie property not exported correctly",
       createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo),
coreSiteProperties.get("hadoop.proxyuser.oozie.hosts"));
 
+    // verify that the oozie properties that can refer to an external DB are not included
in the export
+    assertFalse("oozie_existing_mysql_host should not have been present in the exported configuration",
+      oozieEnvProperties.containsKey("oozie_existing_mysql_host"));
+    assertFalse("oozie.service.JPAService.jdbc.url should not have been present in the exported
configuration",
+      oozieSiteProperties.containsKey("oozie.service.JPAService.jdbc.url"));
+
     mockSupport.verifyAll();
 
   }


Mime
View raw message