ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rnettle...@apache.org
Subject ambari git commit: AMBARI-11308. Oozie HA configuration property not exported properly in a Blueprint. (rnettleton)
Date Fri, 22 May 2015 15:17:40 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 714d2535e -> 832f7e1ec


AMBARI-11308. Oozie HA configuration property not exported properly in a Blueprint. (rnettleton)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/832f7e1e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/832f7e1e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/832f7e1e

Branch: refs/heads/trunk
Commit: 832f7e1ec7a04c8ade400f9bfa679d2d7069ae13
Parents: 714d253
Author: Bob Nettleton <rnettleton@hortonworks.com>
Authored: Fri May 22 11:17:10 2015 -0400
Committer: Bob Nettleton <rnettleton@hortonworks.com>
Committed: Fri May 22 11:17:31 2015 -0400

----------------------------------------------------------------------
 .../BlueprintConfigurationProcessor.java        | 70 +++++++++++++++++++
 .../OOZIE/5.0.0.2.3/metainfo.xml                |  2 +-
 .../BlueprintConfigurationProcessorTest.java    | 71 ++++++++++++++++++++
 3 files changed, 142 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/832f7e1e/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 6984881..a84bf3d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -231,6 +231,10 @@ public class BlueprintConfigurationProcessor {
       doYarnResourceManagerHAUpdate();
     }
 
+    if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) {
+      doOozieServerHAUpdate();
+    }
+
     Collection<Map<String, Map<String, String>>> allConfigs = new ArrayList<Map<String,
Map<String, String>>>();
     allConfigs.add(clusterTopology.getConfiguration().getFullProperties());
     for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) {
@@ -304,6 +308,10 @@ public class BlueprintConfigurationProcessor {
       updaters = addYarnResourceManagerHAUpdaters(updaters);
     }
 
+    if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) {
+      updaters = addOozieServerHAUpdaters(updaters);
+    }
+
     return updaters;
   }
 
@@ -359,6 +367,33 @@ public class BlueprintConfigurationProcessor {
     return highAvailabilityUpdaters;
   }
 
+
+  /**
+   * Creates a Collection of PropertyUpdater maps that include the OozieServer HA properties,
and
+   *   adds these to the list of updaters used to process the cluster configuration.
+
+   *   This new Collection includes the statically-defined updaters,
+   *   in addition to the HA-related updaters.
+   *
+   * @param updaters a Collection of updater maps to be included in the list of updaters
for
+   *                   this cluster config update
+   * @return A Collection of PropertyUpdater maps to handle the cluster config update
+   */
+  private Collection<Map<String, Map<String, PropertyUpdater>>> addOozieServerHAUpdaters(Collection<Map<String,
Map<String, PropertyUpdater>>> updaters) {
+    Collection<Map<String, Map<String, PropertyUpdater>>> highAvailabilityUpdaters
=
+      new LinkedList<Map<String, Map<String, PropertyUpdater>>>();
+
+    // always add the statically-defined list of updaters to the list to use
+    // in processing cluster configuration
+    highAvailabilityUpdaters.addAll(updaters);
+
+    // add the updaters for the Oozie HA properties not defined in stack, but
+    // required to be present/updated in oozie-site
+    highAvailabilityUpdaters.add(createMapOfOozieServerHAUpdaters());
+
+    return highAvailabilityUpdaters;
+  }
+
   /**
    * Performs export update for the set of properties that do not
    * require update during cluster setup, but should be removed
@@ -418,6 +453,20 @@ public class BlueprintConfigurationProcessor {
     }
   }
 
+  /**
+   * Perform export update processing for HA configuration for Oozie servers.  The properties
used
+   * in Oozie HA are not defined in the stack, but need to be added at runtime during an
HA
+   * deployment in order to support exporting/redeploying clusters with Oozie HA config.
+   *
+   */
+  public void doOozieServerHAUpdate() {
+    Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = createMapOfOozieServerHAUpdaters();
+
+    if (highAvailabilityUpdaters.get("oozie-site").size() > 0) {
+      doMultiHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration().getFullProperties());
+    }
+  }
+
 
   /**
    * Creates map of PropertyUpdater instances that are associated with
@@ -481,6 +530,27 @@ public class BlueprintConfigurationProcessor {
   }
 
   /**
+   * Creates map of PropertyUpdater instances that are associated with
+   *   Oozie Server High Availability (HA).
+   *
+   * @return a Map of registered PropertyUpdaters for handling HA properties in oozie-site
+   */
+  private Map<String, Map<String, PropertyUpdater>> createMapOfOozieServerHAUpdaters()
{
+    Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = new
HashMap<String, Map<String, PropertyUpdater>>();
+    Map<String, PropertyUpdater> oozieSiteUpdatersForAvailability = new HashMap<String,
PropertyUpdater>();
+    highAvailabilityUpdaters.put("oozie-site", oozieSiteUpdatersForAvailability);
+
+    // register a multi-host property updater for this Oozie property.
+    // this property is not defined in the stacks, since HA is not supported yet
+    // by the stack definition syntax.  This property should only be considered in
+    // an Oozie HA cluster.
+    oozieSiteUpdatersForAvailability.put("oozie.zookeeper.connection.string", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
+
+    return highAvailabilityUpdaters;
+
+  }
+
+  /**
    * Static convenience function to determine if Oozie HA is enabled
    * @param configProperties configuration properties for this cluster
    * @return true if Oozie HA is enabled

http://git-wip-us.apache.org/repos/asf/ambari/blob/832f7e1e/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml
b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml
index 463abc3..a78c93a 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml
@@ -30,7 +30,7 @@
           <name>OOZIE_SERVER</name>
           <displayName>Oozie Server</displayName>
           <category>MASTER</category>
-          <cardinality>1</cardinality>
+          <cardinality>1+</cardinality>
           <versionAdvertised>true</versionAdvertised>
           <dependencies>
             <dependency>

http://git-wip-us.apache.org/repos/asf/ambari/blob/832f7e1e/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 6849b45..595cc4a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -2223,6 +2223,77 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
+  public void testOozieHAEnabledExport() throws Exception {
+    final String expectedHostName = "c6401.apache.ambari.org";
+    final String expectedHostNameTwo = "c6402.ambari.apache.org";
+    final String expectedExternalHost = "c6408.ambari.apache.org";
+    final String expectedHostGroupName = "host_group_1";
+    final String expectedHostGroupNameTwo = "host_group_2";
+    final String expectedPortNum = "80000";
+
+    Map<String, Map<String, String>> configProperties = new HashMap<String,
Map<String, String>>();
+    Map<String, String> oozieSiteProperties = new HashMap<String, String>();
+    Map<String, String> oozieEnvProperties = new HashMap<String, String>();
+    Map<String, String> coreSiteProperties = new HashMap<String, String>();
+
+    configProperties.put("oozie-site", oozieSiteProperties);
+    configProperties.put("oozie-env", oozieEnvProperties);
+    configProperties.put("hive-env", oozieEnvProperties);
+    configProperties.put("core-site", coreSiteProperties);
+
+
+    oozieSiteProperties.put("oozie.base.url", expectedHostName + ":" + expectedPortNum);
+    oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName);
+    oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName);
+    oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + expectedExternalHost
+ "/ooziedb");
+
+    // simulate the Oozie HA configuration
+    oozieSiteProperties.put("oozie.services.ext",
+      "org.apache.oozie.service.ZKLocksService,org.apache.oozie.service.ZKXLogStreamingService,org.apache.oozie.service.ZKJobsConcurrencyService,org.apache.oozie.service.ZKUUIDService");
+    oozieSiteProperties.put("oozie.zookeeper.connection.string", createHostAddress(expectedHostName,
"2181") + "," + createHostAddress(expectedHostNameTwo, "2181"));
+
+
+    oozieEnvProperties.put("oozie_hostname", expectedHostName);
+    oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost);
+
+    coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo);
+
+    Configuration clusterConfig = new Configuration(configProperties, Collections.<String,
Map<String, Map<String, String>>>emptyMap());
+    Collection<String> hgComponents = new HashSet<String>();
+    hgComponents.add("OOZIE_SERVER");
+    hgComponents.add("ZOOKEEPER_SERVER");
+    TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName));
+
+    Collection<String> hgComponents2 = new HashSet<String>();
+    hgComponents2.add("OOZIE_SERVER");
+    hgComponents2.add("ZOOKEEPER_SERVER");
+    TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo));
+
+    Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+    hostGroups.add(group1);
+    hostGroups.add(group2);
+
+    expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes();
+
+    ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+    BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
+    updater.doUpdateForBlueprintExport();
+
+    assertEquals("oozie property not updated correctly",
+      createExportedHostName(expectedHostGroupName, expectedPortNum), oozieSiteProperties.get("oozie.base.url"));
+    assertEquals("oozie property not updated correctly",
+      createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.authentication.kerberos.principal"));
+    assertEquals("oozie property not updated correctly",
+      createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal"));
+    assertEquals("oozie property not updated correctly",
+      createExportedHostName(expectedHostGroupName), oozieEnvProperties.get("oozie_hostname"));
+    assertEquals("oozie property not updated correctly",
+      createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo),
coreSiteProperties.get("hadoop.proxyuser.oozie.hosts"));
+    assertEquals("oozie property not updated correctly",
+      createExportedAddress("2181", expectedHostGroupName) + "," + createExportedAddress("2181",
expectedHostGroupNameTwo), oozieSiteProperties.get("oozie.zookeeper.connection.string"));
+  }
+
+  @Test
   public void testYarnHighAvailabilityConfigClusterUpdateSpecifyingHostNamesDirectly() throws
Exception {
     final String expectedHostName = "c6401.apache.ambari.org";
     final String expectedPortNum = "808080";


Mime
View raw message