ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject ambari git commit: AMBARI-17802. Blueprint deployment configures "org.apache.atlas.hive.hook.HiveHook" twice for "hive.exec.post.hooks" config (alejandro)
Date Wed, 20 Jul 2016 18:41:07 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 c411be395 -> 8cdd83594


AMBARI-17802. Blueprint deployment configures "org.apache.atlas.hive.hook.HiveHook" twice
for "hive.exec.post.hooks" config (alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8cdd8359
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8cdd8359
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8cdd8359

Branch: refs/heads/branch-2.4
Commit: 8cdd83594ba2880d745b472ba6667a9805afd53e
Parents: c411be3
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Tue Jul 19 19:00:53 2016 -0700
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Wed Jul 20 11:38:28 2016 -0700

----------------------------------------------------------------------
 .../BlueprintConfigurationProcessor.java        |  32 +++-
 .../stacks/HDP/2.3/services/stack_advisor.py    |  39 ++---
 .../BlueprintConfigurationProcessorTest.java    | 147 ++++++++++++-------
 3 files changed, 142 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8cdd8359/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index a295bf7..23c9edc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -2440,16 +2440,34 @@ public class BlueprintConfigurationProcessor {
                                            String origValue,
                                            Map<String, Map<String, String>> properties,
                                            ClusterTopology topology) {
+        String atlasHookClass = "org.apache.atlas.hive.hook.HiveHook";
+        String[] hiveHooks = origValue.split(",");
 
-        if (topology.getBlueprint().getServices().contains("ATLAS")) {
-          String hiveHookClass = "org.apache.atlas.hive.hook.HiveHook";
-          if (origValue == null || origValue.isEmpty()) {
-            return hiveHookClass;
-          } else {
-            return String.format("%s,%s", origValue, hiveHookClass);
+        List<String> hiveHooksClean = new ArrayList<String>();
+        for(String hiveHook : hiveHooks) {
+          if (!StringUtils.isBlank(hiveHook.trim())) {
+            hiveHooksClean.add(hiveHook.trim());
+          }
+        }
+
+        boolean isAtlasInCluster = topology.getBlueprint().getServices().contains("ATLAS");
+
+        // Append atlas hook if not already present.
+        if (isAtlasInCluster) {
+          if (!hiveHooksClean.contains(atlasHookClass)) {
+            hiveHooksClean.add(atlasHookClass);
           }
         } else {
-          return origValue;
+          // Remove the atlas hook since Atlas service is not present.
+          while (hiveHooksClean.contains(atlasHookClass)) {
+            hiveHooksClean.remove(atlasHookClass);
+          }
+        }
+
+        if (!hiveHooksClean.isEmpty()) {
+          return StringUtils.join(hiveHooksClean, ",");
+        } else {
+          return " ";
         }
       }
     });

http://git-wip-us.apache.org/repos/asf/ambari/blob/8cdd8359/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index e1bedb7..19753e5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -232,34 +232,35 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     else:
       putHiveSitePropertyAttribute('datanucleus.rdbms.datastoreAdapterClassName', 'delete',
'true')
 
-    # atlas
+    # Atlas
     hooks_property = "hive.exec.post.hooks"
+    atlas_hook_class = "org.apache.atlas.hive.hook.HiveHook"
     if hooks_property in configurations["hive-site"]["properties"]:
       hooks_value = configurations["hive-site"]["properties"][hooks_property]
     else:
-      hooks_value = " "
+      hooks_value = ""
 
-    include_atlas = "ATLAS" in servicesList
-    atlas_hook_class = "org.apache.atlas.hive.hook.HiveHook"
-    if include_atlas and atlas_hook_class not in hooks_value:
-      if hooks_value == " ":
-        hooks_value = atlas_hook_class
-      else:
-        hooks_value = hooks_value + "," + atlas_hook_class
-    if not include_atlas and atlas_hook_class in hooks_value:
-      hooks_classes = []
-      for hook_class in hooks_value.split(","):
-        if hook_class != atlas_hook_class and hook_class != " ":
-          hooks_classes.append(hook_class)
-      if hooks_classes:
-        hooks_value = ",".join(hooks_classes)
-      else:
-        hooks_value = " "
+
+    hive_hooks = [x.strip() for x in hooks_value.split(",")]
+    hive_hooks = [x for x in hive_hooks if x != ""]
+    is_atlas_present_in_cluster = "ATLAS" in servicesList
+
+    if is_atlas_present_in_cluster:
+      # Append atlas hook if not already present.
+      is_atlas_hook_in_config = atlas_hook_class in hive_hooks
+      if not is_atlas_hook_in_config:
+        hive_hooks.append(atlas_hook_class)
+    else:
+      # Remove the atlas hook since Atlas service is not present.
+      hive_hooks = [x for x in hive_hooks if x != atlas_hook_class]
+
+    # Convert hive_hooks back to a csv, unless there are 0 elements, which should be " "
+    hooks_value = " " if len(hive_hooks) == 0 else ",".join(hive_hooks)
     putHiveSiteProperty(hooks_property, hooks_value)
 
     # This is no longer used in HDP 2.5, but still needed in HDP 2.3 and 2.4
     atlas_server_host_info = self.getHostWithComponent("ATLAS", "ATLAS_SERVER", services,
hosts)
-    if include_atlas and atlas_server_host_info:
+    if is_atlas_present_in_cluster and atlas_server_host_info:
       atlas_rest_host = atlas_server_host_info['Hosts']['host_name']
       scheme = "http"
       metadata_port = "21000"

http://git-wip-us.apache.org/repos/asf/ambari/blob/8cdd8359/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 268bc29..7aa4a4e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -5828,29 +5828,75 @@ public class BlueprintConfigurationProcessorTest {
   
   @Test
   public void testAtlasHiveProperties() throws Exception {
+    Map<String, Map<String, String>> properties = getAtlasHivePropertiesForTestCase();
+    validateAtlasHivePropertiesForTestCase(properties);
+  }
+
+  /**
+   * If the Hive Exec Hooks property doesn't contain the Atlas Hook, then add it.
+   * @throws Exception
+   */
+  @Test
+  public void testAtlasHivePropertiesWithHiveHookSpace() throws Exception {
+    Map<String, Map<String, String>> properties = getAtlasHivePropertiesForTestCase();
+
+    Map<String, String> hiveProperties = properties.get("hive-site");
+    hiveProperties.put("hive.exec.post.hooks", " ");
+    properties.put("hive-site", hiveProperties);
+    validateAtlasHivePropertiesForTestCase(properties);
+  }
+
+  /***
+   * If the Atlas Hook already exists, don't append it.
+   * @throws Exception
+   */
+  @Test
+  public void testAtlasHivePropertiesWithAtlasHookAlreadyExist() throws Exception {
+    Map<String, Map<String, String>> properties = getAtlasHivePropertiesForTestCase();
+
+    Map<String, String> hiveProperties = properties.get("hive-site");
+    hiveProperties.put("hive.exec.post.hooks", "org.apache.atlas.hive.hook.HiveHook");
+    properties.put("hive-site", hiveProperties);
+    validateAtlasHivePropertiesForTestCase(properties);
+  }
+
+  /**
+   * Generate sample collection of properties for some of the test cases.
+   * @return Map of sample properties
+   */
+  private Map<String, Map<String, String>> getAtlasHivePropertiesForTestCase()
{
     Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+
     Map<String, String> atlasProperties = new HashMap<String, String>();
-    properties.put("application-properties", atlasProperties);
     atlasProperties.put("atlas.enableTLS", "false");
     atlasProperties.put("atlas.server.bind.address", "localhost");
     atlasProperties.put("atlas.server.http.port", "21000");
-    Map<String, String> atlasEnv = new HashMap<String, String>();
+    properties.put("application-properties", atlasProperties);
 
+    Map<String, String> atlasEnv = new HashMap<String, String>();
     properties.put("atlas-env", atlasEnv);
+
     Map<String, String> hiveProperties = new HashMap<String, String>();
     hiveProperties.put("hive.exec.post.hooks", "");
     hiveProperties.put("atlas.cluster.name", "primary");
     hiveProperties.put("atlas.rest.address", "http://localhost:21000");
     properties.put("hive-site", hiveProperties);
 
+    return properties;
+  }
 
+  /**
+   * For several test cases, validate that org.apache.atlas.hive.hook.HiveHook has the correct
value.
+   * @param properties Map of properties to validate
+   * @throws Exception
+   */
+  private void validateAtlasHivePropertiesForTestCase(Map<String, Map<String, String>>
properties) throws Exception {
     Map<String, Map<String, String>> parentProperties = new HashMap<String,
Map<String, String>>();
     Configuration parentClusterConfig = new Configuration(parentProperties,
         Collections.<String, Map<String, Map<String, String>>>emptyMap());
     Configuration clusterConfig = new Configuration(properties,
         Collections.<String, Map<String, Map<String, String>>>emptyMap(),
parentClusterConfig);
 
-
     Collection<String> hgComponents1 = new HashSet<String>();
     hgComponents1.add("ATLAS_SERVER");
     hgComponents1.add("HIVE_SERVER");
@@ -5858,8 +5904,8 @@ public class BlueprintConfigurationProcessorTest {
 
     Collection<TestHostGroup> hostGroups = Collections.singletonList(group1);
 
-    ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
-    BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+    ClusterTopology topology1 = createClusterTopology(bp, clusterConfig, hostGroups);
+    BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology1);
 
     configProcessor.doUpdateForClusterCreate();
 
@@ -5870,6 +5916,52 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
+  public void testAtlasHivePropertiesWithHTTPS() throws Exception {
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+
+    Map<String, String> atlasProperties = new HashMap<String, String>();
+    properties.put("application-properties", atlasProperties);
+    // use https
+    atlasProperties.put("atlas.enableTLS", "true");
+    atlasProperties.put("atlas.server.bind.address", "localhost");
+    atlasProperties.put("atlas.server.https.port", "99999");
+    Map<String, String> atlasEnv = new HashMap<String, String>();
+
+    properties.put("atlas-env", atlasEnv);
+    Map<String, String> hiveProperties = new HashMap<String, String>();
+    // default hook registered
+    hiveProperties.put("hive.exec.post.hooks", "foo");
+    // user specified cluster name
+    hiveProperties.put("atlas.cluster.name", "userSpecified");
+    hiveProperties.put("atlas.rest.address", "http://localhost:21000");
+    properties.put("hive-site", hiveProperties);
+
+
+    Map<String, Map<String, String>> parentProperties = new HashMap<String,
Map<String, String>>();
+    Configuration parentClusterConfig = new Configuration(parentProperties,
+        Collections.<String, Map<String, Map<String, String>>>emptyMap());
+    Configuration clusterConfig = new Configuration(properties,
+        Collections.<String, Map<String, Map<String, String>>>emptyMap(),
parentClusterConfig);
+
+
+    Collection<String> hgComponents1 = new HashSet<String>();
+    hgComponents1.add("ATLAS_SERVER");
+    hgComponents1.add("HIVE_SERVER");
+    TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1"));
+
+    Collection<TestHostGroup> hostGroups = Collections.singletonList(group1);
+
+    ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+    BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+    configProcessor.doUpdateForClusterCreate();
+
+    assertEquals("foo,org.apache.atlas.hive.hook.HiveHook", clusterConfig.getPropertyValue("hive-site",
"hive.exec.post.hooks"));
+    assertEquals("userSpecified", clusterConfig.getPropertyValue("hive-site", "atlas.cluster.name"));
+    assertEquals("https://host1:99999", clusterConfig.getPropertyValue("hive-site", "atlas.rest.address"));
+  }
+
+  @Test
   public void testStormAmsPropertiesDefault() throws Exception {
     Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
 
@@ -5996,51 +6088,6 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
-  public void testAtlasHiveProperties2() throws Exception {
-    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
-    Map<String, String> atlasProperties = new HashMap<String, String>();
-    properties.put("application-properties", atlasProperties);
-    // use https
-    atlasProperties.put("atlas.enableTLS", "true");
-    atlasProperties.put("atlas.server.bind.address", "localhost");
-    atlasProperties.put("atlas.server.https.port", "99999");
-    Map<String, String> atlasEnv = new HashMap<String, String>();
-
-    properties.put("atlas-env", atlasEnv);
-    Map<String, String> hiveProperties = new HashMap<String, String>();
-    // default hook registered
-    hiveProperties.put("hive.exec.post.hooks", "foo");
-    // user specified cluster name
-    hiveProperties.put("atlas.cluster.name", "userSpecified");
-    hiveProperties.put("atlas.rest.address", "http://localhost:21000");
-    properties.put("hive-site", hiveProperties);
-
-
-    Map<String, Map<String, String>> parentProperties = new HashMap<String,
Map<String, String>>();
-    Configuration parentClusterConfig = new Configuration(parentProperties,
-        Collections.<String, Map<String, Map<String, String>>>emptyMap());
-    Configuration clusterConfig = new Configuration(properties,
-        Collections.<String, Map<String, Map<String, String>>>emptyMap(),
parentClusterConfig);
-
-
-    Collection<String> hgComponents1 = new HashSet<String>();
-    hgComponents1.add("ATLAS_SERVER");
-    hgComponents1.add("HIVE_SERVER");
-    TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1"));
-
-    Collection<TestHostGroup> hostGroups = Collections.singletonList(group1);
-
-    ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
-    BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
-
-    configProcessor.doUpdateForClusterCreate();
-
-    assertEquals("foo,org.apache.atlas.hive.hook.HiveHook", clusterConfig.getPropertyValue("hive-site",
"hive.exec.post.hooks"));
-    assertEquals("userSpecified", clusterConfig.getPropertyValue("hive-site", "atlas.cluster.name"));
-    assertEquals("https://host1:99999", clusterConfig.getPropertyValue("hive-site", "atlas.rest.address"));
-  }
-
-  @Test
   public void testRecommendConfiguration_applyStackDefaultsOnly() throws Exception {
     // GIVEN
     final String expectedHostName = "c6401.apache.ambari.org";


Mime
View raw message