ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject ambari git commit: AMBARI-14177. Multiple Ranger KMS server cause exception with Blueprint provisioningn (Oliver Szabo via rlevas)
Date Fri, 04 Dec 2015 18:44:37 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 8b9d23ba9 -> cf976c922


AMBARI-14177. Multiple Ranger KMS server cause exception with Blueprint provisioningn (Oliver
Szabo via rlevas)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cf976c92
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cf976c92
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cf976c92

Branch: refs/heads/trunk
Commit: cf976c922ac52d0273187a6cbb9e806a6491da7f
Parents: 8b9d23b
Author: Oliver Szabo <oszabo@hortonworks.com>
Authored: Fri Dec 4 13:44:24 2015 -0500
Committer: Robert Levas <rlevas@hortonworks.com>
Committed: Fri Dec 4 13:44:29 2015 -0500

----------------------------------------------------------------------
 .../BlueprintConfigurationProcessor.java        | 24 +++++++++-------
 .../stacks/HDP/2.2/services/stack_advisor.py    |  2 +-
 .../BlueprintConfigurationProcessorTest.java    | 29 ++++++++++++--------
 .../stacks/2.2/common/test_stack_advisor.py     |  8 +++---
 4 files changed, 36 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/cf976c92/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 462bddc..467889a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -1591,6 +1591,10 @@ public class BlueprintConfigurationProcessor {
         return origValue;
       }
 
+      if (origValue.contains("localhost") && topology.getHostGroupsForComponent(component).size()
== 1) {
+        return origValue.replace("localhost", topology.getHostAssignmentsForComponent(component).iterator().next());
+      }
+
       String prefix = null;
       Collection<String> hostStrings = getHostStrings(origValue, topology);
       if (hostStrings.isEmpty()) {
@@ -1631,12 +1635,12 @@ public class BlueprintConfigurationProcessor {
         }
 
         // parse out suffix if one exists
-        int indexOfEnd = -1;
-        while (matcher.find()) {
+        int indexOfEnd;
+        do {
           indexOfEnd = matcher.end();
-        }
+        } while (matcher.find());
 
-        if ((indexOfEnd > -1) && (indexOfEnd < (origValue.length() - 1))) {
+        if (indexOfEnd < (origValue.length() - 1)) {
           suffix = origValue.substring(indexOfEnd);
         }
       }
@@ -2039,9 +2043,9 @@ public class BlueprintConfigurationProcessor {
     Map<String, PropertyUpdater> multiYarnSiteMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> multiOozieSiteMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> multiAccumuloSiteMap = new HashMap<String, PropertyUpdater>();
+    Map<String, PropertyUpdater> multiRangerKmsSiteMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> dbHiveSiteMap = new HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> rangerAdminPropsMap = new HashMap<String, PropertyUpdater>();
-    Map<String, PropertyUpdater> rangerKmsSitePropsMap = new HashMap<String, PropertyUpdater>();
 
 
 
@@ -2060,7 +2064,6 @@ public class BlueprintConfigurationProcessor {
     singleHostTopologyUpdaters.put("kafka-broker", kafkaBrokerMap);
     singleHostTopologyUpdaters.put("application-properties", atlasPropsMap);
     singleHostTopologyUpdaters.put("admin-properties", rangerAdminPropsMap);
-    singleHostTopologyUpdaters.put("kms-site", rangerKmsSitePropsMap);
 
 
     mPropertyUpdaters.put("hadoop-env", hadoopEnvMap);
@@ -2079,6 +2082,7 @@ public class BlueprintConfigurationProcessor {
     multiHostTopologyUpdaters.put("yarn-site", multiYarnSiteMap);
     multiHostTopologyUpdaters.put("oozie-site", multiOozieSiteMap);
     multiHostTopologyUpdaters.put("accumulo-site", multiAccumuloSiteMap);
+    multiHostTopologyUpdaters.put("kms-site", multiRangerKmsSiteMap);
 
     dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap);
 
@@ -2096,13 +2100,12 @@ public class BlueprintConfigurationProcessor {
     hdfsSiteMap.put("dfs.namenode.http-address", new SingleHostTopologyUpdater("NAMENODE"));
     hdfsSiteMap.put("dfs.namenode.https-address", new SingleHostTopologyUpdater("NAMENODE"));
     hdfsSiteMap.put("dfs.namenode.rpc-address", new SingleHostTopologyUpdater("NAMENODE"));
-    hdfsSiteMap.put("dfs.encryption.key.provider.uri", new OptionalSingleHostTopologyUpdater("RANGER_KMS_SERVER"));
     coreSiteMap.put("fs.defaultFS", new SingleHostTopologyUpdater("NAMENODE"));
-    coreSiteMap.put("hadoop.security.key.provider.path", new OptionalSingleHostTopologyUpdater("RANGER_KMS_SERVER"));
     hbaseSiteMap.put("hbase.rootdir", new SingleHostTopologyUpdater("NAMENODE"));
     accumuloSiteMap.put("instance.volumes", new SingleHostTopologyUpdater("NAMENODE"));
     // HDFS shared.edits JournalNode Quorum URL uses semi-colons as separators
     multiHdfsSiteMap.put("dfs.namenode.shared.edits.dir", new MultipleHostTopologyUpdater("JOURNALNODE",
';', false));
+    multiHdfsSiteMap.put("dfs.encryption.key.provider.uri", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER",
';', false));
 
     // SECONDARY_NAMENODE
     hdfsSiteMap.put("dfs.secondary.http.address", new SingleHostTopologyUpdater("SECONDARY_NAMENODE"));
@@ -2141,6 +2144,7 @@ public class BlueprintConfigurationProcessor {
     multiCoreSiteMap.put("hadoop.proxyuser.HTTP.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
     multiCoreSiteMap.put("hadoop.proxyuser.hcat.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
     multiCoreSiteMap.put("hadoop.proxyuser.yarn.hosts", new MultipleHostTopologyUpdater("RESOURCEMANAGER"));
+    multiCoreSiteMap.put("hadoop.security.key.provider.path", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER",
';', false));
     multiWebhcatSiteMap.put("templeton.hive.properties", new TempletonHivePropertyUpdater());
     multiWebhcatSiteMap.put("templeton.kerberos.principal", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
     hiveEnvMap.put("hive_hostname", new SingleHostTopologyUpdater("HIVE_SERVER"));
@@ -2306,8 +2310,8 @@ public class BlueprintConfigurationProcessor {
     rangerAdminPropsMap.put("policymgr_external_url", new SingleHostTopologyUpdater("RANGER_ADMIN"));
 
     // RANGER KMS
-    rangerKmsSitePropsMap.put("hadoop.kms.key.provider.uri", new SingleHostTopologyUpdater("RANGER_KMS_SERVER"));
-
+    multiRangerKmsSiteMap.put("hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string",
+      new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
     // Required due to AMBARI-4933.  These no longer seem to be required as the default values
in the stack
     // are now correct but are left here in case an existing blueprint still contains an
old value.
     hadoopEnvMap.put("namenode_heapsize", new MPropertyUpdater());

http://git-wip-us.apache.org/repos/asf/ambari/blob/cf976c92/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index a33be1e..12f85b8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -227,7 +227,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       rangerKMSServerHostsArray = []
       for rangeKMSServerHost in rangerKMSServerHosts:
         rangerKMSServerHostsArray.append(rangeKMSServerHost["Hosts"]["host_name"])
-      keyserverHostsString = ",".join(rangerKMSServerHostsArray)
+      keyserverHostsString = ";".join(rangerKMSServerHostsArray)
       if "kms-env" in services["configurations"] and "kms_port" in services["configurations"]["kms-env"]["properties"]:
         keyserverPortString = services["configurations"]["kms-env"]["properties"]["kms_port"]
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/cf976c92/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 648dabc..b4b50cf 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -5721,8 +5721,9 @@ public class BlueprintConfigurationProcessorTest {
     Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
     Map<String, String> kmsSiteProperties = new HashMap<String, String>();
     properties.put(kmsSiteConfigType, kmsSiteProperties);
-    kmsSiteProperties.put("hadoop.kms.key.provider.uri", "dbks://http@%HOSTGROUP::group1%:9292/kms");
-
+    kmsSiteProperties.put("hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string",
+      createHostAddress("%HOSTGROUP::group1%", "2181") + "," + createHostAddress("%HOSTGROUP::group2%",
"2181"));
+    kmsSiteProperties.put("hadoop.kms.key.provider.uri", "dbks://http@localhost:9292/kms");
 
 
     Map<String, Map<String, String>> parentProperties = new HashMap<String,
Map<String, String>>();
@@ -5736,9 +5737,11 @@ public class BlueprintConfigurationProcessorTest {
     kmsServerComponents.add("RANGER_KMS_SERVER");
 
     TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1"));
+    TestHostGroup group2 = new TestHostGroup("group2", kmsServerComponents, Collections.singleton("host2"));
 
-
-    Collection<TestHostGroup> hostGroups = Collections.singleton(group1);
+    Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+    hostGroups.add(group1);
+    hostGroups.add(group2);
 
     ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
     BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
@@ -5747,7 +5750,8 @@ public class BlueprintConfigurationProcessorTest {
     configProcessor.doUpdateForClusterCreate();
 
     // Then
-    assertEquals("dbks://http@host1:9292/kms", clusterConfig.getPropertyValue(kmsSiteConfigType,
"hadoop.kms.key.provider.uri"));
+    assertEquals("host1:2181,host2:2181", clusterConfig.getPropertyValue(kmsSiteConfigType,
"hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string"));
+    assertEquals("dbks://http@localhost:9292/kms", clusterConfig.getPropertyValue(kmsSiteConfigType,
"hadoop.kms.key.provider.uri"));
   }
 
 
@@ -5760,8 +5764,8 @@ public class BlueprintConfigurationProcessorTest {
     Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
     Map<String, String> kmsSiteProperties = new HashMap<String, String>();
     properties.put(kmsSiteConfigType, kmsSiteProperties);
-    kmsSiteProperties.put("hadoop.kms.key.provider.uri", "dbks://http@localhost:9292/kms");
-
+    kmsSiteProperties.put("hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string",
+      createHostAddress("%HOSTGROUP::group1%", "2181"));
 
 
     Map<String, Map<String, String>> parentProperties = new HashMap<String,
Map<String, String>>();
@@ -5786,7 +5790,8 @@ public class BlueprintConfigurationProcessorTest {
     configProcessor.doUpdateForClusterCreate();
 
     // Then
-    assertEquals("dbks://http@host1:9292/kms", clusterConfig.getPropertyValue(kmsSiteConfigType,
"hadoop.kms.key.provider.uri"));
+    assertEquals("host1:2181", clusterConfig.getPropertyValue(kmsSiteConfigType,
+      "hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string"));
   }
 
 
@@ -5798,7 +5803,7 @@ public class BlueprintConfigurationProcessorTest {
     Map<String, String> configProperties = new HashMap<String, String>();
 
     properties.put(configType, configProperties);
-    configProperties.put("dfs.encryption.key.provider.uri", "kms://http@%HOSTGROUP::group1%:9292/kms");
+    configProperties.put("dfs.encryption.key.provider.uri", "kms://http@%HOSTGROUP::group1%;%HOSTGROUP::group2%:9292/kms");
 
 
     Map<String, Map<String, String>> parentProperties = new HashMap<String,
Map<String, String>>();
@@ -5830,7 +5835,7 @@ public class BlueprintConfigurationProcessorTest {
     configProcessor.doUpdateForClusterCreate();
 
     // Then
-    assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType,
"dfs.encryption.key.provider.uri"));
+    assertEquals("kms://http@host1;host2:9292/kms", clusterConfig.getPropertyValue(configType,
"dfs.encryption.key.provider.uri"));
   }
 
 
@@ -5933,7 +5938,7 @@ public class BlueprintConfigurationProcessorTest {
     Map<String, String> configProperties = new HashMap<String, String>();
 
     properties.put(configType, configProperties);
-    configProperties.put("hadoop.security.key.provider.path", "kms://http@%HOSTGROUP::group1%:9292/kms");
+    configProperties.put("hadoop.security.key.provider.path", "kms://http@%HOSTGROUP::group1%;%HOSTGROUP::group2%:9292/kms");
 
 
     Map<String, Map<String, String>> parentProperties = new HashMap<String,
Map<String, String>>();
@@ -5965,7 +5970,7 @@ public class BlueprintConfigurationProcessorTest {
     configProcessor.doUpdateForClusterCreate();
 
     // Then
-    assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType,
"hadoop.security.key.provider.path"));
+    assertEquals("kms://http@host1;host2:9292/kms", clusterConfig.getPropertyValue(configType,
"hadoop.security.key.provider.path"));
   }
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/cf976c92/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index f5c0436..1ee3e7a 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -3013,22 +3013,22 @@ class TestHDP22StackAdvisor(TestCase):
     # Test 6 - Multiple RANGER_KMS_SERVERs
     services["services"][len(services["services"])-1]["components"][0]["StackServiceComponents"]["hostnames"].append("host2")
     self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services,
hosts)
-    self.assertEqual("kms://http@host1,host2:9292/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+    self.assertEqual("kms://http@host1;host2:9292/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
 
     # Test 6 - Multiple RANGER_KMS_SERVERs and custom port
     configurations["kms-env"] = {"properties": {"kms_port": "1111"}}
     self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services,
hosts)
-    self.assertEqual("kms://http@host1,host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+    self.assertEqual("kms://http@host1;host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
 
     # Test 7 - Override by API caller
     configurations["hadoop-env"] = {"properties": {"keyserver_host": "myhost1", "keyserver_port":
"2222"}}
     self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services,
hosts)
-    self.assertEqual("kms://http@host1,host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+    self.assertEqual("kms://http@host1;host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
 
     # Test - 'https' in KMS URL
     configurations["ranger-kms-site"] = {"properties": {"ranger.service.https.attrib.ssl.enabled":
"true"}}
     self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services,
hosts)
-    self.assertEqual("kms://https@host1,host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+    self.assertEqual("kms://https@host1;host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
 
     # Test 8 - Dynamic maximum for 'dfs.namenode.handler.count'
     hosts['items'][1]['Hosts']['cpu_count'] = 9


Mime
View raw message