ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rnettle...@apache.org
Subject ambari git commit: AMBARI-14745. Ambari server throws error when unused configs are present in blueprint. (Oliver Szabo via rnettleton)
Date Fri, 05 Feb 2016 16:32:58 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 d8920559e -> 7ea0744a5


AMBARI-14745. Ambari server throws error when unused configs are present in blueprint. (Oliver
Szabo via rnettleton)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7ea0744a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7ea0744a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7ea0744a

Branch: refs/heads/branch-2.2
Commit: 7ea0744a5a701b5e593895af66bd39586bc255f3
Parents: d892055
Author: Bob Nettleton <rnettleton@hortonworks.com>
Authored: Fri Feb 5 11:32:13 2016 -0500
Committer: Bob Nettleton <rnettleton@hortonworks.com>
Committed: Fri Feb 5 11:32:13 2016 -0500

----------------------------------------------------------------------
 .../topology/ClusterConfigurationRequest.java   | 28 ++++++++
 .../ambari/server/topology/Configuration.java   | 15 ++++
 .../ClusterConfigurationRequestTest.java        | 75 +++++++++++++++++---
 .../ClusterInstallWithoutStartTest.java         |  2 +
 .../server/topology/ConfigurationTest.java      | 14 ++++
 .../server/topology/TopologyManagerTest.java    |  2 +
 6 files changed, 128 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/7ea0744a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 464aee7..a995a3b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -30,6 +30,7 @@ import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurati
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.SecurityType;
 import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -74,11 +75,38 @@ public class ClusterConfigurationRequest {
     // set initial configuration (not topology resolved)
     this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology);
     this.stackAdvisorBlueprintProcessor = stackAdvisorBlueprintProcessor;
+    removeOrphanConfigTypes(clusterTopology);
     if (setInitial) {
       setConfigurationsOnCluster(clusterTopology, TopologyManager.INITIAL_CONFIG_TAG, Collections.<String>emptySet());
     }
   }
 
+  /**
+   * Remove config-types, if there is no any services related to them (except cluster-env
and global).
+   */
+  private void removeOrphanConfigTypes(ClusterTopology clusterTopology) {
+    Configuration configuration = clusterTopology.getConfiguration();
+    Collection<String> configTypes = configuration.getAllConfigTypes();
+    for (String configType : configTypes) {
+      if (!configType.equals("cluster-env") && !configType.equals("global")) {
+        String service = clusterTopology.getBlueprint().getStack().getServiceForConfigType(configType);
+        if (!clusterTopology.getBlueprint().getServices().contains(service)) {
+          configuration.removeConfigType(configType);
+          LOG.info("Not found any service for config type '{}'. It will be removed from configuration.",
configType);
+          Map<String, HostGroupInfo> hostGroupInfoMap = clusterTopology.getHostGroupInfo();
+          if (MapUtils.isNotEmpty(hostGroupInfoMap)) {
+            for (Map.Entry<String, HostGroupInfo> hostGroupInfo : hostGroupInfoMap.entrySet())
{
+              if (hostGroupInfo.getValue().getConfiguration() != null) {
+                hostGroupInfo.getValue().getConfiguration().removeConfigType(configType);
+                LOG.info("Not found any service for config type '{}'. It will be removed
from host group scoped configuration.", configType);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
   public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology topology,
boolean setInitial, StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor, boolean
configureSecurity) {
     this(ambariContext, topology, setInitial, stackAdvisorBlueprintProcessor);
     this.configureSecurity = configureSecurity;

http://git-wip-us.apache.org/repos/asf/ambari/blob/7ea0744a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index 108ff74..79281b4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -344,4 +344,19 @@ public class Configuration {
   public void setParentConfiguration(Configuration parent) {
     parentConfiguration = parent;
   }
+
+  /**
+   * Remove all occurrences of a config type
+   */
+  public void removeConfigType(String configType) {
+    if (properties != null && properties.containsKey(configType)) {
+      properties.remove(configType);
+    }
+    if (attributes != null && attributes.containsKey(configType)) {
+      attributes.remove(configType);
+    }
+    if (parentConfiguration != null) {
+      parentConfiguration.removeConfigType(configType);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/7ea0744a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
index 5967a64..3ed8ed5 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
@@ -28,6 +28,17 @@ import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.Maps;
+import org.easymock.EasyMock;
 import org.easymock.Capture;
 import org.easymock.CaptureType;
 import org.easymock.EasyMockRule;
@@ -40,14 +51,6 @@ import org.powermock.api.easymock.PowerMock;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 import static org.easymock.EasyMock.anyBoolean;
 import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.anyString;
@@ -55,6 +58,8 @@ import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.newCapture;
 import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.easymock.EasyMock.capture;
 import static org.junit.Assert.assertEquals;
 
@@ -183,6 +188,7 @@ public class ClusterConfigurationRequestTest {
     expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes();
 
     expect(blueprint.getStack()).andReturn(stack).anyTimes();
+    expect(stack.getServiceForConfigType(anyString())).andReturn("KERBEROS").anyTimes();
     expect(stack.getAllConfigurationTypes(anyString())).andReturn(Collections.<String>singletonList("testConfigType")
     ).anyTimes();
     expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.<String>emptySet()).anyTimes();
@@ -307,4 +313,57 @@ public class ClusterConfigurationRequestTest {
 
   }
 
+  @Test
+  public void testProcessClusterConfigRequestRemoveUnusedConfigTypes() {
+    // GIVEN
+    Configuration configuration = createConfigurations();
+    Set<String> services = new HashSet<String>();
+    services.add("HDFS");
+    services.add("RANGER");
+    Map<String, HostGroupInfo> hostGroupInfoMap = Maps.newHashMap();
+    HostGroupInfo hg1 = new HostGroupInfo("hg1");
+    hg1.setConfiguration(createConfigurations());
+    hostGroupInfoMap.put("hg1", hg1);
+
+    expect(topology.getConfiguration()).andReturn(configuration).anyTimes();
+    expect(topology.getBlueprint()).andReturn(blueprint).anyTimes();
+    expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap);
+    expect(blueprint.getStack()).andReturn(stack).anyTimes();
+    expect(blueprint.getServices()).andReturn(services).anyTimes();
+    expect(stack.getServiceForConfigType("hdfs-site")).andReturn("HDFS").anyTimes();
+    expect(stack.getServiceForConfigType("admin-properties")).andReturn("RANGER").anyTimes();
+    expect(stack.getServiceForConfigType("yarn-site")).andReturn("YARN").anyTimes();
+
+    EasyMock.replay(stack, blueprint, topology);
+    // WHEN
+    new ClusterConfigurationRequest(ambariContext, topology, false, stackAdvisorBlueprintProcessor);
+    // THEN
+    assertFalse(configuration.getFullProperties().containsKey("yarn-site"));
+    assertFalse(configuration.getFullAttributes().containsKey("yarn-site"));
+    assertTrue(configuration.getFullAttributes().containsKey("hdfs-site"));
+    assertTrue(configuration.getFullProperties().containsKey("cluster-env"));
+    assertTrue(configuration.getFullProperties().containsKey("global"));
+    assertFalse(hg1.getConfiguration().getFullAttributes().containsKey("yarn-site"));
+    verify(stack, blueprint, topology);
+  }
+
+  private Configuration createConfigurations() {
+    Map<String, Map<String, String>> firstLevelConfig = Maps.newHashMap();
+    firstLevelConfig.put("hdfs-site", new HashMap<String, String>());
+    firstLevelConfig.put("yarn-site", new HashMap<String, String>());
+    firstLevelConfig.put("cluster-env", new HashMap<String, String>());
+    firstLevelConfig.put("global", new HashMap<String, String>());
+
+    Map<String, Map<String, Map<String, String>>> firstLevelAttributes
= Maps.newHashMap();
+    firstLevelAttributes.put("hdfs-site", new HashMap<String, Map<String, String>>());
+
+    Map<String, Map<String, String>> secondLevelConfig = Maps.newHashMap();
+    secondLevelConfig.put("admin-properties", new HashMap<String, String>());
+    Map<String, Map<String, Map<String, String>>> secondLevelAttributes
= Maps.newHashMap();
+    secondLevelAttributes.put("admin-properties", new HashMap<String, Map<String, String>>());
+    secondLevelAttributes.put("yarn-site", new HashMap<String, Map<String, String>>());
+
+    Configuration secondLevelConf = new Configuration(secondLevelConfig, secondLevelAttributes);
+    return new Configuration(firstLevelConfig, firstLevelAttributes, secondLevelConf);
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/7ea0744a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
index dd66b1b..156580a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
@@ -251,6 +251,8 @@ public class ClusterInstallWithoutStartTest {
     expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes();
     expect(stack.getName()).andReturn(STACK_NAME).anyTimes();
     expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();
+    expect(stack.getServiceForConfigType("service1-site")).andReturn("service1");
+    expect(stack.getServiceForConfigType("service2-site")).andReturn("service2");
     expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.<String>emptySet()).anyTimes();
     expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.<String>emptySet()).anyTimes();
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/7ea0744a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
index e971e03..c4d70b5 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
@@ -346,6 +346,20 @@ public class ConfigurationTest {
     assertNull(configuration.getPropertyValue("type1", "XXXXX"));
   }
 
+  @Test
+  public void testRemoveConfigTypes() {
+    Configuration configuration = createConfigurationWithParents_PropsOnly();
+    configuration.removeConfigType("type1");
+    assertNull(configuration.getProperties().get("type1"));
+  }
+
+  @Test
+  public void testRemoveConfigTypesForAttributes() {
+    Configuration configuration = createConfigurationWithParents_PropsOnly();
+    configuration.removeConfigType("type1");
+    assertNull(configuration.getAttributes().get("type1"));
+  }
+
   private Configuration createConfigurationWithParents_PropsOnly() {
     // parents parent config properties
     Map<String, Map<String, String>> parentParentProperties = new HashMap<String,
Map<String, String>>();

http://git-wip-us.apache.org/repos/asf/ambari/blob/7ea0744a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
index 7810f92..69c1935 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
@@ -228,6 +228,8 @@ public class TopologyManagerTest {
     expect(stack.getComponents()).andReturn(serviceComponents).anyTimes();
     expect(stack.getComponents("service1")).andReturn(serviceComponents.get("service1")).anyTimes();
     expect(stack.getComponents("service2")).andReturn(serviceComponents.get("service2")).anyTimes();
+    expect(stack.getServiceForConfigType("service1-site")).andReturn("service1");
+    expect(stack.getServiceForConfigType("service2-site")).andReturn("service2");
     expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes();
     expect(stack.getName()).andReturn(STACK_NAME).anyTimes();
     expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();


Mime
View raw message