ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rnettle...@apache.org
Subject ambari git commit: AMBARI-14251. Blueprints fix stale configs and kerberos-env stack defaults are not being applied during cluster deployment. (Sandor Magyari via rnettleton)
Date Tue, 08 Dec 2015 22:52:10 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 7228c76fb -> b1149fa7e


AMBARI-14251. Blueprints fix stale configs and kerberos-env stack defaults are not being applied
during cluster deployment. (Sandor Magyari via rnettleton)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b1149fa7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b1149fa7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b1149fa7

Branch: refs/heads/branch-2.2
Commit: b1149fa7e827c94ab929a25b9b7f9bc6165b8605
Parents: 7228c76
Author: Bob Nettleton <rnettleton@hortonworks.com>
Authored: Tue Dec 8 17:51:23 2015 -0500
Committer: Bob Nettleton <rnettleton@hortonworks.com>
Committed: Tue Dec 8 17:51:23 2015 -0500

----------------------------------------------------------------------
 .../controller/AmbariManagementController.java  |   6 +
 .../AmbariManagementControllerImpl.java         |   5 +
 .../topology/ClusterConfigurationRequest.java   |  38 ++++
 .../ambari/server/topology/TopologyManager.java |  24 ++-
 .../ClusterConfigurationRequestTest.java        | 172 +++++++++++++++++++
 5 files changed, 236 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
index ea7603f..2d11265 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
@@ -772,5 +772,11 @@ public interface AmbariManagementController {
   Set<StackConfigurationDependencyResponse> getStackConfigurationDependencies(Set<StackConfigurationDependencyRequest>
requests) throws AmbariException;
 
   TimelineMetricCacheProvider getTimelineMetricCacheProvider();
+
+  /**
+   * Returns KerberosHelper instance
+   * @return
+   */
+  KerberosHelper getKerberosHelper();
 }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index d0cbd20..3dae467 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -4395,6 +4395,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
     return injector.getInstance(TimelineMetricCacheProvider.class);
   }
 
+  @Override
+  public KerberosHelper getKerberosHelper() {
+    return kerberosHelper;
+  }
+
   /**
    * Queries the CredentialStoreService to gather properties about it.
    * <p/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 1a650ee..6e8b8a3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -26,6 +26,8 @@ import org.apache.ambari.server.controller.internal.BlueprintConfigurationProces
 import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
 import org.apache.ambari.server.controller.internal.ConfigurationTopologyException;
 import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
+import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.SecurityType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -33,6 +35,7 @@ import org.slf4j.LoggerFactory;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -50,6 +53,7 @@ public class ClusterConfigurationRequest {
   private BlueprintConfigurationProcessor configurationProcessor;
   private StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor;
   private Stack stack;
+  private boolean configureSecurity = false;
 
   public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology clusterTopology,
boolean setInitial,
                                      StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor)
{
@@ -65,6 +69,11 @@ public class ClusterConfigurationRequest {
     }
   }
 
+  public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology topology,
boolean setInitial, StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor, boolean
configureSecurity) {
+    this(ambariContext, topology, setInitial, stackAdvisorBlueprintProcessor);
+    this.configureSecurity = configureSecurity;
+  }
+
   // get names of required host groups
   public Collection<String> getRequiredHostGroups() {
     return configurationProcessor.getRequiredHostGroups();
@@ -79,6 +88,11 @@ public class ClusterConfigurationRequest {
       if (!ConfigRecommendationStrategy.NEVER_APPLY.equals(this.clusterTopology.getConfigRecommendationStrategy()))
{
         stackAdvisorBlueprintProcessor.adviseConfiguration(this.clusterTopology);
       }
+
+      if (configureSecurity) {
+        configureKerberos();
+      }
+
       updatedConfigTypes =
         configurationProcessor.doUpdateForClusterCreate();
     } catch (ConfigurationTopologyException e) {
@@ -89,6 +103,30 @@ public class ClusterConfigurationRequest {
     setConfigurationsOnCluster(clusterTopology, TopologyManager.TOPOLOGY_RESOLVED_TAG, updatedConfigTypes);
   }
 
+  private void configureKerberos() throws AmbariException {
+    String clusterName = ambariContext.getClusterName(clusterTopology.getClusterId());
+    Cluster cluster = AmbariContext.getController().getClusters().getCluster(clusterName);
+    Blueprint blueprint = clusterTopology.getBlueprint();
+    Configuration clusterConfiguration = clusterTopology.getConfiguration();
+
+    try {
+      Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper()
+        .getServiceConfigurationUpdates(cluster, clusterConfiguration.getFullProperties(),
+        new HashSet<String>(blueprint.getServices()));
+     for (String configType : updatedConfigs.keySet()) {
+       Map<String, String> propertyMap = updatedConfigs.get(configType);
+       for (String property : propertyMap.keySet()) {
+         LOG.debug("Update Kerberos related config property: {} {} {}", configType, property,
propertyMap.get
+           (property));
+          clusterConfiguration.setProperty(configType, property, propertyMap.get(property));
+       }
+     }
+
+    } catch (KerberosInvalidConfigurationException e) {
+      LOG.error("An exception occurred while doing Kerberos related configuration update:
" + e, e);
+    }
+  }
+
   /**
    * Set all configurations on the cluster resource.
    * @param clusterTopology  cluster topology

http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index 960915f..4b45806 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -18,16 +18,20 @@
 
 package org.apache.ambari.server.topology;
 
+import com.google.inject.Injector;
 import com.google.inject.Singleton;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.HostRoleCommand;
 import org.apache.ambari.server.actionmanager.Request;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorBlueprintProcessor;
+import org.apache.ambari.server.controller.ClusterRequest;
+import org.apache.ambari.server.controller.KerberosHelper;
 import org.apache.ambari.server.controller.RequestStatusResponse;
 import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
 import org.apache.ambari.server.controller.internal.CredentialResourceProvider;
 import org.apache.ambari.server.controller.internal.ProvisionClusterRequest;
 import org.apache.ambari.server.controller.internal.RequestImpl;
+import org.apache.ambari.server.controller.internal.RequestStageContainer;
 import org.apache.ambari.server.controller.internal.ScaleClusterRequest;
 import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
@@ -40,6 +44,8 @@ import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO;
 import org.apache.ambari.server.orm.entities.StageEntity;
 import org.apache.ambari.server.security.encryption.CredentialStoreService;
+import org.apache.ambari.server.serveraction.kerberos.KerberosOperationException;
+import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.SecurityType;
 import org.apache.ambari.server.state.host.HostImpl;
 import org.apache.ambari.server.utils.RetryHelper;
@@ -99,10 +105,6 @@ public class TopologyManager {
   @Inject
   private SecurityConfigurationFactory securityConfigurationFactory;
 
-  @Inject
-  private CredentialStoreService credentialStoreService;
-
-
   /**
    * A boolean not cached thread-local (volatile) to prevent double-checked
    * locking on the synchronized keyword.
@@ -141,14 +143,20 @@ public class TopologyManager {
     // get the id prior to creating ambari resources which increments the counter
     Long provisionId = ambariContext.getNextRequestId();
 
+    final Stack stack = topology.getBlueprint().getStack();
+    boolean configureSecurity = false;
     SecurityConfiguration securityConfiguration = processSecurityConfiguration(request);
     if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.KERBEROS)
{
-
+      configureSecurity = true;
       addKerberosClient(topology);
 
+      // refresh default stack config after adding KERBEROS_CLIENT component to topology
+      topology.getBlueprint().getConfiguration().setParentConfiguration(stack.getConfiguration(topology.getBlueprint
+        ().getServices()));
+
       // create Cluster resource with security_type = KERBEROS, this will trigger cluster
Kerberization
       // upon host install task execution
-      ambariContext.createAmbariResources(topology, clusterName, securityConfiguration.getType());
+      ambariContext.createAmbariResources(topology, clusterName, SecurityType.KERBEROS);
       if (securityConfiguration.getDescriptor() != null) {
         submitKerberosDescriptorAsArtifact(clusterName, securityConfiguration.getDescriptor());
       }
@@ -177,10 +185,8 @@ public class TopologyManager {
 
     clusterTopologyMap.put(clusterId, topology);
 
-    final Stack stack = topology.getBlueprint().getStack();
-
     addClusterConfigRequest(topology, new ClusterConfigurationRequest(
-      ambariContext, topology, true, stackAdvisorBlueprintProcessor));
+      ambariContext, topology, true, stackAdvisorBlueprintProcessor, configureSecurity));
     LogicalRequest logicalRequest = processRequest(persistedRequest, topology, provisionId);
 
     //todo: this should be invoked as part of a generic lifecycle event which could possibly

http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
new file mode 100644
index 0000000..df32684
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorBlueprintProcessor;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.ClusterRequest;
+import org.apache.ambari.server.controller.ConfigurationRequest;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.controller.RequestStatusResponse;
+import org.apache.ambari.server.controller.internal.ProvisionClusterRequest;
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.controller.spi.ClusterController;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.ResourceProvider;
+import org.apache.ambari.server.security.encryption.CredentialStoreService;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.SecurityType;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.easymock.EasyMockRule;
+import org.easymock.EasyMockSupport;
+import org.easymock.Mock;
+import org.easymock.MockType;
+import org.easymock.TestSubject;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.isA;
+import static org.easymock.EasyMock.isNull;
+import static org.easymock.EasyMock.newCapture;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+
+/**
+ * ClusterConfigurationRequest unit tests
+ */
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({AmbariContext.class})
+public class ClusterConfigurationRequestTest {
+
+  @Rule
+  public EasyMockRule mocks = new EasyMockRule(this);
+
+  @Mock(type = MockType.NICE)
+  private Blueprint blueprint;
+
+  @Mock(type = MockType.NICE)
+  private AmbariContext ambariContext;
+
+  @Mock(type = MockType.NICE)
+  private Cluster cluster;
+
+  @Mock(type = MockType.NICE)
+  private Clusters clusters;
+
+  @Mock(type = MockType.NICE)
+  private ClusterTopology topology;
+
+  @Mock(type = MockType.NICE)
+  private StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor;
+
+  @Mock(type = MockType.NICE)
+  private Stack stack;
+
+  @Mock(type = MockType.NICE)
+  private AmbariManagementController controller;
+
+  @Mock(type = MockType.NICE)
+  private KerberosHelper kerberosHelper;
+
+  @Test
+  public void testProcessClusterConfigRequestIncludeKererosConfigs() throws Exception {
+
+    Map<String, Map<String, String>> existingConfig = new HashMap<String,
Map<String, String>>();
+    Configuration stackConfig = new Configuration(existingConfig,
+      new HashMap<String, Map<String, Map<String, String>>>());
+
+    PowerMock.mockStatic(AmbariContext.class);
+    AmbariContext.getController();
+    expectLastCall().andReturn(controller).anyTimes();
+
+    expect(controller.getClusters()).andReturn(clusters).anyTimes();
+    expect(controller.getKerberosHelper()).andReturn(kerberosHelper).once();
+
+    expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes();
+
+    expect(blueprint.getStack()).andReturn(stack).anyTimes();
+    expect(stack.getAllConfigurationTypes(anyString())).andReturn(Collections.<String>singletonList("testConfigType")
+    ).anyTimes();
+    expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.<String>emptySet()).anyTimes();
+    expect(stack.getConfigurationPropertiesWithMetadata(anyString(), anyString())).andReturn(Collections.<String,
+      Stack.ConfigProperty>emptyMap()).anyTimes();
+
+    Set<String> services = new HashSet<>();
+    services.add("HDFS");
+    services.add("KERBEROS");
+    services.add("ZOOKEPER");
+    expect(blueprint.getServices()).andReturn(services).anyTimes();
+    expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes();
+    expect(topology.getBlueprint()).andReturn(blueprint).anyTimes();
+    expect(topology.getConfiguration()).andReturn(stackConfig).anyTimes();
+    expect(topology.getHostGroupInfo()).andReturn(Collections.<String, HostGroupInfo>emptyMap());
+    expect(topology.getClusterId()).andReturn(Long.valueOf(1)).anyTimes();
+    expect(ambariContext.getClusterName(Long.valueOf(1))).andReturn("testCluster").anyTimes();
+    expect(ambariContext.createConfigurationRequests(anyObject(Map.class))).andReturn(Collections
+      .<ConfigurationRequest>emptyList()).anyTimes();
+
+    Map<String, Map<String, String>> kerberosConfig = new HashMap<String,
Map<String, String>>();
+    Map<String, String> properties = new HashMap<>();
+    properties.put("testPorperty", "testValue");
+    kerberosConfig.put("testConfigType", properties);
+    expect(kerberosHelper.getServiceConfigurationUpdates(anyObject(Cluster.class), anyObject(Map.class),
anyObject
+      (Set.class))).andReturn(kerberosConfig).anyTimes();
+
+    PowerMock.replay(stack, blueprint, topology, controller, clusters, kerberosHelper, ambariContext,
+      AmbariContext
+      .class);
+
+    ClusterConfigurationRequest clusterConfigurationRequest = new ClusterConfigurationRequest(
+      ambariContext, topology, false, stackAdvisorBlueprintProcessor, true);
+    clusterConfigurationRequest.process();
+
+    verify(blueprint, topology, ambariContext, controller, kerberosHelper);
+
+  }
+
+}


Mime
View raw message