ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yus...@apache.org
Subject ambari git commit: AMBARI-13582. Allow use passwords references in custom actions. (vbrodetskyi via yusaku)
Date Wed, 28 Oct 2015 18:44:31 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 48c9c7412 -> 64d7b1dd9


AMBARI-13582. Allow use passwords references in custom actions. (vbrodetskyi via yusaku)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/64d7b1dd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/64d7b1dd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/64d7b1dd

Branch: refs/heads/branch-2.1
Commit: 64d7b1dd9b777849b40ee4128a8aa72a1b85a4a2
Parents: 48c9c74
Author: Yusaku Sako <yusaku@hortonworks.com>
Authored: Wed Oct 28 11:42:54 2015 -0700
Committer: Yusaku Sako <yusaku@hortonworks.com>
Committed: Wed Oct 28 11:44:13 2015 -0700

----------------------------------------------------------------------
 .../controller/AmbariActionExecutionHelper.java |  4 +
 .../AmbariManagementControllerImpl.java         | 27 ++++---
 .../ambari/server/controller/AmbariServer.java  |  2 +-
 .../controller/ConfigurationResponse.java       | 13 +---
 .../internal/BlueprintResourceProvider.java     | 49 +++++++++---
 .../apache/ambari/server/state/StackInfo.java   | 19 +++++
 .../server/state/cluster/ClusterImpl.java       | 14 +---
 .../server/topology/BlueprintValidatorImpl.java | 27 +++++++
 .../ambari/server/utils/SecretReference.java    | 78 +++++++++++++++-----
 .../AmbariManagementControllerTest.java         | 24 ++++--
 .../internal/BlueprintResourceProviderTest.java | 47 ++++++++++--
 .../server/topology/BlueprintImplTest.java      | 63 +++++++++++++++-
 .../services/YARN/configuration/yarn-site.xml   |  6 ++
 13 files changed, 291 insertions(+), 82 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index d834731..215aca8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -46,6 +46,7 @@ import org.apache.ambari.server.state.ServiceComponentHost;
 import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent;
+import org.apache.ambari.server.utils.SecretReference;
 import org.apache.ambari.server.utils.StageUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
@@ -406,6 +407,9 @@ public class AmbariActionExecutionHelper {
       }
 
       roleParams.putAll(actionContext.getParameters());
+
+      SecretReference.replaceReferencesWithPasswords(roleParams, cluster);
+
       if (componentInfo != null) {
         roleParams.put(COMPONENT_CATEGORY, componentInfo.getCategory());
       }

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 615b46d..2fb73ee 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -728,9 +728,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
           String passwordPropertyValue = requestProperties.get(passwordProperty);
           if (!SecretReference.isSecret(passwordPropertyValue))
             continue;
-          SecretReference ref = new SecretReference(passwordPropertyValue, passwordProperty,
cluster);
-          if (!ref.getClusterName().equals(request.getClusterName()))
-            throw new AmbariException("Can not reference to different cluster in SECRET");
+          SecretReference ref = new SecretReference(passwordPropertyValue, cluster);
           String refValue = ref.getValue();
           requestProperties.put(passwordProperty, refValue);
         }
@@ -931,9 +929,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("Received a getClusters request"
-        + ", clusterName=" + request.getClusterName()
-        + ", clusterId=" + request.getClusterId()
-        + ", stackInfo=" + request.getStackVersion());
+          + ", clusterName=" + request.getClusterName()
+          + ", clusterId=" + request.getClusterId()
+          + ", stackInfo=" + request.getStackVersion());
     }
 
     Cluster singleCluster = null;
@@ -1400,7 +1398,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
             if (propertiesTypes.containsKey(PropertyType.PASSWORD) &&
                 propertiesTypes.get(PropertyType.PASSWORD).contains(propertyName)) {
               if (SecretReference.isSecret(propertyValue)) {
-                SecretReference ref = new SecretReference(propertyValue, propertyName, cluster);
+                SecretReference ref = new SecretReference(propertyValue, cluster);
                 requestConfigProperties.put(propertyName, ref.getValue());
               }
             }
@@ -3387,30 +3385,31 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
     ExecuteCommandJson jsons = customCommandExecutionHelper.getCommandJson(actionExecContext,
cluster, stackId);
     String commandParamsForStage = jsons.getCommandParamsForStage();
 
+    Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage, new
TypeToken<Map<String, String>>()
+      {}.getType());
     // Ensure that the specified requestContext (if any) is set as the request context
     if (!requestContext.isEmpty()) {
       requestStageContainer.setRequestContext(requestContext);
     }
 
+    // replace password references in requestProperties
+    SecretReference.replaceReferencesWithPasswords(commandParamsStage, cluster);
+
     // If the request is to perform the Kerberos service check, set up the stages to
     // ensure that the (cluster-level) smoke user principal and keytab is available on all
hosts
     boolean kerberosServiceCheck = Role.KERBEROS_SERVICE_CHECK.name().equals(actionRequest.getCommandName());
     if (kerberosServiceCheck) {
       // Parse the command parameters into a map so that additional values may be added to
it
-      Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage,
-          new TypeToken<Map<String, String>>() {
-          }.getType());
 
       try {
         requestStageContainer = kerberosHelper.createTestIdentity(cluster, commandParamsStage,
requestStageContainer);
       } catch (KerberosOperationException e) {
         throw new IllegalArgumentException(e.getMessage(), e);
       }
-
-      // Recreate commandParamsForStage with the added values
-      commandParamsForStage = gson.toJson(commandParamsStage);
     }
 
+    commandParamsForStage = gson.toJson(commandParamsStage);
+
     Stage stage = createNewStage(requestStageContainer.getLastStageId(), cluster, requestId,
requestContext,
         jsons.getClusterHostInfo(), commandParamsForStage, jsons.getHostParamsForStage());
 
@@ -3441,7 +3440,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
     if (kerberosServiceCheck) {
       // Parse the command parameters into a map so that existing values may be accessed
and
       // additional values may be added to it.
-      Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage,
+      commandParamsStage = gson.fromJson(commandParamsForStage,
           new TypeToken<Map<String, String>>() {
           }.getType());
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 021f4c8..57ae04c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -687,7 +687,7 @@ public class AmbariServer {
     StackDefinedPropertyProvider.init(injector);
     AbstractControllerResourceProvider.init(injector.getInstance(ResourceProviderFactory.class));
     BlueprintResourceProvider.init(injector.getInstance(BlueprintFactory.class),
-        injector.getInstance(BlueprintDAO.class), injector.getInstance(Gson.class));
+        injector.getInstance(BlueprintDAO.class), injector.getInstance(Gson.class), ambariMetaInfo);
     StackDependencyResourceProvider.init(ambariMetaInfo);
     ClusterResourceProvider.init(injector.getInstance(TopologyManager.class),
         injector.getInstance(TopologyRequestFactoryImpl.class));

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java
index 3ed9306..eef3474 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java
@@ -79,7 +79,7 @@ public class ConfigurationResponse {
     this.configs = configs;
     this.configAttributes = configAttributes;
     this.propertiesTypes = propertiesTypes;
-    stubPasswords();
+    SecretReference.replacePasswordsWithReferences(propertiesTypes, configs, type, version);
   }
 
   /**
@@ -215,15 +215,4 @@ public class ConfigurationResponse {
   public void setPropertiesTypes(Map<PropertyInfo.PropertyType, Set<String>>
propertiesTypes) {
     this.propertiesTypes = propertiesTypes;
   }
-
-  private void stubPasswords(){
-    if(propertiesTypes != null && propertiesTypes.containsKey(PropertyInfo.PropertyType.PASSWORD))
{
-      for(String pwdPropertyName: propertiesTypes.get(PropertyInfo.PropertyType.PASSWORD))
{
-        if(configs.containsKey(pwdPropertyName)){
-          String stub = SecretReference.generateStub(clusterName, type, version);
-          configs.put(pwdPropertyName, stub);
-        }
-      }
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
index 6cb6a74..5994094 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
@@ -32,6 +32,7 @@ import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.DuplicateResourceException;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
 import org.apache.ambari.server.controller.spi.NoSuchResourceException;
@@ -51,11 +52,13 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
 import org.apache.ambari.server.orm.entities.HostGroupEntity;
 import org.apache.ambari.server.orm.entities.StackEntity;
 import org.apache.ambari.server.stack.NoSuchStackException;
+import org.apache.ambari.server.state.*;
 import org.apache.ambari.server.topology.Blueprint;
 import org.apache.ambari.server.topology.BlueprintFactory;
 import org.apache.ambari.server.topology.InvalidTopologyException;
 
 import com.google.gson.Gson;
+import org.apache.ambari.server.utils.SecretReference;
 
 
 /**
@@ -116,6 +119,10 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
    */
   private static Gson jsonSerializer;
 
+  /**
+   * Used to get stack metainfo.
+   */
+  private static AmbariMetaInfo ambariMetaInfo;
 
   // ----- Constructors ----------------------------------------------------
 
@@ -140,10 +147,11 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
    * @param dao       blueprint data access object
    * @param gson      json serializer
    */
-  public static void init(BlueprintFactory factory, BlueprintDAO dao, Gson gson) {
+  public static void init(BlueprintFactory factory, BlueprintDAO dao, Gson gson, AmbariMetaInfo
metaInfo) {
     blueprintFactory = factory;
     blueprintDAO = dao;
     jsonSerializer = gson;
+    ambariMetaInfo = metaInfo;
   }
 
   // ----- ResourceProvider ------------------------------------------------
@@ -175,7 +183,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
   //todo: continue to use dao/entity directly or use blueprint factory?
   public Set<Resource> getResources(Request request, Predicate predicate)
       throws SystemException, UnsupportedPropertyException,
-             NoSuchResourceException, NoSuchParentResourceException {
+      NoSuchResourceException, NoSuchParentResourceException {
 
     List<BlueprintEntity> results        = null;
     boolean               applyPredicate = false;
@@ -242,8 +250,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
       modifyResources(new Command<Void>() {
         @Override
         public Void invoke() throws AmbariException {
-        blueprintDAO.removeByName(blueprintName);
-        return null;
+          blueprintDAO.removeByName(blueprintName);
+          return null;
         }
       });
     }
@@ -263,7 +271,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
    *
    * @return a new resource instance for the given blueprint entity
    */
-  protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds) {
+  protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds) throws
NoSuchResourceException {
     StackEntity stackEntity = entity.getStack();
     Resource resource = new ResourceImpl(Resource.Type.Blueprint);
     setResourceProperty(resource, BLUEPRINT_NAME_PROPERTY_ID, entity.getBlueprintName(),
requestedIds);
@@ -304,16 +312,39 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
    * @return list of configuration property maps
    */
   List<Map<String, Map<String, Object>>> populateConfigurationList(
-      Collection<? extends BlueprintConfiguration> configurations) {
+      Collection<? extends BlueprintConfiguration> configurations) throws NoSuchResourceException
{
 
     List<Map<String, Map<String, Object>>> listConfigurations = new ArrayList<Map<String,
Map<String, Object>>>();
     for (BlueprintConfiguration config : configurations) {
       Map<String, Map<String, Object>> mapConfigurations = new HashMap<String,
Map<String, Object>>();
       Map<String, Object> configTypeDefinition = new HashMap<String, Object>();
       String type = config.getType();
-      Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson(
-          config.getConfigData(), Map.class);
-      configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
+
+      if(config instanceof BlueprintConfigEntity) {
+        Map<String, String> properties = jsonSerializer.<Map<String, String>>fromJson(
+            config.getConfigData(), Map.class);
+
+        StackEntity stack = ((BlueprintConfigEntity)config).getBlueprintEntity().getStack();
+        StackInfo metaInfoStack;
+
+        try {
+          metaInfoStack = ambariMetaInfo.getStack(stack.getStackName(), stack.getStackVersion());
+        } catch (AmbariException e) {
+          throw new NoSuchResourceException(e.getMessage());
+        }
+
+        Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>>
propertiesTypes =
+            metaInfoStack.getConfigPropertiesTypes(type);
+
+        SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type,
-1l);
+
+        configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
+      } else {
+        Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson(
+            config.getConfigData(), Map.class);
+        configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
+      }
+
       Map<String, Map<String, String>> attributes = jsonSerializer.<Map<String,
Map<String, String>>>fromJson(
           config.getConfigAttributes(), Map.class);
       if (attributes != null && !attributes.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
index e3ac3e0..2b9cd83 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
@@ -436,4 +436,23 @@ public class StackInfo implements Comparable<StackInfo>, Validable{
     }
     return result;
   }
+
+  public Map<PropertyInfo.PropertyType, Set<String>> getConfigPropertiesTypes(String
configType) {
+    Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes = new HashMap<>();
+    Collection<ServiceInfo> services = getServices();
+    for (ServiceInfo serviceInfo : services) {
+      for (PropertyInfo propertyInfo : serviceInfo.getProperties()) {
+        if (propertyInfo.getFilename().contains(configType) && !propertyInfo.getPropertyTypes().isEmpty())
{
+          Set<PropertyInfo.PropertyType> types = propertyInfo.getPropertyTypes();
+          for (PropertyInfo.PropertyType propertyType : types) {
+            if (!propertiesTypes.containsKey(propertyType))
+              propertiesTypes.put(propertyType, new HashSet<String>());
+            propertiesTypes.get(propertyType).add(propertyInfo.getName());
+          }
+        }
+      }
+    }
+    return propertiesTypes;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 4e37e14..7703bfd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -2920,19 +2920,7 @@ public class ClusterImpl implements Cluster {
       try {
         StackId stackId = this.getCurrentStackVersion();
         StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion());
-        Collection<ServiceInfo> services = stackInfo.getServices();
-        for (ServiceInfo serviceInfo : services) {
-          for (PropertyInfo propertyInfo : serviceInfo.getProperties()) {
-            if (propertyInfo.getFilename().contains(configType) && !propertyInfo.getPropertyTypes().isEmpty())
{
-              Set<PropertyInfo.PropertyType> types = propertyInfo.getPropertyTypes();
-              for (PropertyInfo.PropertyType propertyType : types) {
-                if (!propertiesTypes.containsKey(propertyType))
-                  propertiesTypes.put(propertyType, new HashSet<String>());
-                propertiesTypes.get(propertyType).add(propertyInfo.getName());
-              }
-            }
-          }
-        }
+        propertiesTypes = stackInfo.getConfigPropertiesTypes(configType);
       } catch (Exception e) {
 
       }

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index 9e8f163..1c293ee 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -21,6 +21,7 @@ package org.apache.ambari.server.topology;
 import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.state.AutoDeployInfo;
 import org.apache.ambari.server.state.DependencyInfo;
+import org.apache.ambari.server.utils.SecretReference;
 import org.apache.ambari.server.utils.VersionUtils;
 
 import java.util.Collection;
@@ -83,6 +84,32 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
     // we don't want to include default stack properties so we can't just use hostGroup full
properties
     Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties();
 
+    // we need to have real passwords, not references
+    if(clusterConfigurations != null) {
+      StringBuilder errorMessage = new StringBuilder();
+      boolean containsSecretReferences = false;
+      for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet())
{
+        String configType = configEntry.getKey();
+        if (configEntry.getValue() != null) {
+          for (Map.Entry<String, String> propertyEntry : configEntry.getValue().entrySet())
{
+            String propertyName = propertyEntry.getKey();
+            String propertyValue = propertyEntry.getValue();
+            if (propertyValue != null) {
+              if (SecretReference.isSecret(propertyValue)) {
+                errorMessage.append("  Config:" + configType + " Property:" + propertyName+"\n");
+                containsSecretReferences = true;
+              }
+            }
+          }
+        }
+      }
+      if(containsSecretReferences) {
+        throw new InvalidTopologyException("Secret references are not allowed in blueprints,
" +
+            "replace following properties with real passwords:\n"+errorMessage.toString());
+      }
+    }
+
+
     for (HostGroup hostGroup : blueprint.getHostGroups().values()) {
       Collection<String> processedServices = new HashSet<String>();
       Map<String, Collection<String>> allRequiredProperties = new HashMap<String,
Collection<String>>();

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java
b/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java
index 2b1aeae..d801975 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java
@@ -21,37 +21,36 @@ package org.apache.ambari.server.utils;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.PropertyInfo;
 
 import java.util.Map;
+import java.util.Set;
 
 public class SecretReference {
-  private String clusterName;
+  private static final String secretPrefix = "SECRET";
   private String configType;
   private Long version;
   private String value;
-  private String reference;
 
-  public SecretReference(String reference, String propertyName, Cluster cluster) throws AmbariException{
+  public SecretReference(String reference, Cluster cluster) throws AmbariException{
     String[] values = reference.split(":");
-    clusterName = values[1];
-    configType = values[2];
-    version = Long.valueOf(values[3]);
+
+    configType = values[1];
+    version = Long.valueOf(values[2]);
+
+    String propertyName = values[3];
+    String clusterName = cluster.getClusterName();
     Config refConfig = cluster.getConfigByVersion(configType, version);
 
     if(refConfig == null)
-      throw new AmbariException(String.format("Cluster: %s does not contain ConfigType: %s
ConfigVersion: %s",
-          cluster.getClusterName(), configType, version));
+      throw new AmbariException(String.format("Error when parsing secret reference. Cluster:
%s does not contain ConfigType: %s ConfigVersion: %s",
+          clusterName, configType, version));
     Map<String, String> refProperties = refConfig.getProperties();
     if(!refProperties.containsKey(propertyName))
-      throw new AmbariException(String.format("Cluster: %s ConfigType: %s ConfigVersion:
%s does not contain property '%s'",
-          cluster.getClusterName(), configType, version, propertyName));
-    this.value = refProperties.get(propertyName);
-
-    this.reference = reference;
-  }
+      throw new AmbariException(String.format("Error when parsing secret reference. Cluster:
%s ConfigType: %s ConfigVersion: %s does not contain property '%s'",
+          clusterName, configType, version, propertyName));
 
-  public String getClusterName() {
-    return clusterName;
+    this.value = refProperties.get(propertyName);
   }
 
   public void setConfigType(String configType) {
@@ -68,10 +67,51 @@ public class SecretReference {
 
   public static boolean isSecret(String value) {
     String[] values = value.split(":");
-    return values.length == 4 && values[0].equals("SECRET");
+    return values.length == 4 && values[0].equals(secretPrefix);
+  }
+
+  public static String generateStub(String configType, Long configVersion, String propertyName)
{
+    return secretPrefix + ":" + configType + ":" + configVersion.toString() + ":" + propertyName;
+  }
+
+  /**
+   * Replace secret references with appropriate real passwords.
+   * @param targetMap map in which replacement will be performed
+   * @param cluster current cluster
+   * @throws AmbariException
+   */
+  public static void replaceReferencesWithPasswords(Map<String, String> targetMap,
Cluster cluster)
+      throws AmbariException {
+    if(cluster != null) {
+      for (Map.Entry<String, String> propertyValueEntry : targetMap.entrySet()) {
+        String key = propertyValueEntry.getKey();
+        String value = propertyValueEntry.getValue();
+        if (value != null && SecretReference.isSecret(value)) {
+          SecretReference ref = new SecretReference(value, cluster);
+          targetMap.put(key, ref.getValue());
+        }
+      }
+    }
   }
 
-  public static String generateStub(String clusterName, String configType, Long configVersion)
{
-    return "SECRET:" + clusterName + ":" + configType + ":" + configVersion.toString();
+  /**
+   * Replace real passwords with secret references
+   * @param propertiesTypes map with properties types
+   * @param propertiesMap map with properties in which replacement will be performed
+   * @param configType configuration type
+   * @param configVersion configuration version
+   */
+  public static void replacePasswordsWithReferences(Map<PropertyInfo.PropertyType, Set<String>>
propertiesTypes,
+                                                    Map<String, String> propertiesMap,
+                                                    String configType,
+                                                    Long configVersion){
+    if(propertiesTypes != null && propertiesTypes.containsKey(PropertyInfo.PropertyType.PASSWORD))
{
+      for(String pwdPropertyName: propertiesTypes.get(PropertyInfo.PropertyType.PASSWORD))
{
+        if(propertiesMap.containsKey(pwdPropertyName)){
+          String stub = SecretReference.generateStub(configType, configVersion, pwdPropertyName);
+          propertiesMap.put(pwdPropertyName, stub);
+        }
+      }
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 1aa1a6f..36d1174 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -4123,7 +4123,7 @@ public class AmbariManagementControllerTest {
     Config config1 = cf.createNew(cluster, "global",
         new HashMap<String, String>() {{
           put("key1", "value1");
-        }}, new HashMap<String, Map<String,String>>());
+        }}, new HashMap<String, Map<String, String>>());
     config1.setTag("version1");
 
     Config config2 = cf.createNew(cluster, "core-site",
@@ -4132,8 +4132,15 @@ public class AmbariManagementControllerTest {
         }}, new HashMap<String, Map<String,String>>());
     config2.setTag("version1");
 
+    Config config3 = cf.createNew(cluster, "yarn-site",
+        new HashMap<String, String>() {{
+          put("test.password", "supersecret");
+        }}, new HashMap<String, Map<String,String>>());
+    config3.setTag("version1");
+
     cluster.addConfig(config1);
     cluster.addConfig(config2);
+    cluster.addConfig(config3);
 
     Service hdfs = cluster.addService("HDFS");
     hdfs.persist();
@@ -4161,6 +4168,7 @@ public class AmbariManagementControllerTest {
 
     Map<String, String> params = new HashMap<String, String>() {{
       put("test", "test");
+      put("pwd", "SECRET:yarn-site:1:test.password");
     }};
 
     Map<String, String> requestProperties = new HashMap<String, String>();
@@ -4193,6 +4201,8 @@ public class AmbariManagementControllerTest {
     Map<String, String> commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(),
type);
 
     Assert.assertTrue(commandParametersStage.containsKey("test"));
+    Assert.assertTrue(commandParametersStage.containsKey("pwd"));
+    Assert.assertEquals(commandParametersStage.get("pwd"), "supersecret");
     Assert.assertEquals("HDFS", cmd.getServiceName());
     Assert.assertEquals("DATANODE", cmd.getComponentName());
     Assert.assertNotNull(hostParametersStage.get("jdk_location"));
@@ -4233,6 +4243,8 @@ public class AmbariManagementControllerTest {
     commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(),
type);
 
     Assert.assertTrue(commandParametersStage.containsKey("test"));
+    Assert.assertTrue(commandParametersStage.containsKey("pwd"));
+    Assert.assertEquals(commandParametersStage.get("pwd"), "supersecret");
     Assert.assertEquals("HDFS", cmd.getServiceName());
     Assert.assertEquals("DATANODE", cmd.getComponentName());
     Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
@@ -10592,7 +10604,7 @@ public class AmbariManagementControllerTest {
         "hdfs-site",
         "version2",
         new HashMap<String, String>(){{
-          put("test.password", "SECRET:c1:hdfs-site:1");
+          put("test.password", "SECRET:hdfs-site:1:test.password");
           put("new", "new");//need this to mark config as "changed"
         }},
         new HashMap<String, Map<String, String>>()
@@ -10617,7 +10629,7 @@ public class AmbariManagementControllerTest {
         "hdfs-site",
         "version3",
         new HashMap<String, String>(){{
-          put("test.password", "SECRET:c1:hdfs-site:666");
+          put("test.password", "SECRET:hdfs-site:666:test.password");
         }},
         new HashMap<String, Map<String, String>>()
     );
@@ -10645,7 +10657,7 @@ public class AmbariManagementControllerTest {
         "hdfs-site",
         "version5",
         new HashMap<String, String>(){{
-          put("test.password", "SECRET:c1:hdfs-site:4");
+          put("test.password", "SECRET:hdfs-site:4:test.password");
           put("new", "new");
         }},
         new HashMap<String, Map<String, String>>()
@@ -10656,7 +10668,7 @@ public class AmbariManagementControllerTest {
       controller.updateClusters(Collections.singleton(crReq), null);
       fail("Request need to be failed with wrong secret reference");
     } catch (AmbariException e) {
-      assertEquals("Cluster: foo1 ConfigType: hdfs-site ConfigVersion: 4 does not contain
property 'test.password'",
+      assertEquals("Error when parsing secret reference. Cluster: foo1 ConfigType: hdfs-site
ConfigVersion: 4 does not contain property 'test.password'",
           e.getMessage());
     }
     cl.getAllConfigs();
@@ -10679,7 +10691,7 @@ public class AmbariManagementControllerTest {
       add(configRequest);
     }});
     for(ConfigurationResponse resp : requestedConfigs) {
-      String secretName = "SECRET:foo1:hdfs-site:"+resp.getVersion().toString();
+      String secretName = "SECRET:hdfs-site:"+resp.getVersion().toString()+":test.password";
       if(resp.getConfigs().containsKey("test.password")) {
         assertEquals(resp.getConfigs().get("test.password"), secretName);
       }

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
index 5bfdebb..8ef4bbb 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
@@ -44,6 +44,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.internal.BlueprintResourceProvider.BlueprintConfigPopulationStrategy;
@@ -69,6 +70,8 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
 import org.apache.ambari.server.orm.entities.HostGroupConfigEntity;
 import org.apache.ambari.server.orm.entities.HostGroupEntity;
 import org.apache.ambari.server.orm.entities.StackEntity;
+import org.apache.ambari.server.state.*;
+import org.apache.ambari.server.state.PropertyInfo;
 import org.apache.ambari.server.utils.StageUtils;
 import org.apache.ambari.server.topology.Blueprint;
 import org.apache.ambari.server.topology.BlueprintFactory;
@@ -104,7 +107,7 @@ public class BlueprintResourceProviderTest {
 
   @BeforeClass
   public static void initClass() {
-    BlueprintResourceProvider.init(blueprintFactory, dao, gson);
+    BlueprintResourceProvider.init(blueprintFactory, dao, gson, metaInfo);
 
     StackEntity stackEntity = new StackEntity();
     stackEntity.setStackName("test-stack-name");
@@ -381,7 +384,12 @@ public class BlueprintResourceProviderTest {
 
   @Test
   public void testGetResourcesNoPredicate_withConfiguration() throws SystemException, UnsupportedPropertyException,
-      NoSuchParentResourceException, NoSuchResourceException {
+      NoSuchParentResourceException, NoSuchResourceException, AmbariException {
+
+    StackInfo info = createMock(StackInfo.class);
+    expect(info.getConfigPropertiesTypes("core-site")).andReturn(new HashMap<PropertyInfo.PropertyType,
Set<String>>()).anyTimes();
+    expect(metaInfo.getStack("test-stack-name", "test-stack-version")).andReturn(info).anyTimes();
+    replay(info, metaInfo);
     Request request = createNiceMock(Request.class);
 
     Set<Map<String, Object>> testProperties = getBlueprintTestProperties();
@@ -951,20 +959,44 @@ public class BlueprintResourceProviderTest {
 
   @Test
   public void testPopulateConfigurationList() throws Exception {
+    StackEntity stackEntity = new StackEntity();
+    stackEntity.setStackName("test-stack-name");
+    stackEntity.setStackVersion("test-stack-version");
+    BlueprintEntity entity = createMock(BlueprintEntity.class);
+    expect(entity.getStack()).andReturn(stackEntity).anyTimes();
+
+    HashMap<PropertyInfo.PropertyType, Set<String>> pwdProperties = new HashMap<PropertyInfo.PropertyType,
Set<String>>() {{
+      put(PropertyInfo.PropertyType.PASSWORD, new HashSet<String>(){{
+        add("test.password");
+      }});
+    }};
+
+    StackInfo info = createMock(StackInfo.class);
+    expect(info.getConfigPropertiesTypes("type1")).andReturn(new HashMap<PropertyInfo.PropertyType,
Set<String>>()).anyTimes();
+    expect(info.getConfigPropertiesTypes("type2")).andReturn(new HashMap<PropertyInfo.PropertyType,
Set<String>>()).anyTimes();
+    expect(info.getConfigPropertiesTypes("type3")).andReturn(pwdProperties).anyTimes();
+    expect(metaInfo.getStack("test-stack-name", "test-stack-version")).andReturn(info).anyTimes();
+
+    replay(info, metaInfo, entity);
+
+
     // attributes is null
-    BlueprintConfiguration config1 = new BlueprintConfigEntity();
+    BlueprintConfigEntity config1 = new BlueprintConfigEntity();
     config1.setType("type1");
     config1.setConfigData("{\"key1\":\"value1\"}");
+    config1.setBlueprintEntity(entity);
     // attributes is empty
-    BlueprintConfiguration config2 = new BlueprintConfigEntity();
+    BlueprintConfigEntity config2 = new BlueprintConfigEntity();
     config2.setType("type2");
     config2.setConfigData("{\"key2\":\"value2\"}");
     config2.setConfigAttributes("{}");
+    config2.setBlueprintEntity(entity);
     // attributes is provided
-    BlueprintConfiguration config3 = new BlueprintConfigEntity();
+    BlueprintConfigEntity config3 = new BlueprintConfigEntity();
     config3.setType("type3");
-    config3.setConfigData("{\"key3\":\"value3\",\"key4\":\"value4\"}");
+    config3.setConfigData("{\"key3\":\"value3\",\"key4\":\"value4\",\"test.password\":\"pwdValue\"}");
     config3.setConfigAttributes("{\"final\":{\"key3\":\"attrValue1\",\"key4\":\"attrValue2\"}}");
+    config3.setBlueprintEntity(entity);
 
     List<Map<String, Map<String, Object>>> configs =
         provider.populateConfigurationList(Arrays.asList(config1, config2, config3));
@@ -1010,9 +1042,10 @@ public class BlueprintResourceProviderTest {
     Map<String, String> confProperties3
         = (Map<String, String>) typeConfig3.get(BlueprintResourceProvider.PROPERTIES_PROPERTY_ID);
     assertNotNull(confProperties3);
-    assertEquals(2, confProperties3.size());
+    assertEquals(3, confProperties3.size());
     assertEquals("value3", confProperties3.get("key3"));
     assertEquals("value4", confProperties3.get("key4"));
+    assertEquals("SECRET:type3:-1:test.password", confProperties3.get("test.password"));
     assertTrue(typeConfig3.containsKey(BlueprintResourceProvider.PROPERTIES_ATTRIBUTES_PROPERTY_ID));
     Map<String, Map<String, String>> confAttributes3
         = (Map<String, Map<String, String>>) typeConfig3.get(BlueprintResourceProvider.PROPERTIES_ATTRIBUTES_PROPERTY_ID);

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index 9d4163a..de740f4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -183,7 +183,6 @@ public class BlueprintImplTest {
     properties.put("hdfs-site", hdfsProps);
     hdfsProps.put("foo", "val");
     hdfsProps.put("bar", "val");
-
     Map<String, String> category1Props = new HashMap<String, String>();
     properties.put("category1", category1Props);
     category1Props.put("prop1", "val");
@@ -282,6 +281,68 @@ public class BlueprintImplTest {
     verify(stack, group1, group2);
   }
 
+  @Test
+  public void testValidateConfigurations__secretReference(){
+    Stack stack = createNiceMock(Stack.class);
+
+    HostGroup group1 = createNiceMock(HostGroup.class);
+    HostGroup group2 = createNiceMock(HostGroup.class);
+    Collection<HostGroup> hostGroups = new HashSet<HostGroup>();
+    hostGroups.add(group1);
+    hostGroups.add(group2);
+
+    Set<String> group1Components = new HashSet<String>();
+    group1Components.add("c1");
+    group1Components.add("c2");
+
+    Set<String> group2Components = new HashSet<String>();
+    group2Components.add("c1");
+    group2Components.add("c3");
+
+    Map<String, Map<String, String>> group2Props = new HashMap<String, Map<String,
String>>();
+    Map<String, String> group2Category2Props = new HashMap<String, String>();
+    group2Props.put("category2", group2Category2Props);
+    group2Category2Props.put("prop2", "val");
+
+    Collection<Stack.ConfigProperty> requiredHDFSProperties = new HashSet<Stack.ConfigProperty>();
+    requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "foo", null));
+    requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "bar", null));
+    requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "some_password", null));
+
+    requiredHDFSProperties.add(new Stack.ConfigProperty("category1", "prop1", null));
+
+    Collection<Stack.ConfigProperty> requiredService2Properties = new HashSet<Stack.ConfigProperty>();
+    requiredService2Properties.add(new Stack.ConfigProperty("category2", "prop2", null));
+
+
+    // Blueprint config
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> hdfsProps = new HashMap<String, String>();
+    properties.put("hdfs-site", hdfsProps);
+    hdfsProps.put("foo", "val");
+    hdfsProps.put("bar", "val");
+    hdfsProps.put("secret", "SECRET:hdfs-site:1:test");
+
+    Map<String, String> category1Props = new HashMap<String, String>();
+    properties.put("category1", category1Props);
+    category1Props.put("prop1", "val");
+
+    Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String,
Map<String, Map<String, String>>>();
+    Configuration configuration = new Configuration(properties, attributes, EMPTY_CONFIGURATION);
+    // set config for group2 which contains a required property
+
+    replay(stack, group1, group2);
+
+    Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration);
+    try {
+      blueprint.validateRequiredProperties();
+      fail("Expected exception to be thrown for using secret reference");
+    } catch (InvalidTopologyException e) {
+      System.out.println("****" + e.getMessage() + "***");
+    }
+
+  }
+
   //todo: ensure coverage for these existing tests
 
   //  private void validateEntity(BlueprintEntity entity, boolean containsConfig) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/64d7b1dd/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
index 1c02e86..f762116 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
@@ -24,6 +24,12 @@
 <!-- ResourceManager -->
 
   <property>
+    <name>test.password</name>
+    <value> </value>
+    <property-type>PASSWORD</property-type>
+  </property>
+
+  <property>
     <name>yarn.resourcemanager.resource-tracker.address</name>
     <value>localhost:8025</value>
     <deleted>true</deleted>


Mime
View raw message