ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [06/10] ambari git commit: AMBARI-21180. Component command changes must include version numbers for all services (ncole)
Date Mon, 12 Jun 2017 17:19:23 GMT
AMBARI-21180. Component command changes must include version numbers for all services (ncole)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5a1bd715
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5a1bd715
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5a1bd715

Branch: refs/heads/trunk
Commit: 5a1bd715e11b2e3a79b139118a90b66eca32f025
Parents: 12202a2
Author: Nate Cole <ncole@hortonworks.com>
Authored: Mon Jun 5 17:39:42 2017 -0400
Committer: Nate Cole <ncole@hortonworks.com>
Committed: Tue Jun 6 10:33:14 2017 -0400

----------------------------------------------------------------------
 .../actionmanager/ExecutionCommandWrapper.java  | 65 +++++++++++---------
 .../ambari/server/agent/ExecutionCommand.java   | 57 ++++++++++++-----
 .../AmbariCustomCommandExecutionHelper.java     |  4 +-
 .../AmbariManagementControllerImpl.java         |  2 -
 .../STORM/0.9.1/package/scripts/params_linux.py |  2 -
 .../1.0.1.3.0/package/scripts/params_linux.py   |  2 -
 .../AmbariCustomCommandExecutionHelperTest.java | 11 ++--
 7 files changed, 84 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
index 2ec09d9..efd609a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
@@ -35,6 +35,7 @@ import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.ambari.server.state.DesiredConfig;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -143,7 +144,8 @@ public class ExecutionCommandWrapper {
       // now that the tags have been updated (if necessary), fetch the
       // configurations
       Map<String, Map<String, String>> configurationTags = executionCommand.getConfigurationTags();
-      if (null != configurationTags && !configurationTags.isEmpty()) {
+
+      if (MapUtils.isNotEmpty(configurationTags)) {
         Map<String, Map<String, String>> configProperties = configHelper
             .getEffectiveConfigProperties(cluster, configurationTags);
 
@@ -182,43 +184,46 @@ public class ExecutionCommandWrapper {
                 executionCommand.getConfigurationAttributes().get(type));
             }
         }
+      }
 
-        // set the repository version for the component this command is for -
-        // always use the current desired version
-        try {
-          RepositoryVersionEntity repositoryVersion = null;
-          String serviceName = executionCommand.getServiceName();
-          if (!StringUtils.isEmpty(serviceName)) {
-            Service service = cluster.getService(serviceName);
-            if (null != service) {
-              repositoryVersion = service.getDesiredRepositoryVersion();
-            }
+      // set the repository version for the component this command is for -
+      // always use the current desired version
+      try {
+        RepositoryVersionEntity repositoryVersion = null;
+        String serviceName = executionCommand.getServiceName();
+        if (!StringUtils.isEmpty(serviceName)) {
+          Service service = cluster.getService(serviceName);
+          if (null != service) {
+            repositoryVersion = service.getDesiredRepositoryVersion();
+          }
 
-            String componentName = executionCommand.getComponentName();
-            if (!StringUtils.isEmpty(componentName)) {
-              ServiceComponent serviceComponent = service.getServiceComponent(
-                  executionCommand.getComponentName());
+          String componentName = executionCommand.getComponentName();
+          if (!StringUtils.isEmpty(componentName)) {
+            ServiceComponent serviceComponent = service.getServiceComponent(
+                executionCommand.getComponentName());
 
-              if (null != serviceComponent) {
-                repositoryVersion = serviceComponent.getDesiredRepositoryVersion();
-              }
+            if (null != serviceComponent) {
+              repositoryVersion = serviceComponent.getDesiredRepositoryVersion();
             }
           }
+        }
 
-          if (null != repositoryVersion) {
-            executionCommand.getCommandParams().put(KeyNames.VERSION,
-                repositoryVersion.getVersion());
-            executionCommand.getHostLevelParams().put(KeyNames.CURRENT_VERSION,
-                repositoryVersion.getVersion());
-          }
-        } catch (ServiceNotFoundException serviceNotFoundException) {
-          // it's possible that there are commands specified for a service where
-          // the service doesn't exist yet
-          LOG.warn(
-              "The service {} is not installed in the cluster. No repository version will
be sent for this command.",
-              executionCommand.getServiceName());
+        if (null != repositoryVersion) {
+          executionCommand.getCommandParams().put(KeyNames.VERSION, repositoryVersion.getVersion());
+          executionCommand.getHostLevelParams().put(KeyNames.CURRENT_VERSION, repositoryVersion.getVersion());
         }
+      } catch (ServiceNotFoundException serviceNotFoundException) {
+        // it's possible that there are commands specified for a service where
+        // the service doesn't exist yet
+        LOG.warn(
+            "The service {} is not installed in the cluster. No repository version will be
sent for this command.",
+            executionCommand.getServiceName());
       }
+
+      // set the desired versions of versionable components.  This is safe even during an
upgrade because
+      // we are "loading-late": components that have not yet upgraded in an EU will have
the correct versions.
+      executionCommand.setComponentVersions(cluster);
+
     } catch (ClusterNotFoundException cnfe) {
       // it's possible that there are commands without clusters; in such cases,
       // just return the de-serialized command and don't try to read configs

http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 17b89b7..bd4c9dd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -26,8 +26,11 @@ import java.util.Set;
 
 import org.apache.ambari.annotations.Experimental;
 import org.apache.ambari.annotations.ExperimentalFeature;
+import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.RoleCommand;
-import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
 import org.apache.ambari.server.utils.StageUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -110,9 +113,6 @@ public class ExecutionCommand extends AgentCommand {
   @SerializedName("localComponents")
   private Set<String> localComponents = new HashSet<>();
 
-  @SerializedName("availableServices")
-  private Map<String, String> availableServices = new HashMap<>();
-
   /**
    * "true" or "false" indicating whether this
    * service is enabled for credential store use.
@@ -153,6 +153,9 @@ public class ExecutionCommand extends AgentCommand {
   @SerializedName("repositoryFile")
   private CommandRepository commandRepository;
 
+  @SerializedName("componentVersionMap")
+  private Map<String, Map<String, String>> componentVersionMap = new HashMap<>();
+
   public void setConfigurationCredentials(Map<String, Map<String, String>> configurationCredentials)
{
     this.configurationCredentials = configurationCredentials;
   }
@@ -300,18 +303,6 @@ public class ExecutionCommand extends AgentCommand {
     this.localComponents = localComponents;
   }
 
-  public Map<String, String> getAvailableServices() {
-    return availableServices;
-  }
-
-  public void setAvailableServicesFromServiceInfoMap(Map<String, ServiceInfo> serviceInfoMap)
{
-    Map<String, String> serviceVersionMap = new HashMap<>();
-    for (Map.Entry<String, ServiceInfo> entry : serviceInfoMap.entrySet()) {
-      serviceVersionMap.put(entry.getKey(), entry.getValue().getVersion());
-    }
-    availableServices = serviceVersionMap;
-  }
-
   public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes()
{
     return configurationAttributes;
   }
@@ -523,4 +514,38 @@ public class ExecutionCommand extends AgentCommand {
         comment = "This should be replaced by a map of all service component versions")
     String CURRENT_VERSION = "current_version";
   }
+
+  /**
+   * @return
+   */
+  public Map<String, Map<String, String>> getComponentVersionMap() {
+    return componentVersionMap;
+  }
+
+  /**
+   * Used to set a map of {service -> { component -> version}}.  This is necessary
when performing
+   * an upgrade to correct build paths of required binaries.
+   * @param cluster the cluster from which to build the map
+   */
+  public void setComponentVersions(Cluster cluster) throws AmbariException {
+    Map<String, Map<String, String>> componentVersionMap = new HashMap<>();
+
+    for (Service service : cluster.getServices().values()) {
+      Map<String, String> componentMap = new HashMap<>();
+
+      boolean shouldSet = false;
+      for (ServiceComponent component : service.getServiceComponents().values()) {
+        if (component.isVersionAdvertised()) {
+          shouldSet = true;
+          componentMap.put(component.getName(), component.getDesiredVersion());
+        }
+      }
+
+      if (shouldSet) {
+        componentVersionMap.put(service.getName(), componentMap);
+      }
+    }
+
+    this.componentVersionMap = componentVersionMap;
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index cabb9d6..1f3b4b0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -384,12 +384,12 @@ public class AmbariCustomCommandExecutionHelper {
         execCmd.setCommandType(AgentCommandType.BACKGROUND_EXECUTION_COMMAND);
       }
 
+      execCmd.setComponentVersions(cluster);
+
       execCmd.setConfigurations(configurations);
       execCmd.setConfigurationAttributes(configurationAttributes);
       execCmd.setConfigurationTags(configTags);
 
-      execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(),
stackId.getStackVersion()));
-
       // Get the value of credential store enabled from the DB
       Service clusterService = cluster.getService(serviceName);
       execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled()));

http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 186a19e..329998b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2555,11 +2555,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
       cluster.addSuspendedUpgradeParameters(commandParams, roleParams);
     }
 
-
     execCmd.setRoleParams(roleParams);
     execCmd.setCommandParams(commandParams);
 
-    execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(),
stackId.getStackVersion()));
     execCmd.setRepositoryFile(customCommandExecutionHelper.getCommandRepository(cluster,
component, host));
     hostParams.put(KeyNames.CURRENT_VERSION, repoVersion.getVersion());
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
index 78ec165..664c582 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
@@ -394,8 +394,6 @@ if enable_ranger_storm:
 namenode_hosts = default("/clusterHostInfo/namenode_host", [])
 has_namenode = not len(namenode_hosts) == 0
 
-availableServices = config['availableServices']
-
 hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
 hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode
else None
 hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode
else None

http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py
index 78ec165..664c582 100644
--- a/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py
@@ -394,8 +394,6 @@ if enable_ranger_storm:
 namenode_hosts = default("/clusterHostInfo/namenode_host", [])
 has_namenode = not len(namenode_hosts) == 0
 
-availableServices = config['availableServices']
-
 hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
 hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode
else None
 hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode
else None

http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
index a6c0c32..240517f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
@@ -70,7 +70,6 @@ import org.apache.ambari.server.state.RepositoryInfo;
 import org.apache.ambari.server.state.SecurityType;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.ServiceComponent;
-import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.StackInfo;
 import org.apache.ambari.server.state.State;
@@ -79,6 +78,7 @@ import org.apache.ambari.server.state.ValueAttributesInfo;
 import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
 import org.apache.ambari.server.topology.TopologyManager;
 import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.collections.MapUtils;
 import org.easymock.Capture;
 import org.easymock.EasyMock;
 import org.easymock.EasyMockRule;
@@ -553,14 +553,15 @@ public class AmbariCustomCommandExecutionHelperTest {
 
     ambariManagementController.createAction(actionRequest, requestProperties);
     StackId stackId = clusters.getCluster("c1").getDesiredStackVersion();
-    Map<String, ServiceInfo> services = ambariManagementController.getAmbariMetaInfo().getServices(stackId.getStackName(),
stackId.getStackVersion());
     Request request = requestCapture.getValue();
     Stage stage = request.getStages().iterator().next();
     List<ExecutionCommandWrapper> commands = stage.getExecutionCommands("c1-c6401");
     ExecutionCommand command = commands.get(0).getExecutionCommand();
-    for (String service : services.keySet()) {
-      Assert.assertEquals(command.getAvailableServices().get(service), services.get(service).getVersion());
-    }
+
+    // ZK is the only service that is versionable
+    Assert.assertFalse(MapUtils.isEmpty(command.getComponentVersionMap()));
+    Assert.assertEquals(1, command.getComponentVersionMap().size());
+    Assert.assertTrue(command.getComponentVersionMap().containsKey("ZOOKEEPER"));
   }
 
   @Test


Mime
View raw message