ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [4/4] ambari git commit: Merge branch 'trunk' into branch-feature-AMBARI-12556
Date Wed, 31 May 2017 20:21:44 GMT
Merge branch 'trunk' into branch-feature-AMBARI-12556


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fb2076c7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fb2076c7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fb2076c7

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: fb2076c718c5bcafb1e83c35a841111d30c6204d
Parents: 138aa48 dc30b4e
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Wed May 31 15:23:58 2017 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Wed May 31 15:23:58 2017 -0400

----------------------------------------------------------------------
 ambari-infra/ambari-infra-manager/README.md     |  92 ++-
 .../ambari-infra-manager/docs/api/swagger.yaml  | 784 +++++++++++++++++++
 .../docs/images/batch-1.png                     | Bin 0 -> 20521 bytes
 .../docs/images/batch-2.png                     | Bin 0 -> 29388 bytes
 .../docs/images/batch-3.png                     | Bin 0 -> 14105 bytes
 .../docs/images/batch-4.png                     | Bin 0 -> 23277 bytes
 .../infra/common/InfraManagerConstants.java     |   2 +-
 .../infra/conf/InfraManagerApiDocConfig.java    |  35 +-
 .../conf/batch/InfraManagerBatchConfig.java     |   8 +-
 .../ambari/infra/job/dummy/DummyItemWriter.java |  13 +
 .../infra/job/dummy/DummyJobListener.java       |  39 +
 .../infra/job/dummy/DummyStepListener.java      |  41 +
 .../apache/ambari/infra/rest/JobResource.java   |   2 +-
 .../internal/UpgradeResourceProvider.java       |   8 +-
 14 files changed, 1009 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/fb2076c7/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 6f452b0,a8b7fb4..345bf5f
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@@ -99,11 -117,12 +99,12 @@@ import org.apache.ambari.server.state.s
  import org.apache.ambari.server.state.stack.upgrade.Task;
  import org.apache.ambari.server.state.stack.upgrade.TaskWrapper;
  import org.apache.ambari.server.state.stack.upgrade.UpdateStackGrouping;
 -import org.apache.ambari.server.state.stack.upgrade.UpgradeScope;
  import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
  import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent;
+ import org.apache.ambari.server.utils.StageUtils;
  import org.apache.commons.collections.CollectionUtils;
  import org.apache.commons.lang.StringUtils;
 +import org.codehaus.jackson.annotate.JsonProperty;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
@@@ -784,18 -995,287 +785,23 @@@ public class UpgradeResourceProvider ex
      return upgradeEntity;
    }
  
-   private RequestStageContainer createRequest(UpgradeContext upgradeContext) {
 -  /**
 -   * Handles the creation or resetting of configurations based on whether an
 -   * upgrade or downgrade is occurring. This method will not do anything when
 -   * the target stack version is the same as the cluster's current stack version
 -   * since, by definition, no new configurations are automatically created when
 -   * upgrading with the same stack (ie HDP 2.2.0.0 -> HDP 2.2.1.0).
 -   * <p/>
 -   * When upgrading or downgrade between stacks (HDP 2.2.0.0 -> HDP 2.3.0.0)
 -   * then this will perform the following:
 -   * <ul>
 -   * <li>Upgrade: Create new configurations that are a merge between the current
 -   * stack and the desired stack. If a value has changed between stacks, then
 -   * the target stack value should be taken unless the cluster's value differs
 -   * from the old stack. This can occur if a property has been customized after
 -   * installation.</li>
 -   * <li>Downgrade: Reset the latest configurations from the cluster's original
 -   * stack. The new configurations that were created on upgrade must be left
 -   * intact until all components have been reverted, otherwise heartbeats will
 -   * fail due to missing configurations.</li>
 -   * </ul>
 -   *
 -   *
 -   * @param stackName Stack name such as HDP, HDPWIN, BIGTOP
 -   * @param cluster
 -   *          the cluster
 -   * @param version
 -   *          the version
 -   * @param direction
 -   *          upgrade or downgrade
 -   * @param upgradePack
 -   *          upgrade pack used for upgrade or downgrade. This is needed to determine
 -   *          which services are effected.
 -   * @param userName
 -   *          username performing the action
 -   * @throws AmbariException
 -   */
 -  public void applyStackAndProcessConfigurations(String stackName, Cluster cluster, String
version, Direction direction, UpgradePack upgradePack, String userName)
 -    throws AmbariException {
 -    RepositoryVersionEntity targetRve = s_repoVersionDAO.findByStackNameAndVersion(stackName,
version);
 -    if (null == targetRve) {
 -      LOG.info("Could not find version entity for {}; not setting new configs", version);
 -      return;
 -    }
 -
 -    if (null == userName) {
 -      userName = getManagementController().getAuthName();
 -    }
 -
 -    // if the current and target stacks are the same (ie HDP 2.2.0.0 -> 2.2.1.0)
 -    // then we should never do anything with configs on either upgrade or
 -    // downgrade; however if we are going across stacks, we have to do the stack
 -    // checks differently depending on whether this is an upgrade or downgrade
 -    StackEntity targetStack = targetRve.getStack();
 -    StackId currentStackId = cluster.getCurrentStackVersion();
 -    StackId desiredStackId = cluster.getDesiredStackVersion();
 -    StackId targetStackId = new StackId(targetStack);
 -    // Only change configs if moving to a different stack.
 -    switch (direction) {
 -      case UPGRADE:
 -        if (currentStackId.equals(targetStackId)) {
 -          return;
 -        }
 -        break;
 -      case DOWNGRADE:
 -        if (desiredStackId.equals(targetStackId)) {
 -          return;
 -        }
 -        break;
 -    }
 -
 -    Map<String, Map<String, String>> newConfigurationsByType = null;
 -    ConfigHelper configHelper = getManagementController().getConfigHelper();
 -
 -    if (direction == Direction.UPGRADE) {
 -      // populate a map of default configurations for the old stack (this is
 -      // used when determining if a property has been customized and should be
 -      // overriden with the new stack value)
 -      Map<String, Map<String, String>> oldStackDefaultConfigurationsByType =
configHelper.getDefaultProperties(
 -          currentStackId, cluster, true);
 -
 -      // populate a map with default configurations from the new stack
 -      newConfigurationsByType = configHelper.getDefaultProperties(targetStackId, cluster,
true);
 -
 -      // We want to skip updating config-types of services that are not in the upgrade pack.
 -      // Care should be taken as some config-types could be in services that are in and
out
 -      // of the upgrade pack. We should never ignore config-types of services in upgrade
pack.
 -      Set<String> skipConfigTypes = new HashSet<>();
 -      Set<String> upgradePackServices = new HashSet<>();
 -      Set<String> upgradePackConfigTypes = new HashSet<>();
 -      AmbariMetaInfo ambariMetaInfo = s_metaProvider.get();
 -
 -      // ensure that we get the service info from the target stack
 -      // (since it could include new configuration types for a service)
 -      Map<String, ServiceInfo> stackServicesMap = ambariMetaInfo.getServices(
 -          targetStack.getStackName(), targetStack.getStackVersion());
 -
 -      for (Grouping group : upgradePack.getGroups(direction)) {
 -        for (UpgradePack.OrderService service : group.services) {
 -          if (service.serviceName == null || upgradePackServices.contains(service.serviceName))
{
 -            // No need to re-process service that has already been looked at
 -            continue;
 -          }
 -
 -          upgradePackServices.add(service.serviceName);
 -          ServiceInfo serviceInfo = stackServicesMap.get(service.serviceName);
 -          if (serviceInfo == null) {
 -            continue;
 -          }
 -
 -          // add every configuration type for all services defined in the
 -          // upgrade pack
 -          Set<String> serviceConfigTypes = serviceInfo.getConfigTypeAttributes().keySet();
 -          for (String serviceConfigType : serviceConfigTypes) {
 -            if (!upgradePackConfigTypes.contains(serviceConfigType)) {
 -              upgradePackConfigTypes.add(serviceConfigType);
 -            }
 -          }
 -        }
 -      }
 -
 -      // build a set of configurations that should not be merged since their
 -      // services are not installed
 -      Set<String> servicesNotInUpgradePack = new HashSet<>(stackServicesMap.keySet());
 -      servicesNotInUpgradePack.removeAll(upgradePackServices);
 -      for (String serviceNotInUpgradePack : servicesNotInUpgradePack) {
 -        ServiceInfo serviceInfo = stackServicesMap.get(serviceNotInUpgradePack);
 -        Set<String> configTypesOfServiceNotInUpgradePack = serviceInfo.getConfigTypeAttributes().keySet();
 -        for (String configType : configTypesOfServiceNotInUpgradePack) {
 -          if (!upgradePackConfigTypes.contains(configType) && !skipConfigTypes.contains(configType))
{
 -            skipConfigTypes.add(configType);
 -          }
 -        }
 -      }
 -
 -      // remove any configurations from the target stack that are not used
 -      // because the services are not installed
 -      Iterator<String> iterator = newConfigurationsByType.keySet().iterator();
 -      while (iterator.hasNext()) {
 -        String configType = iterator.next();
 -        if (skipConfigTypes.contains(configType)) {
 -          LOG.info("Stack Upgrade: Removing configs for config-type {}", configType);
 -          iterator.remove();
 -        }
 -      }
 -
 -      // now that the map has been populated with the default configurations
 -      // from the stack/service, overlay the existing configurations on top
 -      Map<String, DesiredConfig> existingDesiredConfigurationsByType = cluster.getDesiredConfigs();
 -      for (Map.Entry<String, DesiredConfig> existingEntry : existingDesiredConfigurationsByType.entrySet())
{
 -        String configurationType = existingEntry.getKey();
 -        if(skipConfigTypes.contains(configurationType)) {
 -          LOG.info("Stack Upgrade: Skipping config-type {} as upgrade-pack contains no updates
to its service", configurationType);
 -          continue;
 -        }
 -
 -        // NPE sanity, although shouldn't even happen since we are iterating
 -        // over the desired configs to start with
 -        Config currentClusterConfig = cluster.getDesiredConfigByType(configurationType);
 -        if (null == currentClusterConfig) {
 -          continue;
 -        }
 -
 -        // get current stack default configurations on install
 -        Map<String, String> configurationTypeDefaultConfigurations = oldStackDefaultConfigurationsByType.get(
 -            configurationType);
 -
 -        // NPE sanity for current stack defaults
 -        if (null == configurationTypeDefaultConfigurations) {
 -          configurationTypeDefaultConfigurations = Collections.emptyMap();
 -        }
 -
 -        // get the existing configurations
 -        Map<String, String> existingConfigurations = currentClusterConfig.getProperties();
 -
 -        // if the new stack configurations don't have the type, then simply add
 -        // all of the existing in
 -        Map<String, String> newDefaultConfigurations = newConfigurationsByType.get(
 -            configurationType);
 -
 -        if (null == newDefaultConfigurations) {
 -          newConfigurationsByType.put(configurationType, existingConfigurations);
 -          continue;
 -        } else {
 -          // TODO, should we remove existing configs whose value is NULL even though they
don't have a value in the new stack?
 -
 -          // Remove any configs in the new stack whose value is NULL, unless they currently
exist and the value is not NULL.
 -          Iterator<Map.Entry<String, String>> iter = newDefaultConfigurations.entrySet().iterator();
 -          while (iter.hasNext()) {
 -            Map.Entry<String, String> entry = iter.next();
 -            if (entry.getValue() == null) {
 -              iter.remove();
 -            }
 -          }
 -        }
 -
 -        // for every existing configuration, see if an entry exists; if it does
 -        // not exist, then put it in the map, otherwise we'll have to compare
 -        // the existing value to the original stack value to see if its been
 -        // customized
 -        for (Map.Entry<String, String> existingConfigurationEntry : existingConfigurations.entrySet())
{
 -          String existingConfigurationKey = existingConfigurationEntry.getKey();
 -          String existingConfigurationValue = existingConfigurationEntry.getValue();
 -
 -          // if there is already an entry, we now have to try to determine if
 -          // the value was customized after stack installation
 -          if (newDefaultConfigurations.containsKey(existingConfigurationKey)) {
 -            String newDefaultConfigurationValue = newDefaultConfigurations.get(
 -                existingConfigurationKey);
 -
 -            if (!StringUtils.equals(existingConfigurationValue, newDefaultConfigurationValue))
{
 -              // the new default is different from the existing cluster value;
 -              // only override the default value if the existing value differs
 -              // from the original stack
 -              String oldDefaultValue = configurationTypeDefaultConfigurations.get(
 -                  existingConfigurationKey);
 -
 -              if (!StringUtils.equals(existingConfigurationValue, oldDefaultValue)) {
 -                // at this point, we've determined that there is a difference
 -                // between default values between stacks, but the value was
 -                // also customized, so keep the customized value
 -                newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
 -              }
 -            }
 -          } else {
 -            // there is no entry in the map, so add the existing key/value pair
 -            newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
 -          }
 -        }
 -
 -        /*
 -        for every new configuration which does not exist in the existing
 -        configurations, see if it was present in the current stack
 -
 -        stack 2.x has foo-site/property (on-ambari-upgrade is false)
 -        stack 2.y has foo-site/property
 -        the current cluster (on 2.x) does not have it
 -
 -        In this case, we should NOT add it back as clearly stack advisor has removed it
 -        */
 -        Iterator<Map.Entry<String, String>> newDefaultConfigurationsIterator
= newDefaultConfigurations.entrySet().iterator();
 -        while( newDefaultConfigurationsIterator.hasNext() ){
 -          Map.Entry<String, String> newConfigurationEntry = newDefaultConfigurationsIterator.next();
 -          String newConfigurationPropertyName = newConfigurationEntry.getKey();
 -          if (configurationTypeDefaultConfigurations.containsKey(newConfigurationPropertyName)
 -              && !existingConfigurations.containsKey(newConfigurationPropertyName))
{
 -            LOG.info(
 -                "The property {}/{} exists in both {} and {} but is not part of the current
set of configurations and will therefore not be included in the configuration merge",
 -                configurationType, newConfigurationPropertyName, currentStackId, targetStackId);
 -
 -            // remove the property so it doesn't get merged in
 -            newDefaultConfigurationsIterator.remove();
 -          }
 -        }
 -      }
 -    } else {
 -      // downgrade
 -      cluster.applyLatestConfigurations(cluster.getCurrentStackVersion());
 -    }
 -
 -    // !!! update the stack
 -    cluster.setDesiredStackVersion(
 -        new StackId(targetStack.getStackName(), targetStack.getStackVersion()), true);
 -
 -    // !!! configs must be created after setting the stack version
 -    if (null != newConfigurationsByType) {
 -      configHelper.createConfigTypes(cluster, getManagementController(), newConfigurationsByType,
 -          userName, "Configuration created for Upgrade");
 -    }
 -  }
 -
 -  private RequestStageContainer createRequest(Cluster cluster, Direction direction, String
version) throws AmbariException {
++  private RequestStageContainer createRequest(UpgradeContext upgradeContext) throws AmbariException
{
      ActionManager actionManager = getManagementController().getActionManager();
  
      RequestStageContainer requestStages = new RequestStageContainer(
          actionManager.getNextRequestId(), null, s_requestFactory.get(), actionManager);
 -    requestStages.setRequestContext(String.format("%s to %s", direction.getVerb(true), version));
  
 +    Direction direction = upgradeContext.getDirection();
 +    RepositoryVersionEntity repositoryVersion = upgradeContext.getRepositoryVersion();
 +
 +    requestStages.setRequestContext(String.format("%s %s %s", direction.getVerb(true),
 +        direction.getPreposition(), repositoryVersion.getVersion()));
 +
++    Cluster cluster = upgradeContext.getCluster();
+     Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
+     String clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo);
+     requestStages.setClusterHostInfo(clusterHostInfoJson);
+ 
      return requestStages;
    }
  


Mime
View raw message