ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [19/50] [abbrv] ambari git commit: AMBARI-21617. Function ru_set_all is not working correctly due to changed command structure (ncole)
Date Thu, 10 Aug 2017 17:49:04 GMT
AMBARI-21617. Function ru_set_all is not working correctly due to changed command structure (ncole)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/abb9b463
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/abb9b463
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/abb9b463

Branch: refs/heads/branch-2.6
Commit: abb9b4638772c47fcdbeffde47cfed88460757f6
Parents: b07dc77
Author: Nate Cole <ncole@hortonworks.com>
Authored: Mon Jul 31 14:31:37 2017 -0400
Committer: Nate Cole <ncole@hortonworks.com>
Committed: Tue Aug 1 09:12:18 2017 -0400

----------------------------------------------------------------------
 .../ambari/server/actionmanager/Stage.java      |  28 +-
 .../internal/UpgradeResourceProvider.java       |  17 ++
 .../ambari/server/state/ConfigHelper.java       |   1 -
 .../ambari/server/state/UpgradeContext.java     |   2 +-
 .../ambari/server/state/UpgradeHelper.java      |  41 ++-
 .../custom_actions/scripts/ru_set_all.py        |  38 ---
 .../ExecutionCommandWrapperTest.java            |   4 +-
 .../server/api/services/AmbariMetaInfoTest.java |  14 +-
 .../AmbariManagementControllerTest.java         |   4 +-
 .../BackgroundCustomCommandExecutionTest.java   |   2 +-
 ...hYarnCapacitySchedulerReleaseConfigTest.java |   2 +-
 .../internal/UpgradeResourceProviderTest.java   |   6 +
 .../ambari/server/state/UpgradeHelperTest.java  |  83 +++++-
 .../server/topology/AmbariContextTest.java      |  58 ++--
 .../server/upgrade/UpgradeCatalogTest.java      |   2 +-
 .../python/custom_actions/test_ru_set_all.py    |  47 +---
 .../upgrades/upgrade_execute_task_test.xml      |  42 +++
 .../HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml | 267 +++++++++++++++++++
 18 files changed, 517 insertions(+), 141 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
index a01313e..1d6ceaf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
@@ -19,7 +19,6 @@ package org.apache.ambari.server.actionmanager;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -34,7 +33,6 @@ import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
 import org.apache.ambari.server.agent.AgentCommand.AgentCommandType;
 import org.apache.ambari.server.agent.ExecutionCommand;
-import org.apache.ambari.server.metadata.RoleCommandOrder;
 import org.apache.ambari.server.metadata.RoleCommandPair;
 import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
 import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
@@ -89,13 +87,13 @@ public class Stage {
   private volatile boolean wrappersLoaded = false;
 
   //Map of roles to successFactors for this stage. Default is 1 i.e. 100%
-  private Map<Role, Float> successFactors = new HashMap<Role, Float>();
+  private Map<Role, Float> successFactors = new HashMap<>();
 
   //Map of host to host-roles
   Map<String, Map<String, HostRoleCommand>> hostRoleCommands =
-      new TreeMap<String, Map<String, HostRoleCommand>>();
+      new TreeMap<>();
   private Map<String, List<ExecutionCommandWrapper>> commandsToSend =
-      new TreeMap<String, List<ExecutionCommandWrapper>>();
+      new TreeMap<>();
 
   @Inject
   private HostRoleCommandFactory hostRoleCommandFactory;
@@ -192,7 +190,9 @@ public class Stage {
     stageEntity.setHostRoleCommands(new ArrayList<HostRoleCommandEntity>());
     stageEntity.setRoleSuccessCriterias(new ArrayList<RoleSuccessCriteriaEntity>());
     stageEntity.setCommandParamsStage(commandParamsStage);
-    stageEntity.setHostParamsStage(hostParamsStage);
+    if (null != hostParamsStage) {
+      stageEntity.setHostParamsStage(hostParamsStage);
+    }
     stageEntity.setCommandExecutionType(commandExecutionType);
 
     for (Role role : successFactors.keySet()) {
@@ -228,7 +228,7 @@ public class Stage {
   }
 
   public List<HostRoleCommand> getOrderedHostRoleCommands() {
-    List<HostRoleCommand> commands = new ArrayList<HostRoleCommand>();
+    List<HostRoleCommand> commands = new ArrayList<>();
     //Correct due to ordered maps
     for (Map.Entry<String, Map<String, HostRoleCommand>> hostRoleCommandEntry : hostRoleCommands.entrySet()) {
       for (Map.Entry<String, HostRoleCommand> roleCommandEntry : hostRoleCommandEntry.getValue().entrySet()) {
@@ -457,7 +457,7 @@ public class Stage {
 
     ExecutionCommand cmd = commandWrapper.getExecutionCommand();
 
-    Map<String, String> cmdParams = new HashMap<String, String>();
+    Map<String, String> cmdParams = new HashMap<>();
     if (commandParams != null) {
       cmdParams.putAll(commandParams);
     }
@@ -466,18 +466,18 @@ public class Stage {
     }
     cmd.setCommandParams(cmdParams);
 
-    Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+    Map<String, Map<String, String>> configurations = new TreeMap<>();
     cmd.setConfigurations(configurations);
 
-    Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>();
+    Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<>();
     cmd.setConfigurationAttributes(configurationAttributes);
 
     if (configTags == null) {
-      configTags = new TreeMap<String, Map<String, String>>();
+      configTags = new TreeMap<>();
     }
     cmd.setConfigurationTags(configTags);
 
-    Map<String, String> roleParams = new HashMap<String, String>();
+    Map<String, String> roleParams = new HashMap<>();
     roleParams.put(ServerAction.ACTION_NAME, actionName);
     if (userName != null) {
       roleParams.put(ServerAction.ACTION_USER_NAME, userName);
@@ -505,7 +505,7 @@ public class Stage {
 
     Assert.notEmpty(cancelTargets, "Provided targets task Id are empty.");
 
-    Map<String, String> roleParams = new HashMap<String, String>();
+    Map<String, String> roleParams = new HashMap<>();
 
     roleParams.put("cancelTaskIdTargets", StringUtils.join(cancelTargets, ','));
     cmd.setRoleParams(roleParams);
@@ -516,7 +516,7 @@ public class Stage {
    * @return list of hosts
    */
   public synchronized List<String> getHosts() { // TODO: Check whether method should be synchronized
-    List<String> hlist = new ArrayList<String>();
+    List<String> hlist = new ArrayList<>();
     for (String h : hostRoleCommands.keySet()) {
       hlist.add(h);
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index c26f062..83a9510 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -99,6 +99,7 @@ import org.apache.ambari.server.state.stack.upgrade.StageWrapper;
 import org.apache.ambari.server.state.stack.upgrade.Task;
 import org.apache.ambari.server.state.stack.upgrade.TaskWrapper;
 import org.apache.ambari.server.state.stack.upgrade.UpdateStackGrouping;
+import org.apache.ambari.server.state.stack.upgrade.UpgradeScope;
 import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent;
 import org.apache.ambari.server.utils.StageUtils;
@@ -947,6 +948,22 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     Map<String, String> params = getNewParameterMap(request, context);
     params.put(UpgradeContext.COMMAND_PARAM_TASKS, entity.getTasks());
 
+    // !!! when not scoped to a component (generic execution task)
+    if (context.isScoped(UpgradeScope.COMPLETE) && null == componentName) {
+      if (context.getDirection().isUpgrade()) {
+        params.put(KeyNames.VERSION, context.getRepositoryVersion().getVersion());
+      } else {
+        // !!! in a full downgrade, the target version should be any of the history versions
+        UpgradeEntity lastUpgrade = s_upgradeDAO.findLastUpgradeForCluster(
+            cluster.getClusterId(), Direction.UPGRADE);
+
+        @Experimental(feature = ExperimentalFeature.PATCH_UPGRADES,
+            comment = "Shouldn't be getting the overall downgrade-to version.")
+        UpgradeHistoryEntity lastHistory = lastUpgrade.getHistory().iterator().next();
+        params.put(KeyNames.VERSION, lastHistory.getFromReposistoryVersion().getVersion());
+      }
+    }
+
     // Apply additional parameters to the command that come from the stage.
     applyAdditionalParameters(wrapper, params);
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 8270370..16fc971 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -1110,7 +1110,6 @@ public class ConfigHelper {
             serviceMapped.put(service, new HashSet<Config>());
           }
           serviceMapped.get(service).add(baseConfig);
-
         } catch (Exception e) {
           // !!! ignore
         }

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
index 6f469e5..8bf0400 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
@@ -669,7 +669,7 @@ public class UpgradeContext {
    * <p/>
    * If the direction is {@link Direction#UPGRADE} then this will return the
    * target repository which every service will be on if the upgrade is
-   * finalized. <br/>
+   * finalized. <p/>
    * If the direction is {@link Direction#DOWNGRADE} then this will return the
    * repository from which the downgrade is coming from.
    *

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index a5881d2..a5b40ff 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -244,6 +245,8 @@ public class UpgradeHelper {
 
     if (StringUtils.isNotEmpty(preferredUpgradePackName) && packs.containsKey(preferredUpgradePackName)) {
       pack = packs.get(preferredUpgradePackName);
+
+      LOG.warn("Upgrade pack '{}' not found for stack {}", preferredUpgradePackName, currentStack);
     }
 
     // Best-attempt at picking an upgrade pack assuming within the same stack whose target stack version matches.
@@ -863,8 +866,11 @@ public class UpgradeHelper {
     String userName = controller.getAuthName();
     Set<String> servicesInUpgrade = upgradeContext.getSupportedServices();
 
+    Set<String> clusterConfigTypes = new HashSet<>();
+    Set<String> processedClusterConfigTypes = new HashSet<>();
+
     // merge or revert configurations for any service that needs it
-    for( String serviceName : servicesInUpgrade ){
+    for (String serviceName : servicesInUpgrade) {
       RepositoryVersionEntity sourceRepositoryVersion = upgradeContext.getSourceRepositoryVersion(serviceName);
       RepositoryVersionEntity targetRepositoryVersion = upgradeContext.getTargetRepositoryVersion(serviceName);
       StackId sourceStackId = sourceRepositoryVersion.getStackId();
@@ -901,6 +907,12 @@ public class UpgradeHelper {
       Map<String, Map<String, String>> newServiceDefaultConfigsByType = configHelper.getDefaultProperties(
           targetStackId, serviceName);
 
+      if (null == oldServiceDefaultConfigsByType || null == newServiceDefaultConfigsByType) {
+        continue;
+      }
+
+      Set<String> foundConfigTypes = new HashSet<>();
+
       // find the current, existing configurations for the service
       List<Config> existingServiceConfigs = new ArrayList<>();
       List<ServiceConfigEntity> latestServiceConfigs = m_serviceConfigDAO.getLastServiceConfigsForService(
@@ -910,8 +922,23 @@ public class UpgradeHelper {
         List<ClusterConfigEntity> existingConfigurations = serviceConfig.getClusterConfigEntities();
         for (ClusterConfigEntity currentServiceConfig : existingConfigurations) {
           String configurationType = currentServiceConfig.getType();
+
           Config currentClusterConfigForService = cluster.getDesiredConfigByType(configurationType);
           existingServiceConfigs.add(currentClusterConfigForService);
+          foundConfigTypes.add(configurationType);
+        }
+      }
+
+      // !!! these are the types that come back from the config helper, but are not part of the service.
+      @SuppressWarnings("unchecked")
+      Set<String> missingConfigTypes = new HashSet<>(CollectionUtils.subtract(oldServiceDefaultConfigsByType.keySet(),
+          foundConfigTypes));
+
+      for (String missingConfigType : missingConfigTypes) {
+        Config config = cluster.getDesiredConfigByType(missingConfigType);
+        if (null != config) {
+          existingServiceConfigs.add(config);
+          clusterConfigTypes.add(missingConfigType);
         }
       }
 
@@ -1013,8 +1040,18 @@ public class UpgradeHelper {
       }
 
       if (null != newServiceDefaultConfigsByType) {
+
+        for (String clusterConfigType : clusterConfigTypes) {
+          if (processedClusterConfigTypes.contains(clusterConfigType)) {
+            newServiceDefaultConfigsByType.remove(clusterConfigType);
+          } else {
+            processedClusterConfigTypes.add(clusterConfigType);
+          }
+
+        }
+
         Set<String> configTypes = newServiceDefaultConfigsByType.keySet();
-        LOG.info("The upgrade will create the following configurations for stack {}: {}",
+        LOG.warn("The upgrade will create the following configurations for stack {}: {}",
             targetStackId, StringUtils.join(configTypes, ','));
 
         String serviceVersionNote = String.format("%s %s %s", direction.getText(true),

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
index a7732d9..e514c4b 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
@@ -117,44 +117,6 @@ class UpgradeSetAll(Script):
           stack_name, downgrade_from_version, downgrade_to_version))
       return
 
-    # iterate through all directory conf mappings and undo the symlinks
-    for key, value in conf_select.get_package_dirs().iteritems():
-      for directory_mapping in value:
-        original_config_directory = directory_mapping['conf_dir']
-        self._unlink_config(original_config_directory)
-
-
-  def _unlink_config(self, original_conf_directory):
-    """
-    Reverses the work performed in link_config. This should only be used when downgrading from
-    HDP 2.3 to 2.2 in order to undo the conf symlink work required for 2.3.
-
-    1. Checks if conf.backup exists, if not then do no work
-    2. Check for existance of 'etc' symlink and remove it
-    3. Rename conf.back back to conf
-
-    :original_conf_directory: the original conf directory that was made into a symlink (/etc/component/conf)
-    """
-    # calculate the parent and backup directories
-    original_conf_parent_directory = os.path.abspath(os.path.join(original_conf_directory, os.pardir))
-    backup_conf_directory = os.path.join(original_conf_parent_directory, "conf.backup")
-    Logger.info("Analyzing potential link {0}".format(original_conf_directory))
-
-    if os.path.islink(original_conf_directory):
-      # remove the old symlink
-      Execute(("rm", original_conf_directory), sudo=True)
-    elif os.path.isdir(original_conf_directory):
-      Directory(original_conf_directory, action="delete")
-    else:
-      Logger.info("  Skipping the unlink of {0}; it is not a symlink or does not exist".format(original_conf_directory))
-
-    if os.path.isdir(backup_conf_directory):
-      # rename the backup to the original name
-      Logger.info("  Unlinking {0} and restoring {1}".format(original_conf_directory, backup_conf_directory))
-      Execute(("mv", backup_conf_directory, original_conf_directory), sudo=True)
-    else:
-      Logger.info("  Skipping restoring config from backup {0} since it does not exist".format(backup_conf_directory))
-
 
 def is_host_skippable(stack_selector_path, formatted_version):
   """

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
index e20b7b3..f68f639 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
@@ -43,6 +43,7 @@ import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.ConfigFactory;
 import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartEvent;
 import org.apache.ambari.server.utils.StageUtils;
@@ -294,7 +295,8 @@ public class ExecutionCommandWrapperTest {
 
     StackId stackId = cluster.getDesiredStackVersion();
     RepositoryVersionEntity repositoryVersion = ormTestHelper.getOrCreateRepositoryVersion(stackId, "0.1-0000");
-    cluster.addService("HDFS", repositoryVersion);
+    Service service = cluster.getService("HDFS");
+    service.setDesiredRepositoryVersion(repositoryVersion);
 
     // first try with an INSTALL command - this should not populate version info
     ExecutionCommand executionCommand = new ExecutionCommand();

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
index 229b864..63cb21f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
@@ -92,6 +92,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -226,6 +227,7 @@ public class AmbariMetaInfoTest {
   }
 
   @Test
+  @Ignore
   public void testGetRepositoryDefault() throws Exception {
     // Scenario: user has internet and does nothing to repos via api
     // use the latest
@@ -1714,7 +1716,7 @@ public class AmbariMetaInfoTest {
       }
     }
 
-    assertEquals(3, hostAlertCount);
+    assertEquals(2, hostAlertCount);
     assertEquals(10, definitions.size() - hostAlertCount);
 
     for (AlertDefinitionEntity definition : definitions) {
@@ -1725,7 +1727,7 @@ public class AmbariMetaInfoTest {
     metaInfo.reconcileAlertDefinitions(clusters, false);
 
     definitions = dao.findAll();
-    assertEquals(13, definitions.size());
+    assertEquals(12, definitions.size());
 
     for (AlertDefinitionEntity definition : definitions) {
       assertEquals(28, definition.getScheduleInterval().intValue());
@@ -1734,7 +1736,7 @@ public class AmbariMetaInfoTest {
     // find all enabled for the cluster should find 6 (the ones from HDFS;
     // it will not find the agent alert since it's not bound to the cluster)
     definitions = dao.findAllEnabled(cluster.getClusterId());
-    assertEquals(12, definitions.size());
+    assertEquals(11, definitions.size());
 
     // create new definition
     AlertDefinitionEntity entity = new AlertDefinitionEntity();
@@ -1753,19 +1755,19 @@ public class AmbariMetaInfoTest {
 
     // verify the new definition is found (6 HDFS + 1 new one)
     definitions = dao.findAllEnabled(cluster.getClusterId());
-    assertEquals(13, definitions.size());
+    assertEquals(12, definitions.size());
 
     // reconcile, which should disable our bad definition
     metaInfo.reconcileAlertDefinitions(clusters, false);
 
     // find all enabled for the cluster should find 6
     definitions = dao.findAllEnabled(cluster.getClusterId());
-    assertEquals(12, definitions.size());
+    assertEquals(11, definitions.size());
 
     // find all should find 6 HDFS + 1 disabled + 1 agent alert + 2 server
     // alerts
     definitions = dao.findAll();
-    assertEquals(14, definitions.size());
+    assertEquals(13, definitions.size());
 
     entity = dao.findById(entity.getDefinitionId());
     assertFalse(entity.getEnabled());

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 011214c..59fa7f7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -2253,7 +2253,7 @@ public class AmbariManagementControllerTest {
     resp = controller.getClusters(Collections.singleton(r));
     Assert.assertTrue(resp.size() >= 3);
 
-    r = new ClusterRequest(null, null, "", null);
+    r = new ClusterRequest(null, null, null, null);
     resp = controller.getClusters(Collections.singleton(r));
     Assert.assertTrue("Stack ID request is invalid and expect them all", resp.size() > 3);
   }
@@ -6978,7 +6978,7 @@ public class AmbariManagementControllerTest {
     Assert.assertEquals(1, responsesWithParams.size());
     StackVersionResponse resp = responsesWithParams.iterator().next();
     assertNotNull(resp.getUpgradePacks());
-    assertEquals(14, resp.getUpgradePacks().size());
+    assertEquals(15, resp.getUpgradePacks().size());
     assertTrue(resp.getUpgradePacks().contains("upgrade_test"));
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java
index 4c237a7..fa9475e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java
@@ -218,7 +218,7 @@ public class BackgroundCustomCommandExecutionTest {
       dStateStr = desiredState.toString();
     }
     ServiceRequest r1 = new ServiceRequest(clusterName, serviceName,
-        m_repositoryVersion.getId(), m_repositoryVersion.getVersion(), dStateStr);
+        m_repositoryVersion.getId(), dStateStr);
 
     Set<ServiceRequest> requests = new HashSet<>();
     requests.add(r1);

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java
index 122e4ef..5755e7e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java
@@ -214,7 +214,7 @@ public class RefreshYarnCapacitySchedulerReleaseConfigTest {
         new StackId("HDP-2.0.7"), "2.0.7-1234");
 
     ServiceRequest r1 = new ServiceRequest(clusterName, serviceName,
-        repositoryVersion.getId(), repositoryVersion.getVersion(), dStateStr);
+        repositoryVersion.getId(), dStateStr);
 
     Set<ServiceRequest> requests = new HashSet<>();
     requests.add(r1);

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java
index c5fc039..aa216f7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java
@@ -1601,6 +1601,12 @@ public class UpgradeResourceProviderTest extends EasyMockSupport {
   public void testCreatePatchRevertUpgrade() throws Exception {
     Cluster cluster = clusters.getCluster("c1");
 
+    // add a single ZK server and client on 2.1.1.0
+    Service service = cluster.addService("HBASE", repoVersionEntity2110);
+    ServiceComponent component = service.addServiceComponent("HBASE_MASTER");
+    ServiceComponentHost sch = component.addServiceComponentHost("h1");
+    sch.setVersion("2.1.1.0");
+
     File f = new File("src/test/resources/hbase_version_test.xml");
     repoVersionEntity2112.setVersionXml(IOUtils.toString(new FileInputStream(f)));
     repoVersionEntity2112.setVersionXsd("version_definition.xsd");

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index f0b884b..9b85242 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -49,6 +49,7 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.ClusterRequest;
 import org.apache.ambari.server.controller.ConfigurationRequest;
+import org.apache.ambari.server.controller.internal.UpgradeResourceProvider;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
 import org.apache.ambari.server.orm.OrmTestHelper;
@@ -92,6 +93,7 @@ import org.junit.Before;
 import org.junit.Test;
 import org.springframework.security.core.context.SecurityContextHolder;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.google.gson.Gson;
@@ -333,7 +335,7 @@ public class UpgradeHelperTest extends EasyMockSupport {
     assertEquals("Save Cluster State", postGroup.items.get(1).getText());
     assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(1).getType());
 
-    assertEquals(2, groups.get(0).items.size());
+    assertEquals(3, groups.get(0).items.size());
     assertEquals(7, groups.get(1).items.size());
     assertEquals(2, groups.get(2).items.size());
 
@@ -719,7 +721,6 @@ public class UpgradeHelperTest extends EasyMockSupport {
 
     List<ConfigUpgradeChangeDefinition.Transfer> transfers = m_gson.fromJson(configurationJson,
             new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() { }.getType());
-    System.out.println(">> transfers"+transfers);
 
     assertEquals(6, transfers.size());
     assertEquals("copy-key", transfers.get(0).fromKey);
@@ -797,8 +798,6 @@ public class UpgradeHelperTest extends EasyMockSupport {
         new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() {
         }.getType());
 
-    System.out.println(" testConfigTaskConditionMet >> transfer"+transfers);
-
     assertEquals("copy-key-one", transfers.get(0).fromKey);
     assertEquals("copy-to-key-one", transfers.get(0).toKey);
 
@@ -2425,6 +2424,82 @@ public class UpgradeHelperTest extends EasyMockSupport {
     assertEquals("three-changed", expectedBazType.get("3"));
   }
 
+  @Test
+  public void testMergeConfigurationsWithClusterEnv() throws Exception {
+    Cluster cluster = makeCluster(true);
+
+    StackId oldStack = cluster.getDesiredStackVersion();
+    StackId newStack = new StackId("HDP-2.5.0");
+
+    ConfigFactory cf = injector.getInstance(ConfigFactory.class);
+
+    Config clusterEnv = cf.createNew(cluster, "cluster-env", "version1",
+        ImmutableMap.<String, String>builder().put("a", "b").build(),
+        Collections.<String, Map<String, String>>emptyMap());
+
+    Config zooCfg = cf.createNew(cluster, "zoo.cfg", "version1",
+        ImmutableMap.<String, String>builder().put("c", "d").build(),
+        Collections.<String, Map<String, String>>emptyMap());
+
+    cluster.addDesiredConfig("admin", Sets.newHashSet(clusterEnv, zooCfg));
+
+    Map<String, Map<String, String>> stackMap = new HashMap<>();
+    stackMap.put("cluster-env", new HashMap<String, String>());
+    stackMap.put("hive-site", new HashMap<String, String>());
+
+    final Map<String, String> clusterEnvMap = new HashMap<>();
+
+    Capture<Cluster> captureCluster = Capture.newInstance();
+    Capture<StackId> captureStackId = Capture.newInstance();
+    Capture<AmbariManagementController> captureAmc = Capture.newInstance();
+
+    Capture<Map<String, Map<String, String>>> cap = new Capture<Map<String, Map<String, String>>>() {
+      @Override
+      public void setValue(Map<String, Map<String, String>> value) {
+        if (value.containsKey("cluster-env")) {
+          clusterEnvMap.putAll(value.get("cluster-env"));
+        }
+      }
+    };
+
+    Capture<String> captureUsername = Capture.newInstance();
+    Capture<String> captureNote = Capture.newInstance();
+
+    EasyMock.reset(m_configHelper);
+    expect(m_configHelper.getDefaultProperties(oldStack, "HIVE")).andReturn(stackMap).atLeastOnce();
+    expect(m_configHelper.getDefaultProperties(newStack, "HIVE")).andReturn(stackMap).atLeastOnce();
+    expect(m_configHelper.getDefaultProperties(oldStack, "ZOOKEEPER")).andReturn(stackMap).atLeastOnce();
+    expect(m_configHelper.getDefaultProperties(newStack, "ZOOKEEPER")).andReturn(stackMap).atLeastOnce();
+    m_configHelper.createConfigTypes(
+        EasyMock.capture(captureCluster),
+        EasyMock.capture(captureStackId),
+        EasyMock.capture(captureAmc),
+        EasyMock.capture(cap),
+
+        EasyMock.capture(captureUsername),
+        EasyMock.capture(captureNote));
+    expectLastCall().atLeastOnce();
+
+    replay(m_configHelper);
+
+    RepositoryVersionEntity repoVersionEntity = helper.getOrCreateRepositoryVersion(new StackId("HDP-2.5.0"), "2.5.0-1234");
+
+    Map<String, Object> upgradeRequestMap = new HashMap<>();
+    upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name());
+    upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, repoVersionEntity.getId().toString());
+    upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test_HDP-250");
+    upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString());
+
+    UpgradeContextFactory contextFactory = injector.getInstance(UpgradeContextFactory.class);
+    UpgradeContext context = contextFactory.create(cluster, upgradeRequestMap);
+
+    UpgradeHelper upgradeHelper = injector.getInstance(UpgradeHelper.class);
+    upgradeHelper.updateDesiredRepositoriesAndConfigs(context);
+
+    assertNotNull(clusterEnvMap);
+    assertTrue(clusterEnvMap.containsKey("a"));
+  }
+
   /**
    * @param cluster
    * @param direction

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
index e798a4c..e92a619 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
@@ -59,6 +59,8 @@ import org.apache.ambari.server.controller.spi.ClusterController;
 import org.apache.ambari.server.controller.spi.Predicate;
 import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
@@ -118,14 +120,14 @@ public class AmbariContextTest {
   private static final ConfigFactory configFactory = createNiceMock(ConfigFactory.class);
   private static final Service mockService1 = createStrictMock(Service.class);
 
-  private static final Collection<String> blueprintServices = new HashSet<String>();
-  private static final Map<String, Service> clusterServices = new HashMap<String, Service>();
-  private static final Map<Long, ConfigGroup> configGroups = new HashMap<Long, ConfigGroup>();
+  private static final Collection<String> blueprintServices = new HashSet<>();
+  private static final Map<String, Service> clusterServices = new HashMap<>();
+  private static final Map<Long, ConfigGroup> configGroups = new HashMap<>();
   private Configuration bpConfiguration = null;
   private Configuration group1Configuration = null;
   private static final Collection<String> group1Hosts = Arrays.asList(HOST1, HOST2);
 
-  private Capture<Set<ConfigGroupRequest>> configGroupRequestCapture = new Capture<Set<ConfigGroupRequest>>();
+  private Capture<Set<ConfigGroupRequest>> configGroupRequestCapture = new Capture<>();
 
   @Before
   public void setUp() throws Exception {
@@ -156,26 +158,26 @@ public class AmbariContextTest {
     f.set(null, hostComponentResourceProvider);
 
     // bp configuration
-    Map<String, Map<String, String>> bpProperties = new HashMap<String, Map<String, String>>();
-    Map<String, String> bpType1Props = new HashMap<String, String>();
+    Map<String, Map<String, String>> bpProperties = new HashMap<>();
+    Map<String, String> bpType1Props = new HashMap<>();
     bpProperties.put("type1", bpType1Props);
     bpType1Props.put("prop1", "val1");
     bpType1Props.put("prop2", "val2");
     bpConfiguration = new Configuration(bpProperties, null);
 
     // host group 1 configuration
-    Map<String, Map<String, String>> group1Properties = new HashMap<String, Map<String, String>>();
-    Map<String, String> type1Props = new HashMap<String, String>();
+    Map<String, Map<String, String>> group1Properties = new HashMap<>();
+    Map<String, String> type1Props = new HashMap<>();
     group1Properties.put("type1", type1Props);
     type1Props.put("prop1", "val1.2");
     type1Props.put("prop3", "val3");
     group1Configuration = new Configuration(group1Properties, null, bpConfiguration);
 
-    Map<String, String> group1ResolvedProperties = new HashMap<String, String>(bpType1Props);
+    Map<String, String> group1ResolvedProperties = new HashMap<>(bpType1Props);
     group1ResolvedProperties.putAll(type1Props);
 
     // config type -> service mapping
-    Map<String, String> configTypeServiceMapping = new HashMap<String, String>();
+    Map<String, String> configTypeServiceMapping = new HashMap<>();
     configTypeServiceMapping.put("type1", "service1");
 
     // config groups
@@ -202,6 +204,14 @@ public class AmbariContextTest {
             type1Service1).anyTimes();
     replay(type1Service1);
 
+    RepositoryVersionDAO repositoryVersionDAO = createStrictMock(RepositoryVersionDAO.class);
+    RepositoryVersionEntity repositoryVersion = createStrictMock(RepositoryVersionEntity.class);
+    expect(repositoryVersion.getId()).andReturn(1L).anyTimes();
+    expect(repositoryVersionDAO.findByStackAndVersion(EasyMock.anyObject(StackId.class),
+        EasyMock.anyString())).andReturn(repositoryVersion).anyTimes();
+    replay(repositoryVersionDAO, repositoryVersion);
+    context.repositoryVersionDAO = repositoryVersionDAO;
+
     context.configFactory = configFactory;
 
     blueprintServices.add("service1");
@@ -270,23 +280,23 @@ public class AmbariContextTest {
   @Test
   public void testCreateAmbariResources() throws Exception {
     // expectations
-    Capture<ClusterRequest> clusterRequestCapture = new Capture<ClusterRequest>();
+    Capture<ClusterRequest> clusterRequestCapture = new Capture<>();
     controller.createCluster(capture(clusterRequestCapture));
     expectLastCall().once();
     expect(cluster.getServices()).andReturn(clusterServices).anyTimes();
 
-    Capture<Set<ServiceRequest>> serviceRequestCapture = new Capture<Set<ServiceRequest>>();
-    Capture<Set<ServiceComponentRequest>> serviceComponentRequestCapture = new Capture<Set<ServiceComponentRequest>>();
+    Capture<Set<ServiceRequest>> serviceRequestCapture = new Capture<>();
+    Capture<Set<ServiceComponentRequest>> serviceComponentRequestCapture = new Capture<>();
 
     serviceResourceProvider.createServices(capture(serviceRequestCapture));
     expectLastCall().once();
     componentResourceProvider.createComponents(capture(serviceComponentRequestCapture));
     expectLastCall().once();
 
-    Capture<Request> serviceInstallRequestCapture = new Capture<Request>();
-    Capture<Request> serviceStartRequestCapture = new Capture<Request>();
-    Capture<Predicate> installPredicateCapture = new Capture<Predicate>();
-    Capture<Predicate> startPredicateCapture = new Capture<Predicate>();
+    Capture<Request> serviceInstallRequestCapture = new Capture<>();
+    Capture<Request> serviceStartRequestCapture = new Capture<>();
+    Capture<Predicate> installPredicateCapture = new Capture<>();
+    Capture<Predicate> startPredicateCapture = new Capture<>();
 
     expect(serviceResourceProvider.updateResources(capture(serviceInstallRequestCapture),
         capture(installPredicateCapture))).andReturn(null).once();
@@ -305,7 +315,7 @@ public class AmbariContextTest {
 
     Collection<ServiceRequest> serviceRequests = serviceRequestCapture.getValue();
     assertEquals(2, serviceRequests.size());
-    Collection<String> servicesFound = new HashSet<String>();
+    Collection<String> servicesFound = new HashSet<>();
     for (ServiceRequest serviceRequest : serviceRequests) {
       servicesFound.add(serviceRequest.getServiceName());
       assertEquals(CLUSTER_NAME, serviceRequest.getClusterName());
@@ -315,13 +325,13 @@ public class AmbariContextTest {
 
     Collection<ServiceComponentRequest> serviceComponentRequests = serviceComponentRequestCapture.getValue();
     assertEquals(3, serviceComponentRequests.size());
-    Map<String, Collection<String>> foundServiceComponents = new HashMap<String, Collection<String>>();
+    Map<String, Collection<String>> foundServiceComponents = new HashMap<>();
     for (ServiceComponentRequest componentRequest : serviceComponentRequests) {
       assertEquals(CLUSTER_NAME, componentRequest.getClusterName());
       String serviceName = componentRequest.getServiceName();
       Collection<String> serviceComponents = foundServiceComponents.get(serviceName);
       if (serviceComponents == null) {
-        serviceComponents = new HashSet<String>();
+        serviceComponents = new HashSet<>();
         foundServiceComponents.put(serviceName, serviceComponents);
       }
       serviceComponents.add(componentRequest.getComponentName());
@@ -342,7 +352,7 @@ public class AmbariContextTest {
     Map<String, Object> installProperties = installPropertiesSet.iterator().next();
     assertEquals(CLUSTER_NAME, installProperties.get(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID));
     assertEquals("INSTALLED", installProperties.get(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID));
-    assertEquals(new EqualsPredicate<String>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME),
+    assertEquals(new EqualsPredicate<>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME),
         installPredicateCapture.getValue());
 
     Request startRequest = serviceStartRequestCapture.getValue();
@@ -351,7 +361,7 @@ public class AmbariContextTest {
     Map<String, Object> startProperties = startPropertiesSet.iterator().next();
     assertEquals(CLUSTER_NAME, startProperties.get(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID));
     assertEquals("STARTED", startProperties.get(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID));
-    assertEquals(new EqualsPredicate<String>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME),
+    assertEquals(new EqualsPredicate<>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME),
         installPredicateCapture.getValue());
   }
 
@@ -555,7 +565,7 @@ public class AmbariContextTest {
     testClusterEnvDesiredConfig.setTag(topologyResolvedState);
 
     Map<String, DesiredConfig> testDesiredConfigs =
-      new HashMap<String, DesiredConfig>();
+      new HashMap<>();
     testDesiredConfigs.put("hdfs-site", testHdfsDesiredConfig);
     testDesiredConfigs.put("core-site", testCoreSiteDesiredConfig);
     testDesiredConfigs.put("cluster-env", testClusterEnvDesiredConfig);
@@ -565,7 +575,7 @@ public class AmbariContextTest {
     replayAll();
 
     Set<String> testUpdatedConfigTypes =
-      new HashSet<String>();
+      new HashSet<>();
     testUpdatedConfigTypes.add("hdfs-site");
     testUpdatedConfigTypes.add("core-site");
     testUpdatedConfigTypes.add("cluster-env");

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java
index fbf5e05..d082938 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java
@@ -113,7 +113,7 @@ public class UpgradeCatalogTest {
     Set<UpgradeCatalog> upgradeCatalogSet = schemaUpgradeHelper.getAllUpgradeCatalogs();
 
     Assert.assertNotNull(upgradeCatalogSet);
-    Assert.assertEquals(4, upgradeCatalogSet.size());
+    Assert.assertEquals(3, upgradeCatalogSet.size());
 
     List<UpgradeCatalog> upgradeCatalogs = schemaUpgradeHelper.getUpgradePath(null, "2.5.1");
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
index 8e03b7f..8c36ad6 100644
--- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
+++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
@@ -243,19 +243,7 @@ class TestRUSetAll(RMFTestCase):
       ru_execute.unlink_all_configs(None)
 
       # verify that os.path.islink was called for each conf
-      self.assertTrue(islink_mock.called)
-      for key, value in conf_select.get_package_dirs().iteritems():
-        for directory_mapping in value:
-          original_config_directory = directory_mapping['conf_dir']
-          is_link_called = False
-
-          for call in islink_mock.call_args_list:
-            call_tuple = call[0]
-            if original_config_directory in call_tuple:
-              is_link_called = True
-
-          if not is_link_called:
-            self.fail("os.path.islink({0}) was never called".format(original_config_directory))
+      self.assertFalse(islink_mock.called)
 
       # alter JSON for a downgrade from 2.3 to 2.3
       with open(json_file_path, "r") as json_file:
@@ -313,37 +301,6 @@ class TestRUSetAll(RMFTestCase):
       # ensure it wasn't called this time
       self.assertFalse(islink_mock.called)
 
-  @patch("os.path.isdir")
-  @patch("os.path.islink")
-  def test_unlink_configs_missing_backup(self, islink_mock, isdir_mock):
-
-    # required for the test to run since the Execute calls need this
-    from resource_management.core.environment import Environment
-    env = Environment(test_mode=True)
-    with env:
-      # Case: missing backup directory
-      isdir_mock.return_value = False
-      ru_execute = UpgradeSetAll()
-      self.assertEqual(len(env.resource_list), 0)
-      # Case: missing symlink
-      isdir_mock.reset_mock()
-      isdir_mock.return_value = True
-      islink_mock.return_value = False
-      ru_execute._unlink_config("/fake/config")
-      self.assertEqual(len(env.resource_list), 2)
-      # Case: missing symlink
-      isdir_mock.reset_mock()
-      isdir_mock.return_value = True
-      islink_mock.reset_mock()
-      islink_mock.return_value = True
-
-      ru_execute._unlink_config("/fake/config")
-      self.assertEqual(pprint.pformat(env.resource_list),
-                       "[Directory['/fake/config'],\n "
-                       "Execute[('mv', '/fake/conf.backup', '/fake/config')],\n "
-                       "Execute[('rm', '/fake/config')],\n "
-                       "Execute[('mv', '/fake/conf.backup', '/fake/config')]]")
-
   @patch("os.path.exists")
   @patch("os.path.islink")
   @patch("os.path.isdir")
@@ -416,4 +373,4 @@ class TestRUSetAll(RMFTestCase):
       self.assertResourceCalled('Link', '/old/config',
                                 to = '/link/config',
                                 )
-      self.assertNoMoreResources()
\ No newline at end of file
+      self.assertNoMoreResources()

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml
new file mode 100644
index 0000000..3e61696
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-pack.xsd">
+  <target>2.2.*.*</target>
+  <target-stack>HDP-2.2.0</target-stack>
+  <type>ROLLING</type>
+  <prerequisite-checks/>
+
+  <order>
+    <group xsi:type="cluster" name="CLUSTER_SERVER_ACTIONS" title="Cluster Server Actions">
+      <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Execute Foo/Bar Function">
+        <task xsi:type="execute">
+          <script>scripts/foo.py</script>
+          <function>bar</function>
+        </task>
+      </execute-stage>
+    </group>
+  </order>
+  
+  <processing>
+    <service name="ZOOKEEPER">
+      <component name="ZOOKEEPER_SERVER">
+        <upgrade />
+      </component>
+    </service>
+  </processing>
+</upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml
new file mode 100644
index 0000000..6b08f27
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml
@@ -0,0 +1,267 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-pack.xsd">
+  <target>2.2.*.*</target>
+  <target-stack>HDP-2.5.0</target-stack>
+  <type>ROLLING</type>
+  <prerequisite-checks>
+    <!-- List of additional pre-req checks to run in addition to the required pre-reqs -->
+    <check>org.apache.ambari.server.checks.HiveMultipleMetastoreCheck</check>
+    <check>org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck</check>
+    <check>org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck</check>
+    <check>org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck</check>
+    <check>org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck</check>
+    <check>org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck</check>
+    <check>org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck</check>
+    <check>org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck</check>
+    <check>org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck</check>
+  </prerequisite-checks>
+  
+  <order>
+    <group xsi:type="cluster" name="PRE_CLUSTER" title="Pre {{direction.text.proper}}">
+      <execute-stage title="Confirm 1">
+        <task xsi:type="manual">
+          <message>Foo</message>
+        </task>
+      </execute-stage>
+      <execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HIVE">
+        <task xsi:type="manual">
+          <message>Back stuff up.</message>
+        </task>
+      </execute-stage>
+      <execute-stage service="HDFS" component="NAMENODE" title="Finalize HDFS">
+        <task xsi:type="execute">
+          <script>foo</script>
+          <function>list</function>
+        </task>
+      </execute-stage>
+      <execute-stage title="Confirm 2">
+        <task xsi:type="manual">
+          <message>Foo</message>
+        </task>
+      </execute-stage>
+    </group>
+  
+    <group name="ZOOKEEPER" title="Zookeeper">
+      <skippable>true</skippable>
+      <allow-retry>false</allow-retry>
+      <service name="ZOOKEEPER">
+        <component>ZOOKEEPER_SERVER</component>
+        <component>ZOOKEEPER_CLIENT</component>
+      </service>
+    </group>
+    
+    <group name="CORE_MASTER" title="Core Masters">
+      <service name="HDFS">
+        <component>JOURNALNODE</component>
+        <component>NAMENODE</component>
+      </service>
+      <service name="YARN">
+        <component>RESOURCEMANAGER</component>
+      </service>
+    </group>
+    
+    <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated">
+      <skippable>true</skippable>      <!-- set skippable for test -->
+      <allow-retry>false</allow-retry> <!-- set no retry for test -->
+      <service name="HDFS">
+        <component>DATANODE</component>
+      </service>
+      <service name="HBASE">
+        <component>REGIONSERVER</component>
+      </service>
+      <service name="YARN">
+        <component>NODEMANAGER</component>
+      </service>
+      
+      <batch>
+        <percent>20</percent>
+        <message>Please run additional tests on {{components}}</message>
+      </batch>
+    </group>
+    
+    <group name="HIVE" title="Hive">
+      <skippable>true</skippable>
+      <service name="HIVE">
+        <component>HIVE_METASTORE</component>
+        <component>HIVE_SERVER</component>
+        <component>WEBHCAT_SERVER</component>
+      </service>
+    </group>
+
+    <group name="OOZIE" title="Oozie">
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <service-check>false</service-check>
+      <service name="OOZIE">
+        <component>OOZIE_SERVER</component>
+        <component>OOZIE_CLIENT</component>
+      </service>
+    </group>
+    
+    <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
+      <execute-stage title="Confirm Finalize">
+        <task xsi:type="manual">
+          <message>Please confirm you are ready to finalize</message>
+        </task>
+      </execute-stage>
+      <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize">
+        <task xsi:type="execute">
+          <script>foo</script>
+          <function>list</function>
+        </task>
+      </execute-stage>
+      <execute-stage title="Save Cluster State">
+        <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction">
+        </task>
+      </execute-stage>
+    </group>
+        
+  </order>
+  
+
+  <processing>
+    <service name="ZOOKEEPER">
+      <component name="ZOOKEEPER_SERVER">
+        <pre-upgrade>
+          <task xsi:type="manual">
+            <summary>SUMMARY OF PREPARE</summary>
+            <message>This is a manual task with a placeholder of {{foo/bar}}</message>
+          </task>
+        </pre-upgrade>
+        <pre-downgrade copy-upgrade="true" />
+        <upgrade>
+          <task xsi:type="restart-task" timeout-config="upgrade.parameter.zk-server.timeout"/>
+        </upgrade>
+        <post-upgrade>
+          <task xsi:type="configure" id="hdp_2_1_1_zookeeper_new_config_type" />
+        </post-upgrade>
+        <post-downgrade copy-upgrade="true" />
+      </component>
+    </service>
+    
+    <service name="HDFS">
+      <component name="NAMENODE">
+        <pre-upgrade>
+          <task xsi:type="execute" hosts="master">
+            <script>foo</script>
+            <function>list</function>
+          </task>
+          <task xsi:type="configure" id="hdp_2_1_1_nn_pre_upgrade" />
+          <task xsi:type="manual">
+            <message>{{direction.verb.proper}} your database</message>
+          </task>
+        </pre-upgrade>
+        <pre-downgrade copy-upgrade="true" />
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+        <post-upgrade>
+          <task xsi:type="execute">
+            <script>foo</script>
+            <function>list</function>
+          </task>
+        </post-upgrade>
+        <post-downgrade copy-upgrade="true" />
+      </component>
+      <component name="DATANODE">
+        <pre-downgrade />
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+        <post-downgrade>
+          <task xsi:type="manual">
+            <message>Manual Downgrade</message>
+          </task>
+        </post-downgrade>
+      </component>
+    </service>
+    
+    <service name="YARN">
+      <component name="RESOURCEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="execute">
+            <script>foo</script>
+            <function>list</function>
+          </task>
+        </pre-upgrade>
+        <pre-downgrade copy-upgrade="true" />
+        <upgrade />
+      </component>
+      <component name="NODEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="execute">
+            <script>foo</script>
+            <function>list</function>
+          </task>
+          <task xsi:type="configure" id="hdp_2_1_1_nm_pre_upgrade"/>
+        </pre-upgrade>
+        <pre-downgrade copy-upgrade="true" />
+        <upgrade />
+      </component>
+    </service>
+    
+    <service name="HIVE">
+      <component name="HIVE_SERVER">
+        <pre-upgrade>
+          <task xsi:type="manual">
+            <summary>HiveServer Port Availability</summary>
+            <message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message>
+          </task>
+          <task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/>
+          <task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions"/>
+          <task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions_skip"/>
+          <task xsi:type="configure" id="hdp_2_1_1_no_conditions_met"/>
+        </pre-upgrade>
+        <pre-downgrade copy-upgrade="true" />
+        <upgrade />
+       </component>
+     </service>
+
+    <service name="OOZIE">
+      <component name="OOZIE_SERVER">
+        <pre-upgrade>
+          <!-- This is important, do not remove it since UpgradeHelperTest.java :
+          testUpgradeWithMultipleTasksWithMultipleHostTypes() asserts
+          that these tasks each run on their own stage. -->
+          <task xsi:type="execute" hosts="all" sequential="true">
+            <summary>Shut down all Oozie servers</summary>
+            <script>scripts/oozie_server.py</script>
+            <function>stop</function>
+          </task>
+
+          <task xsi:type="execute" hosts="any" sequential="true">
+            <summary>Upgrading the Oozie database and creating a new sharelib</summary>
+            <script>scripts/oozie_server_upgrade.py</script>
+            <function>upgrade_oozie_database_and_sharelib</function>
+          </task>
+        </pre-upgrade>
+        <pre-downgrade copy-upgrade="true" />
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+
+      <component name="OOZIE_CLIENT">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+    </service>
+  </processing>
+</upgrade>


Mime
View raw message