ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [45/50] [abbrv] ambari git commit: Merge remote-tracking branch 'remotes/origin/trunk' into branch-3.0-perf
Date Thu, 28 Sep 2017 13:25:34 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
index 0000000,30bc47f..697f1d1
mode 000000,100644..100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
@@@ -1,0 -1,573 +1,573 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.ambari.server.serveraction.upgrades;
+ 
+ import static org.apache.ambari.server.controller.KerberosHelper.DEFAULT_REALM;
+ import static org.apache.ambari.server.controller.KerberosHelper.KERBEROS_ENV;
+ import static org.apache.ambari.server.controller.KerberosHelper.PRECONFIGURE_SERVICES;
+ 
+ import java.io.IOException;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+ import java.util.concurrent.ConcurrentMap;
+ 
+ import org.apache.ambari.server.AmbariException;
+ import org.apache.ambari.server.actionmanager.HostRoleStatus;
+ import org.apache.ambari.server.agent.CommandReport;
+ import org.apache.ambari.server.controller.AmbariManagementController;
+ import org.apache.ambari.server.controller.KerberosHelper;
+ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+ import org.apache.ambari.server.serveraction.kerberos.PreconfigureServiceType;
+ import org.apache.ambari.server.state.Cluster;
+ import org.apache.ambari.server.state.ConfigHelper;
+ import org.apache.ambari.server.state.Host;
+ import org.apache.ambari.server.state.SecurityType;
+ import org.apache.ambari.server.state.Service;
+ import org.apache.ambari.server.state.ServiceComponentHost;
+ import org.apache.ambari.server.state.StackId;
+ import org.apache.ambari.server.state.UpgradeContext;
+ import org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptorContainer;
+ import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
+ import org.apache.ambari.server.state.kerberos.KerberosConfigurationDescriptor;
+ import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
+ import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
+ import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
+ import org.apache.ambari.server.state.kerberos.VariableReplacementHelper;
+ import org.apache.ambari.server.state.stack.upgrade.Direction;
+ import org.apache.commons.collections.CollectionUtils;
+ import org.apache.commons.collections.MapUtils;
+ import org.apache.commons.lang.StringUtils;
+ 
+ import com.google.inject.Inject;
+ 
+ /**
+  * PreconfigureKerberos updates existing service configurations with properties from service-level
+  * Kerberos descriptors, flagged for pre-configuring, during stack upgrades in order to prevent service
+  * restarts when the flagged services are installed.
+  */
+ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction {
+   static final String UPGRADE_DIRECTION_KEY = "upgrade_direction";
+ 
+   @Inject
+   private AmbariManagementController ambariManagementController;
+ 
+   @Inject
+   private KerberosHelper kerberosHelper;
+ 
+   @Inject
+   private ConfigHelper configHelper;
+ 
+   @Inject
+   private VariableReplacementHelper variableReplacementHelper;
+ 
+   @Override
+   public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) throws AmbariException, InterruptedException {
+     Map<String, String> commandParameters = getCommandParameters();
+     if (null == commandParameters || commandParameters.isEmpty()) {
+       return createCommandReport(0, HostRoleStatus.FAILED, "{}", "",
+           "Unable to change configuration values without command parameters");
+     }
+ 
+     if (!isDowngrade()) {
+       String clusterName = commandParameters.get("clusterName");
+       Cluster cluster = m_clusters.getCluster(clusterName);
+ 
+       if (cluster.getSecurityType() == SecurityType.KERBEROS) {
+         StackId stackId;
+ 
+         try {
+           stackId = getTargetStackId(cluster);
+         } catch (AmbariException e) {
+           return createCommandReport(0, HostRoleStatus.FAILED, "{}", "", e.getLocalizedMessage());
+         }
+ 
+         if (stackId == null) {
+           return createCommandReport(0, HostRoleStatus.FAILED, "{}", "",
+               "The target stack Id was not specified.");
+         }
+ 
+         KerberosDescriptor kerberosDescriptor = kerberosHelper.getKerberosDescriptor(KerberosHelper.KerberosDescriptorType.COMPOSITE, cluster, stackId, true);
+ 
+         // Calculate the current host-specific configurations. These will be used to replace
+         // variables within the Kerberos descriptor data
+         Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, true, false);
+ 
+         PreconfigureServiceType preconfigureServiceType = getPreconfigureServiceType(configurations);
+ 
+         if (preconfigureServiceType != PreconfigureServiceType.NONE) {
+           Map<String, Map<String, String>> kerberosConfigurations = new HashMap<>();
+           Map<String, Set<String>> propertiesToRemove = new HashMap<>();
+           Map<String, Set<String>> propertiesToIgnore = new HashMap<>();
+ 
+           if (preconfigureServiceType == PreconfigureServiceType.ALL) {
+             // Force all services to be flagged for pre-configuration...
+             Map<String, KerberosServiceDescriptor> serviceDescriptors = kerberosDescriptor.getServices();
+             if (serviceDescriptors != null) {
+               for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors.values()) {
+                 serviceDescriptor.setPreconfigure(true);
+               }
+             }
+           }
+ 
+           processServiceComponentHosts(cluster, kerberosDescriptor, configurations, kerberosConfigurations, propertiesToIgnore);
+ 
+           // Calculate the set of configurations to update and replace any variables
+           // using the previously calculated Map of configurations for the host.
+           kerberosConfigurations = kerberosHelper.processPreconfiguredServiceConfigurations(kerberosConfigurations, configurations, cluster, kerberosDescriptor);
+ 
+           Map<String, Set<String>> installedServices = calculateInstalledServices(cluster);
+ 
+           kerberosHelper.applyStackAdvisorUpdates(cluster, installedServices.keySet(), configurations, kerberosConfigurations,
+               propertiesToIgnore, propertiesToRemove, true);
+ 
+           kerberosHelper.setAuthToLocalRules(cluster, kerberosDescriptor, getDefaultRealm(configurations), installedServices,
+               configurations, kerberosConfigurations, true);
+ 
+           processConfigurationChanges(cluster, stackId, kerberosDescriptor, kerberosConfigurations, propertiesToRemove, configurations);
+         } else {
+           actionLog.writeStdOut("Skipping: This facility is only available when kerberos-env/preconfigure_services is not \"NONE\"");
+         }
+       } else {
+         actionLog.writeStdOut("Skipping: This facility is only available when Kerberos is enabled");
+       }
+     } else {
+       actionLog.writeStdOut("Skipping: This facility is only available during an upgrade");
+     }
+ 
+     return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
+   }
+ 
+   /**
+    * Given a Cluster object creates a map of service names to sets of the installed components for that
+    * service.
+    *
+    * @param cluster the cluster
+    * @return a map of (installed) service names to the relevant set of (installed) component names
+    */
+   private Map<String, Set<String>> calculateInstalledServices(Cluster cluster) {
+     Map<String, Set<String>> installedServices = new HashMap<>();
+     Map<String, Service> services = cluster.getServices();
+ 
+     for (Service service : services.values()) {
+       installedServices.put(service.getName(), service.getServiceComponents().keySet());
+     }
+ 
+     return installedServices;
+   }
+ 
+   /**
+    * Safely retrieves the specified property from the specified configuration type from a map of
+    * configurations.
+    *
+    * @param configurations the existing configurations for the cluster
+    * @return the requested value or null if the configuration does not exist
+    */
+   private String getValueFromConfiguration(Map<String, Map<String, String>> configurations, String configType, String propertyName) {
+     String value = null;
+ 
+     if (configurations != null) {
+       Map<String, String> kerberosEnv = configurations.get(configType);
+ 
+       if (kerberosEnv != null) {
+         value = kerberosEnv.get(propertyName);
+       }
+     }
+ 
+     return value;
+   }
+ 
+   /**
+    * Safely retrieves the <code>realm</code> property of the <code>kerberos-env</code> configuration.
+    *
+    * @param configurations the existing configurations for the cluster
+    * @return the requested value or null if the configuration does not exist
+    * @see #getValueFromConfiguration(Map, String, String)
+    */
+   private String getDefaultRealm(Map<String, Map<String, String>> configurations) {
+     return getValueFromConfiguration(configurations, KERBEROS_ENV, DEFAULT_REALM);
+   }
+ 
+   /**
+    * Safely retrieves the <code>preconfigure_services</code> property of the <code>kerberos-env</code> configuration.
+    *
+    * @param configurations the existing configurations for the cluster
+    * @return the requested value or null if the configuration does not exist
+    * @see #getValueFromConfiguration(Map, String, String)
+    */
+   private PreconfigureServiceType getPreconfigureServiceType(Map<String, Map<String, String>> configurations) {
+     String preconfigureServices = getValueFromConfiguration(configurations, KERBEROS_ENV, PRECONFIGURE_SERVICES);
+ 
+     PreconfigureServiceType preconfigureServiceType = null;
+     if (!StringUtils.isEmpty(preconfigureServices)) {
+       try {
+         preconfigureServiceType = PreconfigureServiceType.valueOf(preconfigureServices.toUpperCase());
+       } catch (Throwable t) {
+         preconfigureServiceType = PreconfigureServiceType.DEFAULT;
+       }
+     }
+ 
+     return (preconfigureServiceType == null) ? PreconfigureServiceType.DEFAULT : preconfigureServiceType;
+   }
+ 
+   /**
+    * Determines if upgrade direction is {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}.
+    *
+    * @return {@code true} if {@link Direction#DOWNGRADE}; {@code false} if {@link Direction#UPGRADE}
+    */
+   private boolean isDowngrade() {
+     return Direction.DOWNGRADE.name().equalsIgnoreCase(getCommandParameterValue(UPGRADE_DIRECTION_KEY));
+   }
+ 
+   /**
+    * Retrieves the target stack ID for the stack upgrade or downgrade operation.
+    *
+    * @param cluster the cluster
+    * @return the target {@link StackId}
+    * @throws AmbariException if multiple stack id's are detected
+    */
+   private StackId getTargetStackId(Cluster cluster) throws AmbariException {
+     UpgradeContext upgradeContext = getUpgradeContext(cluster);
+ 
+     // !!! FIXME in a per-service view, what does this become?
+     Set<StackId> stackIds = new HashSet<>();
+ 
+     for (Service service : cluster.getServices().values()) {
+       RepositoryVersionEntity targetRepoVersion = upgradeContext.getTargetRepositoryVersion(service.getName());
+       StackId targetStackId = targetRepoVersion.getStackId();
+       stackIds.add(targetStackId);
+     }
+ 
+     if (1 != stackIds.size()) {
+       throw new AmbariException("Services are deployed from multiple stacks and cannot determine a unique one.");
+     }
+ 
+     return stackIds.iterator().next();
+   }
+ 
+   /**
+    * Find and iterate through the {@link ServiceComponentHost} objects for the current {@link Cluster}
+    * to calculate property updates and auth-to-local rules.
+    *
+    * @param cluster                the cluster
+    * @param kerberosDescriptor     the Kerberos descriptor
+    * @param currentConfigurations  the current configurations for the cluster
+    * @param kerberosConfigurations the (Kerberos-specific) configuration updates
+    * @param propertiesToBeIgnored  a map to store properties that should be ignored by operations that update property values
+    * @throws AmbariException if an issue occurs
+    */
+   private void processServiceComponentHosts(Cluster cluster, KerberosDescriptor kerberosDescriptor,
+                                             Map<String, Map<String, String>> currentConfigurations,
+                                             Map<String, Map<String, String>> kerberosConfigurations,
+                                             Map<String, Set<String>> propertiesToBeIgnored)
+       throws AmbariException {
+ 
+     Collection<Host> hosts = cluster.getHosts();
+     if (!hosts.isEmpty()) {
+       // Create the context to use for filtering Kerberos Identities based on the state of the cluster
+       Map<String, Object> filterContext = new HashMap<>();
+       filterContext.put("configurations", currentConfigurations);
+       filterContext.put("services", cluster.getServices().keySet());
+ 
+       try {
+         Map<String, Set<String>> propertiesToIgnore = null;
+ 
+         for (Host host : hosts) {
+           // Iterate over the components installed on the current host to get the service and
+           // component-level Kerberos descriptors in order to determine which principals,
+           // keytab files, and configurations need to be created or updated.
+           for (ServiceComponentHost sch : cluster.getServiceComponentHosts(host.getHostName())) {
+             String hostName = sch.getHostName();
+ 
+             String serviceName = sch.getServiceName();
+             String componentName = sch.getServiceComponentName();
+ 
+             KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(serviceName);
+ 
+             if (serviceDescriptor != null) {
+               List<KerberosIdentityDescriptor> serviceIdentities = serviceDescriptor.getIdentities(true, filterContext);
+ 
+               // Add service-level principals (and keytabs)
+               kerberosHelper.addIdentities(null, serviceIdentities,
 -                  null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, false);
++                  null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations);
+               propertiesToIgnore = gatherPropertiesToIgnore(serviceIdentities, propertiesToIgnore);
+ 
+               KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(componentName);
+ 
+               if (componentDescriptor != null) {
+                 List<KerberosIdentityDescriptor> componentIdentities = componentDescriptor.getIdentities(true, filterContext);
+ 
+                 // Calculate the set of configurations to update and replace any variables
+                 // using the previously calculated Map of configurations for the host.
+                 kerberosHelper.mergeConfigurations(kerberosConfigurations,
+                     componentDescriptor.getConfigurations(true), currentConfigurations, null);
+ 
+                 // Add component-level principals (and keytabs)
+                 kerberosHelper.addIdentities(null, componentIdentities,
 -                    null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, false);
++                    null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations);
+                 propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore);
+               }
+             }
+           }
+         }
+ 
+         // Add ambari-server identities only if 'kerberos-env.create_ambari_principal = true'
+         if (kerberosHelper.createAmbariIdentities(currentConfigurations.get(KERBEROS_ENV))) {
+           List<KerberosIdentityDescriptor> ambariIdentities = kerberosHelper.getAmbariServerIdentities(kerberosDescriptor);
+ 
+           for (KerberosIdentityDescriptor identity : ambariIdentities) {
+             // If the identity represents the ambari-server user, use the component name "AMBARI_SERVER_SELF"
+             // so it can be distinguished between other identities related to the AMBARI-SERVER
+             // component.
+             String componentName = KerberosHelper.AMBARI_SERVER_KERBEROS_IDENTITY_NAME.equals(identity.getName())
+                 ? "AMBARI_SERVER_SELF"
+                 : "AMBARI_SERVER";
+ 
+             List<KerberosIdentityDescriptor> componentIdentities = Collections.singletonList(identity);
+             kerberosHelper.addIdentities(null, componentIdentities,
 -                null, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations, false);
++                null, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations);
+             propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore);
+           }
+         }
+ 
+         if ((propertiesToBeIgnored != null) && (propertiesToIgnore != null)) {
+           propertiesToBeIgnored.putAll(propertiesToIgnore);
+         }
+       } catch (IOException e) {
+         throw new AmbariException(e.getMessage(), e);
+       }
+     }
+   }
+ 
+   private Map<String, Set<String>> gatherPropertiesToIgnore(List<KerberosIdentityDescriptor> identities,
+                                                             Map<String, Set<String>> propertiesToIgnore) {
+     Map<String, Map<String, String>> identityConfigurations = kerberosHelper.getIdentityConfigurations(identities);
+     if (!MapUtils.isEmpty(identityConfigurations)) {
+       if (propertiesToIgnore == null) {
+         propertiesToIgnore = new HashMap<>();
+       }
+ 
+       for (Map.Entry<String, Map<String, String>> entry : identityConfigurations.entrySet()) {
+         String configType = entry.getKey();
+         Map<String, String> properties = entry.getValue();
+ 
+         if (MapUtils.isEmpty(properties)) {
+           Set<String> propertyNames = propertiesToIgnore.get(configType);
+           if (propertyNames == null) {
+             propertyNames = new HashSet<>();
+             propertiesToIgnore.put(configType, propertyNames);
+           }
+           propertyNames.addAll(properties.keySet());
+         }
+       }
+     }
+ 
+     return propertiesToIgnore;
+   }
+ 
+   /**
+    * Processes configuration changes to determine if any work needs to be done.
+    * <p/>
+    * If work is to be done, a data file containing the details is created so it they changes may be
+    * processed in the appropriate stage.
+    *
+    * @param cluster                the cluster
+    * @param targetStackId          the target stack id
+    * @param kerberosConfigurations the Kerberos-specific configuration map
+    * @param propertiesToBeRemoved  a map of properties to be removed from the current configuration,
+    *                               grouped by configuration type.
+    * @param variableReplaments     replacement values to use when attempting to perform variable replacements on the property names
+    * @throws AmbariException if an issue is encountered
+    */
+   private void processConfigurationChanges(Cluster cluster, StackId targetStackId,
+                                            KerberosDescriptor kerberosDescriptor,
+                                            Map<String, Map<String, String>> kerberosConfigurations,
+                                            Map<String, Set<String>> propertiesToBeRemoved,
+                                            Map<String, Map<String, String>> variableReplaments)
+       throws AmbariException {
+     actionLog.writeStdOut("Determining configuration changes");
+ 
+     if (!kerberosConfigurations.isEmpty()) {
+       Map<String, Service> installedServices = cluster.getServices();
+ 
+       // Build a map of configuration types to properties that indicate which properties should be altered
+       // This map should contain only properties defined in service-level Kerberos descriptors that
+       // have been flagged to be preconfigured and that have not yet been installed.
+       Map<String, Set<String>> propertyFilter = new HashMap<>();
+       Map<String, KerberosServiceDescriptor> serviceDescriptors = kerberosDescriptor.getServices();
+       if (serviceDescriptors != null) {
+         for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors.values()) {
+           if (!installedServices.containsKey(serviceDescriptor.getName()) && serviceDescriptor.shouldPreconfigure()) {
+             buildFilter(Collections.singleton(serviceDescriptor), propertyFilter, variableReplaments);
+           }
+         }
+       }
+ 
+       // Add the auth-to-local rule configuration specifications to the filter
+       Map<String, Set<String>> authToLocalProperties = kerberosHelper.translateConfigurationSpecifications(kerberosDescriptor.getAllAuthToLocalProperties());
+       if (!MapUtils.isEmpty(authToLocalProperties)) {
+         for (Map.Entry<String, Set<String>> entry : authToLocalProperties.entrySet()) {
+           Set<String> properties = entry.getValue();
+ 
+           if (!CollectionUtils.isEmpty(properties)) {
+             String configurationType = entry.getKey();
+ 
+             Set<String> propertyNames = propertyFilter.get(configurationType);
+             if (propertyNames == null) {
+               propertyNames = new HashSet<>();
+               propertyFilter.put(configurationType, propertyNames);
+             }
+ 
+             propertyNames.addAll(properties);
+           }
+         }
+       }
+ 
+       Set<String> visitedTypes = new HashSet<>();
+ 
+       for (Map.Entry<String, Map<String, String>> entry : kerberosConfigurations.entrySet()) {
+         String configType = entry.getKey();
+ 
+         String service = cluster.getServiceByConfigType(configType);
+         Set<String> allowedProperties = propertyFilter.get(configType);
+ 
+         // Update properties for services that are installed and not filtered out
+         if (installedServices.containsKey(service) && !CollectionUtils.isEmpty(allowedProperties)) {
+           Map<String, String> propertiesToUpdate = entry.getValue();
+           Set<String> propertiesToRemove = (propertiesToBeRemoved == null) ? null : propertiesToBeRemoved.get(configType);
+ 
+           // Filter the properties to update
+           if (propertiesToUpdate != null) {
+             Iterator<Map.Entry<String, String>> mapIterator = propertiesToUpdate.entrySet().iterator();
+             while (mapIterator.hasNext()) {
+               Map.Entry<String, String> mapEntry = mapIterator.next();
+ 
+               if (!allowedProperties.contains(mapEntry.getKey())) {
+                 mapIterator.remove();
+               }
+             }
+           }
+ 
+           // Filter the properties to remove
+           if (propertiesToRemove != null) {
+             Iterator<String> setIterator = propertiesToRemove.iterator();
+             while (setIterator.hasNext()) {
+               String setEntry = setIterator.next();
+               if (!allowedProperties.contains(setEntry)) {
+                 setIterator.remove();
+               }
+             }
+           }
+ 
+           visitedTypes.add(configType);
+ 
+           if (!MapUtils.isEmpty(propertiesToUpdate) || !CollectionUtils.isEmpty(propertiesToRemove)) {
+             if (!MapUtils.isEmpty(propertiesToUpdate)) {
+               for (Map.Entry<String, String> property : propertiesToUpdate.entrySet()) {
+                 actionLog.writeStdOut(String.format("Setting: %s/%s = %s", configType, property.getKey(), property.getValue()));
+               }
+             }
+ 
+             if (!CollectionUtils.isEmpty(propertiesToRemove)) {
+               for (String property : propertiesToRemove) {
+                 actionLog.writeStdOut(String.format("Removing: %s/%s", configType, property));
+               }
+             }
+ 
+             configHelper.updateConfigType(cluster, targetStackId,
+                 ambariManagementController, configType, propertiesToUpdate, propertiesToRemove,
+                 ambariManagementController.getAuthName(), "Preconfiguring for Kerberos during upgrade");
+           }
+         }
+       }
+ 
+       if (!MapUtils.isEmpty(propertiesToBeRemoved)) {
+         for (Map.Entry<String, Set<String>> entry : propertiesToBeRemoved.entrySet()) {
+           String configType = entry.getKey();
+ 
+           if (!visitedTypes.contains(configType)) {
+             Set<String> propertiesToRemove = entry.getValue();
+ 
+             if (!CollectionUtils.isEmpty(propertiesToRemove)) {
+               for (String property : propertiesToRemove) {
+                 actionLog.writeStdOut(String.format("Removing: %s/%s", configType, property));
+               }
+ 
+               configHelper.updateConfigType(cluster, targetStackId,
+                   ambariManagementController, configType, null, entry.getValue(),
+                   ambariManagementController.getAuthName(), "Preconfiguring for Kerberos during upgrade");
+             }
+           }
+         }
+       }
+     }
+   }
+ 
+   /**
+    * Adds entries to the property filter (<code>propertyFilter</code>) found in the {@link KerberosConfigurationDescriptor}s
+    * within the specified node of the Kerberos descriptor.
+    *
+    * @param containers     the Kerberos descriptor containers to process
+    * @param propertyFilter the property filter map to update
+    * @param replacements   replacement values to use when attempting to perform variable replacements on the property names
+    * @throws AmbariException if an issue occurs while replacing variables in the property names
+    */
+   private void buildFilter(Collection<? extends AbstractKerberosDescriptorContainer> containers,
+                            Map<String, Set<String>> propertyFilter,
+                            Map<String, Map<String, String>> replacements)
+       throws AmbariException {
+     if (containers != null) {
+       for (AbstractKerberosDescriptorContainer container : containers) {
+         Map<String, KerberosConfigurationDescriptor> configurationDescriptors = container.getConfigurations(false);
+ 
+         if (!MapUtils.isEmpty(configurationDescriptors)) {
+           for (KerberosConfigurationDescriptor configurationDescriptor : configurationDescriptors.values()) {
+             Map<String, String> properties = configurationDescriptor.getProperties();
+ 
+             if (!MapUtils.isEmpty(properties)) {
+               String configType = configurationDescriptor.getType();
+ 
+               Set<String> propertyNames = propertyFilter.get(configType);
+               if (propertyNames == null) {
+                 propertyNames = new HashSet<>();
+                 propertyFilter.put(configType, propertyNames);
+               }
+ 
+               // Replace variables in the property name. For example ${knox-env/knox_user}.
+               for (String propertyName : properties.keySet()) {
+                 propertyNames.add(variableReplacementHelper.replaceVariables(propertyName, replacements));
+               }
+             }
+           }
+         }
+ 
+         Collection<? extends AbstractKerberosDescriptorContainer> childContainers = container.getChildContainers();
+         if (childContainers != null) {
+           buildFilter(childContainers, propertyFilter, replacements);
+         }
+       }
+     }
+   }
+ }
+ 

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java
index f68570b,2992027..db9d178
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java
@@@ -25,13 -29,17 +25,11 @@@ import java.util.concurrent.ExecutorSer
  import java.util.concurrent.Executors;
  import java.util.concurrent.Future;
  import java.util.concurrent.ThreadFactory;
 -import java.util.concurrent.TimeUnit;
  
- import org.apache.ambari.server.api.services.AmbariMetaInfo;
  import org.apache.ambari.server.metadata.ActionMetadata;
  import org.apache.ambari.server.orm.dao.MetainfoDAO;
- import org.apache.ambari.server.orm.entities.MetainfoEntity;
 +import org.apache.ambari.server.state.stack.LatestRepoCallable;
  import org.apache.ambari.server.state.stack.OsFamily;
 -import org.apache.ambari.server.state.stack.RepoUrlInfoCallable;
 -import org.apache.ambari.server.state.stack.RepoUrlInfoCallable.RepoUrlInfoResult;
 -import org.apache.ambari.server.state.stack.RepoVdfCallable;
 -import org.apache.commons.collections.MapUtils;
 -import org.slf4j.Logger;
 -import org.slf4j.LoggerFactory;
  
  /**
   * Provides external functionality to the Stack framework.

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index d3ad351,6dc2b93..3688727
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@@ -1236,6 -1276,9 +1232,9 @@@ public class StackModule extends BaseMo
          RepositoryXml serviceRepoXml = ssd.getRepoFile();
          if (null != serviceRepoXml) {
            repos.addAll(serviceRepoXml.getRepositories());
+           if (null != serviceRepoXml.getLatestURI()) {
 -            registerRepoUpdateTask(serviceRepoXml);
++            stackContext.registerRepoUpdateTask(serviceRepoXml.getLatestURI(), this);
+           }
          }
        }
      }

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
index 3e04a87,90dd611..23f4078
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
@@@ -26,12 -26,8 +26,9 @@@ import java.util.Set
  import org.apache.ambari.server.AmbariException;
  import org.apache.ambari.server.controller.ClusterResponse;
  import org.apache.ambari.server.controller.ServiceConfigVersionResponse;
 +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
  import org.apache.ambari.server.events.ClusterConfigChangedEvent;
  import org.apache.ambari.server.metadata.RoleCommandOrder;
- import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
- import org.apache.ambari.server.orm.entities.HostEntity;
- import org.apache.ambari.server.orm.entities.HostVersionEntity;
  import org.apache.ambari.server.orm.entities.PrivilegeEntity;
  import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
  import org.apache.ambari.server.orm.entities.UpgradeEntity;

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/Clusters.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
index 31a00ca,8ab1fe9..d57b5d6
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
@@@ -29,9 -29,10 +29,8 @@@ public class RepositoryInfo 
    private String osType;
    private String repoId;
    private String repoName;
 -  private String distribution;
 -  private String components;
    private String mirrorsList;
    private String defaultBaseUrl;
-   private String latestBaseUrl;
    private boolean repoSaved = false;
    private boolean unique = false;
    private boolean ambariManagedRepositories = true;
@@@ -186,21 -191,23 +171,20 @@@
          Objects.equal(osType, that.osType) &&
          Objects.equal(repoId, that.repoId) &&
          Objects.equal(repoName, that.repoName) &&
 -        Objects.equal(distribution, that.distribution) &&
 -        Objects.equal(components, that.components) &&
          Objects.equal(mirrorsList, that.mirrorsList) &&
          Objects.equal(defaultBaseUrl, that.defaultBaseUrl) &&
-         Objects.equal(latestBaseUrl, that.latestBaseUrl) &&
          Objects.equal(ambariManagedRepositories, that.ambariManagedRepositories);
    }
  
    @Override
    public int hashCode() {
-     return Objects.hashCode(baseUrl, osType, repoId, repoName, mirrorsList, defaultBaseUrl, latestBaseUrl, repoSaved, unique, ambariManagedRepositories);
 -    return Objects.hashCode(baseUrl, osType, repoId, repoName, distribution, components, mirrorsList, defaultBaseUrl,
 -           ambariManagedRepositories);
++    return Objects.hashCode(baseUrl, osType, repoId, repoName, mirrorsList, defaultBaseUrl, repoSaved, unique, ambariManagedRepositories);
    }
  
    public RepositoryResponse convertToResponse()
    {
      return new RepositoryResponse(getBaseUrl(), getOsType(), getRepoId(),
-         getRepoName(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl());
 -            getRepoName(), getDistribution(), getComponents(), getMirrorsList(), getDefaultBaseUrl());
++        getRepoName(), getMirrorsList(), getDefaultBaseUrl());
    }
  
    /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
index 91884f6,65189ca..b6203f9
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
@@@ -23,7 -22,7 +23,8 @@@ import java.util.Set
  
  import org.apache.ambari.server.AmbariException;
  import org.apache.ambari.server.controller.ServiceResponse;
 +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
+ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
  
  public interface Service {
  

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
index d755568,9fb2aba..ed19023
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
@@@ -23,7 -22,7 +23,8 @@@ import java.util.Set
  
  import org.apache.ambari.server.AmbariException;
  import org.apache.ambari.server.controller.ServiceComponentResponse;
 +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
+ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
  
  public interface ServiceComponent {
  
@@@ -102,5 -102,20 +108,20 @@@
    ServiceComponentHost addServiceComponentHost(
        String hostName) throws AmbariException;
  
 -  void delete() throws AmbariException;
 +  void delete(DeleteHostComponentStatusMetaData deleteMetaData);
+ 
+   /**
+    * This method computes the state of the repository that's associated with the desired
+    * version.  It is used, for example, when a host component reports its version and the
+    * state can be in flux.
+    *
+    * @param reportedVersion
+    * @throws AmbariException
+    */
+   void updateRepositoryState(String reportedVersion) throws AmbariException;
+ 
+   /**
+    * @return the repository state for the desired version
+    */
+   RepositoryVersionState getRepositoryState();
  }

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
index 08f9f41,5ff9e37..575193f
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
@@@ -23,9 -23,8 +23,9 @@@ import java.util.Map
  
  import org.apache.ambari.server.AmbariException;
  import org.apache.ambari.server.controller.ServiceComponentHostResponse;
 +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
  import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
- import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+ import org.apache.ambari.server.orm.entities.HostVersionEntity;
  import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
  
  
@@@ -130,30 -107,9 +108,9 @@@ public interface ServiceComponentHost 
     *
     * @param version component version (e.g. 2.2.0.0-2041)
     */
 -  void setVersion(String version);
 +  void setVersion(String version) throws AmbariException;
  
    /**
-    * Gets the desired security state for this ServiceComponent
-    * <p/>
-    * The returned SecurityState is a valid endpoint state where
-    * SecurityState.isEndpoint() == true.
-    *
-    * @return the desired SecurityState for this ServiceComponent
-    */
-   SecurityState getDesiredSecurityState();
- 
-   /**
-    * Sets the desired security state for this ServiceComponent
-    * <p/>
-    * It is expected that the new SecurityState is a valid endpoint state such that
-    * SecurityState.isEndpoint() == true.
-    *
-    * @param securityState the desired SecurityState for this ServiceComponent
-    * @throws AmbariException if the new state is not an endpoint state
-    */
-   void setDesiredSecurityState(SecurityState securityState) throws AmbariException;
- 
-   /**
     * @param upgradeState the upgrade state
     */
    void setUpgradeState(UpgradeState upgradeState);

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
index 32ecbf8,22c97ed..74b592b
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
@@@ -19,10 -19,9 +19,11 @@@
  package org.apache.ambari.server.state;
  
  import java.util.HashMap;
 +import java.util.HashSet;
+ import java.util.List;
  import java.util.Map;
  import java.util.Map.Entry;
 +import java.util.Set;
  import java.util.concurrent.ConcurrentHashMap;
  import java.util.concurrent.ConcurrentMap;
  import java.util.concurrent.locks.ReadWriteLock;
@@@ -32,9 -31,10 +33,10 @@@ import org.apache.ambari.server.AmbariE
  import org.apache.ambari.server.ObjectNotFoundException;
  import org.apache.ambari.server.ServiceComponentHostNotFoundException;
  import org.apache.ambari.server.api.services.AmbariMetaInfo;
 -import org.apache.ambari.server.controller.MaintenanceStateHelper;
  import org.apache.ambari.server.controller.ServiceComponentResponse;
 +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
  import org.apache.ambari.server.events.ServiceComponentRecoveryChangedEvent;
+ import org.apache.ambari.server.events.listeners.upgrade.StackVersionListener;
  import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
  import org.apache.ambari.server.orm.dao.ClusterServiceDAO;
  import org.apache.ambari.server.orm.dao.HostComponentDesiredStateDAO;
@@@ -84,11 -92,15 +94,12 @@@ public class ServiceComponentImpl imple
     */
    private final long desiredStateEntityId;
  
-   /**
-    * Data access object used for lookup up stacks.
-    */
-   private final StackDAO stackDAO;
+   @Inject
+   private RepositoryVersionDAO repoVersionDAO;
+ 
+   @Inject
+   private HostComponentStateDAO hostComponentDAO;
  
 -  @Inject
 -  private MaintenanceStateHelper maintenanceStateHelper;
 -
    @AssistedInject
    public ServiceComponentImpl(@Assisted Service service, @Assisted String componentName,
        AmbariMetaInfo ambariMetaInfo,

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
index c22bfb2,1104d19..8ba41ae
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
@@@ -18,14 -18,13 +18,16 @@@
  
  package org.apache.ambari.server.state;
  
+ import java.util.ArrayList;
+ import java.util.Collection;
  import java.util.HashMap;
 +import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
 +import java.util.Set;
  import java.util.concurrent.ConcurrentHashMap;
  import java.util.concurrent.ConcurrentMap;
 +import java.util.concurrent.atomic.AtomicReference;
  import java.util.concurrent.locks.Lock;
  import java.util.concurrent.locks.ReentrantLock;
  
@@@ -588,25 -577,22 +595,30 @@@ public class ServiceImpl implements Ser
  
    @Override
    @Transactional
 -  public void delete() throws AmbariException {
 +  public void delete(DeleteHostComponentStatusMetaData deleteMetaData) {
+     List<Component> components = getComponents(); // XXX temporal coupling, need to call this BEFORE deletingAllComponents
 -    deleteAllComponents();
 -    deleteAllServiceConfigs();
 +    deleteAllComponents(deleteMetaData);
 +    if (deleteMetaData.getAmbariException() != null) {
 +      return;
 +    }
 +    try {
 +      deleteAllServiceConfigs();
  
 -    StackId stackId = getDesiredStackId();
 +      removeEntities();
 +    } catch (AmbariException e) {
 +      deleteMetaData.setAmbariException(e);
 +      return;
 +    }
  
 -    removeEntities();
++    StackId stackId = getDesiredStackId();
+ 
      // publish the service removed event
-     StackId stackId = cluster.getDesiredStackVersion();
+     if (null == stackId) {
+       return;
+     }
  
      ServiceRemovedEvent event = new ServiceRemovedEvent(getClusterId(), stackId.getStackName(),
-         stackId.getStackVersion(), getName());
+         stackId.getStackVersion(), getName(), components);
  
      eventPublisher.publish(event);
    }
@@@ -635,16 -626,9 +652,12 @@@
  
    @Override
    public MaintenanceState getMaintenanceState() {
 -    return getServiceDesiredStateEntity().getMaintenanceState();
 +    if (maintenanceState.get() == null) {
 +      maintenanceState.set(getServiceDesiredStateEntity().getMaintenanceState());
 +    }
 +    return maintenanceState.get();
    }
  
-   private ClusterServiceEntity getServiceEntity() {
-     return clusterServiceDAO.findByPK(serviceEntityPK);
-   }
- 
    private ClusterServiceEntityPK getServiceEntityPK(ClusterServiceEntity serviceEntity) {
      ClusterServiceEntityPK pk = new ClusterServiceEntityPK();
      pk.setClusterId(serviceEntity.getClusterId());

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
index 353dd86,a3886ab..6184b94
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
@@@ -33,6 -33,6 +33,7 @@@ import org.apache.ambari.server.control
  import org.apache.ambari.server.stack.Validable;
  import org.apache.ambari.server.state.repository.VersionDefinitionXml;
  import org.apache.ambari.server.state.stack.ConfigUpgradePack;
++import org.apache.ambari.server.state.stack.LatestRepoCallable;
  import org.apache.ambari.server.state.stack.RepositoryXml;
  import org.apache.ambari.server.state.stack.StackRoleCommandOrder;
  import org.apache.ambari.server.state.stack.UpgradePack;

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index 92e01c2,8f9d8e1..464cb41
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@@ -491,6 -553,51 +553,36 @@@ public class UpgradeHelper 
    }
  
    /**
+    * Merges two service check groups when they have been orchestrated back-to-back.
+    * @param newHolder   the "new" group holder, which was orchestrated after the "old" one
+    * @param oldHolder   the "old" group holder, which is one that was already orchestrated
+    */
+   @SuppressWarnings("unchecked")
+   private void mergeServiceChecks(UpgradeGroupHolder newHolder, UpgradeGroupHolder oldHolder) {
+ 
+     LinkedHashSet<StageWrapper> priority = new LinkedHashSet<>();
+     LinkedHashSet<StageWrapper> others = new LinkedHashSet<>();
+ 
 -    Set<String> extraKeys = new HashSet<>();
 -    LinkedHashSet<StageWrapper> extras = new LinkedHashSet<>();
 -
+     for (List<StageWrapper> holderItems : new List[] { oldHolder.items, newHolder.items }) {
+       for (StageWrapper stageWrapper : holderItems) {
 -        if (stageWrapper instanceof ServiceCheckStageWrapper) {
 -          ServiceCheckStageWrapper wrapper = (ServiceCheckStageWrapper) stageWrapper;
 -          if (wrapper.priority) {
 -            priority.add(stageWrapper);
 -          } else {
 -            others.add(stageWrapper);
 -          }
++        ServiceCheckStageWrapper wrapper = (ServiceCheckStageWrapper) stageWrapper;
++
++        if (wrapper.priority) {
++          priority.add(stageWrapper);
+         } else {
 -          // !!! It's a good chance that back-to-back service check groups are adding the
 -          // same non-service-check wrappers.
 -          // this should be "equal enough" to prevent them from duplicating on merge
 -          String key = stageWrapper.toString();
 -          if (!extraKeys.contains(key)) {
 -            extras.add(stageWrapper);
 -            extraKeys.add(key);
 -          }
++          others.add(stageWrapper);
+         }
 -
+       }
+     }
+ 
+     // !!! remove duplicate wrappers that are now in the priority list
+     others = new LinkedHashSet<>(CollectionUtils.subtract(others, priority));
+ 
+     oldHolder.items = Lists.newLinkedList(priority);
+     oldHolder.items.addAll(others);
 -    oldHolder.items.addAll(extras);
+   }
+ 
+   /**
     * Walks through the UpgradeGroupHolder and updates titles and manual tasks,
     * replacing keyword tokens needed for display purposes
     *
@@@ -768,8 -905,250 +890,241 @@@
              serviceComponentHost.setVersion(StackVersionListener.UNKNOWN_VERSION);
            }
          }
-         serviceComponent.setDesiredVersion(desiredVersion);
  
+         // set component desired repo
+         serviceComponent.setDesiredRepositoryVersion(targetRepositoryVersion);
+       }
+     }
+   }
+ 
+   /**
+    * Handles the creation or resetting of configurations based on whether an
+    * upgrade or downgrade is occurring. This method will not do anything when
+    * the service is not crossing major stack versions, since, by definition, no
+    * new configurations are automatically created when upgrading with the same
+    * stack (ie HDP 2.2.0.0 -> HDP 2.2.1.0).
+    * <p/>
+    * When upgrading or downgrade between stacks (HDP 2.2.0.0 -> HDP 2.3.0.0)
+    * then this will perform the following:
+    * <ul>
+    * <li>Upgrade: Create new configurations that are a merge between the source
+    * stack and the target stack. If a value has changed between stacks, then the
+    * target stack value should be taken unless the cluster's value differs from
+    * the old stack. This can occur if a property has been customized after
 -   * installation. Read-only properties, however, are always taken from the new
 -   * stack.</li>
++   * installation.</li>
+    * <li>Downgrade: Reset the latest configurations from the service's original
+    * stack. The new configurations that were created on upgrade must be left
+    * intact until all components have been reverted, otherwise heartbeats will
+    * fail due to missing configurations.</li>
+    * </ul>
+    *
+    * @param upgradeContext
+    *          the upgrade context (not {@code null}).
+    * @throws AmbariException
+    */
+   private void processConfigurationsIfRequired(UpgradeContext upgradeContext)
+       throws AmbariException {
+ 
+     AmbariManagementController controller = m_controllerProvider.get();
+ 
+     Cluster cluster = upgradeContext.getCluster();
+     Direction direction = upgradeContext.getDirection();
+     String userName = controller.getAuthName();
+     Set<String> servicesInUpgrade = upgradeContext.getSupportedServices();
+ 
+     Set<String> clusterConfigTypes = new HashSet<>();
+     Set<String> processedClusterConfigTypes = new HashSet<>();
+ 
+     // merge or revert configurations for any service that needs it
+     for (String serviceName : servicesInUpgrade) {
+       RepositoryVersionEntity sourceRepositoryVersion = upgradeContext.getSourceRepositoryVersion(serviceName);
+       RepositoryVersionEntity targetRepositoryVersion = upgradeContext.getTargetRepositoryVersion(serviceName);
+       StackId sourceStackId = sourceRepositoryVersion.getStackId();
+       StackId targetStackId = targetRepositoryVersion.getStackId();
+ 
+       // only work with configurations when crossing stacks
+       if (sourceStackId.equals(targetStackId)) {
+         RepositoryVersionEntity associatedRepositoryVersion = upgradeContext.getRepositoryVersion();
+         LOG.info(
+             "The {} {} {} will not change stack configurations for {} since the source and target are both {}",
+             direction.getText(false), direction.getPreposition(),
+             associatedRepositoryVersion.getVersion(), serviceName, targetStackId);
+ 
+         continue;
+       }
+ 
+       ConfigHelper configHelper = m_configHelperProvider.get();
+ 
+       // downgrade is easy - just remove the new and make the old current
+       if (direction == Direction.DOWNGRADE) {
+         cluster.applyLatestConfigurations(targetStackId, serviceName);
+         continue;
+       }
+ 
 -      // the auto-merge must take read-only properties even if they have changed
 -      // - if the properties was read-only in the source stack, then we must
 -      // take the new stack's value
 -      Map<String, Set<String>> readOnlyProperties = getReadOnlyProperties(sourceStackId, serviceName);
 -
+       // upgrade is a bit harder - we have to merge new stack configurations in
+ 
+       // populate a map of default configurations for the service on the old
+       // stack (this is used when determining if a property has been
+       // customized and should be overriden with the new stack value)
+       Map<String, Map<String, String>> oldServiceDefaultConfigsByType = configHelper.getDefaultProperties(
+           sourceStackId, serviceName);
+ 
+       // populate a map with default configurations from the new stack
+       Map<String, Map<String, String>> newServiceDefaultConfigsByType = configHelper.getDefaultProperties(
+           targetStackId, serviceName);
+ 
+       if (null == oldServiceDefaultConfigsByType || null == newServiceDefaultConfigsByType) {
+         continue;
+       }
+ 
+       Set<String> foundConfigTypes = new HashSet<>();
+ 
+       // find the current, existing configurations for the service
+       List<Config> existingServiceConfigs = new ArrayList<>();
+       List<ServiceConfigEntity> latestServiceConfigs = m_serviceConfigDAO.getLastServiceConfigsForService(
+           cluster.getClusterId(), serviceName);
+ 
+       for (ServiceConfigEntity serviceConfig : latestServiceConfigs) {
+         List<ClusterConfigEntity> existingConfigurations = serviceConfig.getClusterConfigEntities();
+         for (ClusterConfigEntity currentServiceConfig : existingConfigurations) {
+           String configurationType = currentServiceConfig.getType();
+ 
+           Config currentClusterConfigForService = cluster.getDesiredConfigByType(configurationType);
+           existingServiceConfigs.add(currentClusterConfigForService);
+           foundConfigTypes.add(configurationType);
+         }
+       }
+ 
+       // !!! these are the types that come back from the config helper, but are not part of the service.
+       @SuppressWarnings("unchecked")
+       Set<String> missingConfigTypes = new HashSet<>(CollectionUtils.subtract(oldServiceDefaultConfigsByType.keySet(),
+           foundConfigTypes));
+ 
+       for (String missingConfigType : missingConfigTypes) {
+         Config config = cluster.getDesiredConfigByType(missingConfigType);
+         if (null != config) {
+           existingServiceConfigs.add(config);
+           clusterConfigTypes.add(missingConfigType);
+         }
+       }
+ 
+       // now that we have found, old, new, and existing confgs, overlay the
+       // existing on top of the new
+       for (Config existingServiceConfig : existingServiceConfigs) {
+         String configurationType = existingServiceConfig.getType();
+ 
+         // get current stack default configurations on install
+         Map<String, String> oldServiceDefaultConfigs = oldServiceDefaultConfigsByType.get(
+             configurationType);
+ 
+         // NPE sanity for current stack defaults
+         if (null == oldServiceDefaultConfigs) {
+           oldServiceDefaultConfigs = Collections.emptyMap();
+         }
+ 
+         // get the existing configurations
+         Map<String, String> existingConfigurations = existingServiceConfig.getProperties();
+ 
+         // get the new configurations
 -        Map<String, String> newDefaultConfigurations = newServiceDefaultConfigsByType.get(configurationType);
++        Map<String, String> newDefaultConfigurations = newServiceDefaultConfigsByType.get(
++            configurationType);
+ 
+         // if the new stack configurations don't have the type, then simply add
+         // all of the existing in
+         if (null == newDefaultConfigurations) {
+           newServiceDefaultConfigsByType.put(configurationType, existingConfigurations);
+           continue;
+         } else {
+           // Remove any configs in the new stack whose value is NULL, unless
+           // they currently exist and the value is not NULL.
+           Iterator<Map.Entry<String, String>> iter = newDefaultConfigurations.entrySet().iterator();
+           while (iter.hasNext()) {
+             Map.Entry<String, String> entry = iter.next();
+             if (entry.getValue() == null) {
+               iter.remove();
+             }
+           }
+         }
+ 
 -        // process every existing configuration property for this configuration type
++        // process every existing configuration property for this configuration
++        // type
+         for (Map.Entry<String, String> existingConfigurationEntry : existingConfigurations.entrySet()) {
+           String existingConfigurationKey = existingConfigurationEntry.getKey();
+           String existingConfigurationValue = existingConfigurationEntry.getValue();
+ 
+           // if there is already an entry, we now have to try to determine if
+           // the value was customized after stack installation
+           if (newDefaultConfigurations.containsKey(existingConfigurationKey)) {
+             String newDefaultConfigurationValue = newDefaultConfigurations.get(
+                 existingConfigurationKey);
+ 
+             if (!StringUtils.equals(existingConfigurationValue, newDefaultConfigurationValue)) {
+               // the new default is different from the existing cluster value;
+               // only override the default value if the existing value differs
+               // from the original stack
+               String oldDefaultValue = oldServiceDefaultConfigs.get(existingConfigurationKey);
+ 
 -              // see if this property is a read-only property which means that
 -              // we shouldn't care if it was changed - we should take the new
 -              // stack's value
 -              Set<String> readOnlyPropertiesForType = readOnlyProperties.get(configurationType);
 -              boolean readOnly = (null != readOnlyPropertiesForType
 -                  && readOnlyPropertiesForType.contains(existingConfigurationKey));
 -
 -              if (!readOnly && !StringUtils.equals(existingConfigurationValue, oldDefaultValue)) {
 -                // at this point, we've determined that there is a difference
 -                // between default values between stacks, but the value was also
 -                // customized, so keep the customized value
++              if (!StringUtils.equals(existingConfigurationValue, oldDefaultValue)) {
++                // at this point, we've determined that there is a
++                // difference
++                // between default values between stacks, but the value was
++                // also customized, so keep the customized value
+                 newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
+               }
+             }
+           } else {
 -            // there is no entry in the map, so add the existing key/value pair
++            // there is no entry in the map, so add the existing key/value
++            // pair
+             newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
+           }
+         }
+ 
+         /*
+         for every new configuration which does not exist in the existing
+         configurations, see if it was present in the current stack
+ 
+         stack 2.x has foo-site/property (on-ambari-upgrade is false)
+         stack 2.y has foo-site/property
+         the current cluster (on 2.x) does not have it
+ 
+         In this case, we should NOT add it back as clearly stack advisor has removed it
+         */
+         Iterator<Map.Entry<String, String>> newDefaultConfigurationsIterator = newDefaultConfigurations.entrySet().iterator();
+         while (newDefaultConfigurationsIterator.hasNext()) {
+           Map.Entry<String, String> newConfigurationEntry = newDefaultConfigurationsIterator.next();
+           String newConfigurationPropertyName = newConfigurationEntry.getKey();
+           if (oldServiceDefaultConfigs.containsKey(newConfigurationPropertyName)
+               && !existingConfigurations.containsKey(newConfigurationPropertyName)) {
+             LOG.info(
+                 "The property {}/{} exists in both {} and {} but is not part of the current set of configurations and will therefore not be included in the configuration merge",
+                 configurationType, newConfigurationPropertyName, sourceStackId, targetStackId);
+ 
+             // remove the property so it doesn't get merged in
+             newDefaultConfigurationsIterator.remove();
+           }
+         }
+       }
+ 
+       if (null != newServiceDefaultConfigsByType) {
+ 
+         for (String clusterConfigType : clusterConfigTypes) {
+           if (processedClusterConfigTypes.contains(clusterConfigType)) {
+             newServiceDefaultConfigsByType.remove(clusterConfigType);
+           } else {
+             processedClusterConfigTypes.add(clusterConfigType);
+           }
+ 
+         }
+ 
+         Set<String> configTypes = newServiceDefaultConfigsByType.keySet();
+         LOG.warn("The upgrade will create the following configurations for stack {}: {}",
+             targetStackId, StringUtils.join(configTypes, ','));
+ 
+         String serviceVersionNote = String.format("%s %s %s", direction.getText(true),
+             direction.getPreposition(), upgradeContext.getRepositoryVersion().getVersion());
+ 
+         configHelper.createConfigTypes(cluster, targetStackId, controller,
+             newServiceDefaultConfigsByType, userName, serviceVersionNote);
        }
      }
    }

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinitionHash.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/ParameterizedSource.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Source.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 7147ce3,9c0b0ca..e6da65e
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@@ -145,14 -128,12 +132,16 @@@ import org.apache.ambari.server.state.U
  import org.apache.ambari.server.state.configgroup.ConfigGroup;
  import org.apache.ambari.server.state.configgroup.ConfigGroupFactory;
  import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
+ import org.apache.ambari.server.state.repository.ClusterVersionSummary;
+ import org.apache.ambari.server.state.repository.VersionDefinitionXml;
  import org.apache.ambari.server.state.scheduler.RequestExecution;
  import org.apache.ambari.server.state.scheduler.RequestExecutionFactory;
 +import org.apache.ambari.server.state.stack.upgrade.Direction;
 +import org.apache.ambari.server.state.svccomphost.ServiceComponentHostSummary;
 +import org.apache.ambari.server.topology.TopologyDeleteFormer;
  import org.apache.ambari.server.topology.TopologyRequest;
  import org.apache.commons.collections.CollectionUtils;
 +import org.apache.commons.collections.MapUtils;
  import org.apache.commons.lang.StringUtils;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
@@@ -2315,15 -1545,10 +1569,15 @@@ public class ClusterImpl implements Clu
        serviceConfigEntity.setVersion(nextServiceConfigVersion);
        serviceConfigEntity.setUser(user);
        serviceConfigEntity.setNote(note);
-       serviceConfigEntity.setStack(clusterEntity.getDesiredStack());
+       serviceConfigEntity.setStack(stackEntity);
  
        serviceConfigDAO.create(serviceConfigEntity);
 +      List<String> groupHostNames = null;
        if (configGroup != null) {
 +        if (MapUtils.isNotEmpty(configGroup.getHosts())) {
 +          groupHostNames = configGroup.getHosts().entrySet().stream().map(h -> h.getValue().getHostName())
 +              .collect(Collectors.toList());
 +        }
          serviceConfigEntity.setHostIds(new ArrayList<>(configGroup.getHosts().keySet()));
          serviceConfigEntity = serviceConfigDAO.merge(serviceConfigEntity);
        }
@@@ -2585,19 -1806,31 +1837,32 @@@
        throw new ObjectNotFoundException("Service config version with serviceName={} and version={} not found");
      }
  
 +    String configGroupName = null;
      // disable all configs related to service
      if (serviceConfigEntity.getGroupId() == null) {
+       // Here was fixed bug with entity changes revert. More you can find here AMBARI-21173.
+       // This issue reproduces only if you are changing same entity in first and second loop.
+       // In that case eclipselink will revert changes to cached, if entity has fluchGroup and it
+       // needs to be refreshed. Actually we don't need to change same antities in few steps, so i
+       // decided to filter out. duplicates and do not change them. It will be better for performance and bug will be fixed.
        Collection<String> configTypes = serviceConfigTypes.get(serviceName);
        List<ClusterConfigEntity> enabledConfigs = clusterDAO.getEnabledConfigsByTypes(clusterId, configTypes);
+       List<ClusterConfigEntity> serviceConfigEntities = serviceConfigEntity.getClusterConfigEntities();
+       ArrayList<ClusterConfigEntity> duplicatevalues = new ArrayList<>(serviceConfigEntities);
+       duplicatevalues.retainAll(enabledConfigs);
+ 
        for (ClusterConfigEntity enabledConfig : enabledConfigs) {
-         enabledConfig.setSelected(false);
-         clusterDAO.merge(enabledConfig);
+         if (!duplicatevalues.contains(enabledConfig)) {
+           enabledConfig.setSelected(false);
+           clusterDAO.merge(enabledConfig);
+         }
        }
  
-       for (ClusterConfigEntity configEntity : serviceConfigEntity.getClusterConfigEntities()) {
-         configEntity.setSelected(true);
-         clusterDAO.merge(configEntity);
+       for (ClusterConfigEntity configEntity : serviceConfigEntities) {
+         if (!duplicatevalues.contains(configEntity)) {
+           configEntity.setSelected(true);
+           clusterDAO.merge(configEntity);
+         }
        }
      } else {
        Long configGroupId = serviceConfigEntity.getGroupId();
@@@ -3159,9 -2415,14 +2463,14 @@@
        // since the entities which were modified came from the cluster entity's
        // list to begin with, we can just save them right back - no need for a
        // new collection since the entity instances were modified directly
 -      clusterEntity = clusterDAO.merge(clusterEntity, true);
 +      clusterEntity = clusterDAO.merge(clusterEntity);
  
        cacheConfigurations();
+ 
+       LOG.info(
+           "Applied latest configurations for {} on stack {}. The the following types were modified: {}",
+           serviceName, stackId, StringUtils.join(configTypesForService, ','));
+ 
      } finally {
        clusterGlobalLock.writeLock().unlock();
      }

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
index 702d776,5ac1ac3..8743f63
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
@@@ -46,13 -40,10 +46,11 @@@ import org.apache.ambari.server.control
  import org.apache.ambari.server.events.HostRegisteredEvent;
  import org.apache.ambari.server.events.HostsAddedEvent;
  import org.apache.ambari.server.events.HostsRemovedEvent;
 +import org.apache.ambari.server.events.TopologyUpdateEvent;
  import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
  import org.apache.ambari.server.orm.dao.ClusterDAO;
- import org.apache.ambari.server.orm.dao.ClusterVersionDAO;
  import org.apache.ambari.server.orm.dao.HostConfigMappingDAO;
  import org.apache.ambari.server.orm.dao.HostDAO;
- import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
  import org.apache.ambari.server.orm.dao.HostStateDAO;
  import org.apache.ambari.server.orm.dao.HostVersionDAO;
  import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
@@@ -276,17 -243,9 +260,17 @@@ public class ClustersImpl implements Cl
      clusters.put(clusterName, cluster);
      clustersById.put(cluster.getClusterId(), cluster);
      clusterHostMap.put(clusterName,
-         Collections.newSetFromMap(new ConcurrentHashMap<Host, Boolean>()));
+         Collections.newSetFromMap(new ConcurrentHashMap<>()));
  
      cluster.setCurrentStackVersion(stackId);
 +
 +    TreeMap<String, TopologyCluster> addedClusters = new TreeMap<>();
 +    TopologyCluster addedCluster = new TopologyCluster();
 +    addedClusters.put(Long.toString(cluster.getClusterId()), addedCluster);
 +    TopologyUpdateEvent topologyUpdateEvent = new TopologyUpdateEvent(addedClusters,
 +        TopologyUpdateEvent.EventType.UPDATE);
 +    m_topologyHolder.get().updateData(topologyUpdateEvent);
 +    m_metadataHolder.get().updateData(m_ambariManagementController.get().getClusterMetadata(cluster));
    }
  
    @Override

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
index 0b54eec,274a425..2a9e69e
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
@@@ -595,14 -580,9 +586,10 @@@ public class HostImpl implements Host 
        writeLock.unlock();
      }
      if (oldState != getState()) {
 +      ambariEventPublisher.publish(new HostStateUpdateEvent(getHostName(), getState()));
        if (LOG.isDebugEnabled()) {
-         LOG.debug("Host transitioned to a new state"
-             + ", host=" + getHostName()
-             + ", oldState=" + oldState
-             + ", currentState=" + getState()
-             + ", eventType=" + event.getType().name()
-             + ", event=" + event);
+         LOG.debug("Host transitioned to a new state, host={}, oldState={}, currentState={}, eventType={}, event={}",
+           getHostName(), oldState, getState(), event.getType().name(), event);
        }
      }
    }

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
index a1b9e5c,0c7a9a9..9432f6c
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
@@@ -193,13 -201,17 +198,17 @@@ public class KerberosDescriptor extend
        }
  
        if (services == null) {
 -        services = new TreeMap<String, KerberosServiceDescriptor>();
 +        services = new TreeMap<>();
        }
  
-       services.put(name, service);
- 
-       // Set the service's parent to this KerberosDescriptor
-       service.setParent(this);
+       KerberosServiceDescriptor existing = services.get(name);
+       if (existing == null) {
+         services.put(name, service);
+         // Set the service's parent to this KerberosDescriptor
+         service.setParent(this);
+       } else {
+         existing.update(service);
+       }
      }
    }
  
@@@ -418,4 -425,43 +422,43 @@@
  
      return authToLocalProperties;
    }
+ 
+   /**
+    * Get a map of principals, where the key is the principal path (SERVICE/COMPONENT/principal_name or SERVICE/principal_name) and the value is the principal.
+    * <p>
+    * For example if the kerberos principal of the HISTORYSERVER is defined in the kerberos.json:
+    * "name": "history_server_jhs",
+    * "principal": {
+    * "value": "jhs/_HOST@${realm}",
+    * "type" : "service",
+    * },
+    * Then "jhs/_HOST@EXAMPLE.COM" will be put into the map under the "MAPREDUCE2/HISTORYSERVER/history_server_jhs" key.
+    */
+   public Map<String, String> principals() throws AmbariException {
+     Map<String, String> result = new HashMap<>();
+     for (AbstractKerberosDescriptorContainer each : nullToEmpty(getChildContainers())) {
+       if ((each instanceof KerberosServiceDescriptor)) {
+         collectFromComponents(each.getName(), nullToEmpty(((KerberosServiceDescriptor) each).getComponents()).values(), result);
+         collectFromIdentities(each.getName(), "", nullToEmpty(each.getIdentities()), result);
+       }
+     }
+     return result;
+   }
+ 
+   private static void collectFromComponents(String service, Collection<KerberosComponentDescriptor> components, Map<String, String> result) {
+     for (KerberosComponentDescriptor each : components) {
+       collectFromIdentities(service, each.getName(), nullToEmpty(each.getIdentities()), result);
+     }
+   }
+ 
+   private static void collectFromIdentities(String service, String component, Collection<KerberosIdentityDescriptor> identities, Map<String, String> result) {
+     for (KerberosIdentityDescriptor each : identities) {
 -      if (each.getPrincipalDescriptor() != null && !each.getReferencedServiceName().isPresent()) {
++      if (each.getPrincipalDescriptor() != null && !each.getReferencedServiceName().isPresent() && !each.getName().startsWith("/")) {
+         String path = StringUtils.isBlank(component)
+             ? String.format("%s/%s", service, each.getName())
+             : String.format("%s/%s/%s", service, component, each.getName());
+         result.put(path, each.getPrincipalDescriptor().getName());
+       }
+     }
+   }
  }

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
index 68cafe5,51b7cd0..5da3399
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
@@@ -157,6 -150,8 +146,8 @@@ public class KerberosServiceDescriptor 
            }
          }
        }
+ 
 -      setPreconfigure(getBooleanValue(data, KEY_PRECONFIGURE));
++      setPreconfigure(getBooleanValue(data, "preconfigure"));
      }
    }
  
@@@ -266,9 -279,13 +275,13 @@@
        for (KerberosComponentDescriptor component : components.values()) {
          list.add(component.toMap());
        }
 -      map.put(KEY_COMPONENTS, list);
 +      map.put(Type.COMPONENT.getDescriptorPluralName(), list);
      }
  
+     if (preconfigure != null) {
 -      map.put(KEY_PRECONFIGURE, preconfigure.toString());
++      map.put("preProcess", preconfigure.toString());
+     }
+ 
      return map;
    }
  

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
index 078b4ae,c2209bb..03b3705
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
@@@ -146,7 -146,8 +146,6 @@@ public class RepositoryXml implements V
      private String mirrorslist = null;
      private String repoid = null;
      private String reponame = null;
-     private String latest = null;
 -    private String distribution = null;
 -    private String components = null;
      private boolean unique = false;
  
      private Repo() {
@@@ -180,10 -181,13 +179,6 @@@
        return reponame;
      }
  
-     public String getLatestUri() {
-       return latest;
 -    public String getDistribution() {
 -      return distribution;
--    }
--
 -    public String getComponents() {
 -      return components;
 -    }
      /**
       * @return true if version of HDP that change with each release
       */
@@@ -217,7 -221,8 +212,6 @@@
            ri.setOsType(os.trim());
            ri.setRepoId(r.getRepoId());
            ri.setRepoName(r.getRepoName());
-           ri.setLatestBaseUrl(r.getBaseUrl());
 -          ri.setDistribution(r.getDistribution());
 -          ri.setComponents(r.getComponents());
            ri.setUnique(r.isUnique());
  
            repos.add(ri);

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
index fdb7c8d,f540d8d..9524c09
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
@@@ -218,4 -246,98 +236,97 @@@ public class RepositoryVersionHelper 
      throw new AmbariException("There were no suitable upgrade packs for stack " + stackName + " " + stackVersion +
          ((null != upgradeType) ? " and upgrade type " + upgradeType : ""));
    }
+ 
+   /**
+    * Build the role parameters for an install command.
+    *
+    * @param amc           the management controller.  Tests don't use the same instance that gets injected.
+    * @param repoVersion   the repository version
+    * @param osFamily      the os family
+    * @param servicesOnHost the set of services to check for packages
+    * @return a Map<String, String> to use in
+    */
+   public Map<String, String> buildRoleParams(AmbariManagementController amc, RepositoryVersionEntity repoVersion, String osFamily, Set<String> servicesOnHost)
+     throws SystemException {
+ 
+     StackId stackId = repoVersion.getStackId();
+ 
+     List<ServiceOsSpecific.Package> packages = new ArrayList<>();
+ 
+     for (String serviceName : servicesOnHost) {
+       ServiceInfo info;
+ 
+       try {
+         if (ami.get().isServiceRemovedInStack(stackId.getStackName(), stackId.getStackVersion(), serviceName)) {
+           LOG.info(String.format("%s has been removed from stack %s-%s. Skip calculating its installation packages", stackId.getStackName(), stackId.getStackVersion(), serviceName));
+           continue; //No need to calculate install packages for removed services
+         }
+ 
+         info = ami.get().getService(stackId.getStackName(), stackId.getStackVersion(), serviceName);
+       } catch (AmbariException e) {
+         throw new SystemException(String.format("Cannot obtain stack information for %s-%s", stackId.getStackName(), stackId.getStackVersion()), e);
+       }
+ 
+       List<ServiceOsSpecific.Package> packagesForService = amc.getPackagesForServiceHost(info,
+         new HashMap<>(), osFamily);
+ 
+       List<String> blacklistedPackagePrefixes = configuration.get().getRollingUpgradeSkipPackagesPrefixes();
+ 
+       for (ServiceOsSpecific.Package aPackage : packagesForService) {
+         if (!aPackage.getSkipUpgrade()) {
+           boolean blacklisted = false;
+           for (String prefix : blacklistedPackagePrefixes) {
+             if (aPackage.getName().startsWith(prefix)) {
+               blacklisted = true;
+               break;
+             }
+           }
+           if (! blacklisted) {
+             packages.add(aPackage);
+           }
+         }
+       }
+     }
+ 
+     Map<String, String> roleParams = new HashMap<>();
+     roleParams.put("stack_id", stackId.getStackId());
+     // !!! TODO make roleParams <String, Object> so we don't have to do this awfulness.
+     roleParams.put(KeyNames.PACKAGE_LIST, gson.toJson(packages));
+ 
+     return roleParams;
+   }
+ 
+   /**
+    * Adds a command repository to the action context
+    * @param context       the context
+    * @param osFamily      the OS family
+    * @param repoVersion   the repository version entity
+    * @param repos         the repository entities
+    */
+   public void addCommandRepository(ActionExecutionContext context,
+       RepositoryVersionEntity repoVersion, OperatingSystemEntity osEntity) {
+ 
+     final CommandRepository commandRepo = new CommandRepository();
+     commandRepo.setRepositories(osEntity.getOsType(), osEntity.getRepositories());
+     commandRepo.setRepositoryVersion(repoVersion.getVersion());
+     commandRepo.setRepositoryVersionId(repoVersion.getId());
 -    commandRepo.setResolved(repoVersion.isResolved());
+     commandRepo.setStackName(repoVersion.getStackId().getStackName());
+ 
+     if (!osEntity.isAmbariManagedRepos()) {
+       commandRepo.setNonManaged();
+     } else {
+       commandRepo.setUniqueSuffix(String.format("-repo-%s", repoVersion.getId()));
+     }
+ 
+     context.addVisitor(new ExecutionCommandVisitor() {
+       @Override
+       public void visit(ExecutionCommand command) {
+         if (null == command.getRepositoryFile()) {
+           command.setRepositoryFile(commandRepo);
+         }
+       }
+     });
+   }
+ 
+ 
  }

http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index 446ae8c,230b031..ecb616a
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@@ -25,10 -24,8 +25,11 @@@ import java.util.HashSet
  import java.util.List;
  import java.util.Map;
  import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.TreeMap;
+ import java.util.concurrent.ConcurrentHashMap;
  import java.util.concurrent.ConcurrentMap;
 +import java.util.concurrent.atomic.AtomicReference;
  import java.util.concurrent.locks.Lock;
  import java.util.concurrent.locks.ReadWriteLock;
  import java.util.concurrent.locks.ReentrantReadWriteLock;
@@@ -93,8 -83,8 +95,9 @@@ import org.slf4j.Logger
  import org.slf4j.LoggerFactory;
  
  import com.google.common.collect.ImmutableList;
+ import com.google.common.util.concurrent.Striped;
  import com.google.inject.Inject;
 +import com.google.inject.Provider;
  import com.google.inject.assistedinject.Assisted;
  import com.google.inject.assistedinject.AssistedInject;
  import com.google.inject.persist.Transactional;
@@@ -880,8 -869,8 +888,9 @@@ public class ServiceComponentHostImpl i
    }
  
    @Override
+   @Transactional
    public void setState(State state) {
 +    State oldState = getState();
      stateMachine.setCurrentState(state);
      HostComponentStateEntity stateEntity = getStateEntity();
      if (stateEntity != null) {
@@@ -913,7 -898,8 +922,8 @@@
    }
  
    @Override
+   @Transactional
 -  public void setVersion(String version) {
 +  public void setVersion(String version) throws AmbariException {
      HostComponentStateEntity stateEntity = getStateEntity();
      if (stateEntity != null) {
        stateEntity.setVersion(version);


Mime
View raw message