Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F2740200CFC for ; Thu, 28 Sep 2017 15:25:00 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id F0E2C1609C2; Thu, 28 Sep 2017 13:25:00 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 996ED160BDB for ; Thu, 28 Sep 2017 15:24:58 +0200 (CEST) Received: (qmail 18998 invoked by uid 500); 28 Sep 2017 13:24:51 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 18624 invoked by uid 99); 28 Sep 2017 13:24:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Sep 2017 13:24:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 548E2DFA25; Thu, 28 Sep 2017 13:24:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aonishuk@apache.org To: commits@ambari.apache.org Date: Thu, 28 Sep 2017 13:25:34 -0000 Message-Id: <8b343f474487445b9113d5ff3a05b1eb@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [45/50] [abbrv] ambari git commit: Merge remote-tracking branch 'remotes/origin/trunk' into branch-3.0-perf archived-at: Thu, 28 Sep 2017 13:25:01 -0000 http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java index 0000000,30bc47f..697f1d1 mode 000000,100644..100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java @@@ -1,0 -1,573 +1,573 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.ambari.server.serveraction.upgrades; + + import static org.apache.ambari.server.controller.KerberosHelper.DEFAULT_REALM; + import static org.apache.ambari.server.controller.KerberosHelper.KERBEROS_ENV; + import static org.apache.ambari.server.controller.KerberosHelper.PRECONFIGURE_SERVICES; + + import java.io.IOException; + import java.util.Collection; + import java.util.Collections; + import java.util.HashMap; + import java.util.HashSet; + import java.util.Iterator; + import java.util.List; + import java.util.Map; + import java.util.Set; + import java.util.concurrent.ConcurrentMap; + + import org.apache.ambari.server.AmbariException; + import org.apache.ambari.server.actionmanager.HostRoleStatus; + import org.apache.ambari.server.agent.CommandReport; + import org.apache.ambari.server.controller.AmbariManagementController; + import org.apache.ambari.server.controller.KerberosHelper; + import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; + import org.apache.ambari.server.serveraction.kerberos.PreconfigureServiceType; + import org.apache.ambari.server.state.Cluster; + import org.apache.ambari.server.state.ConfigHelper; + import org.apache.ambari.server.state.Host; + import org.apache.ambari.server.state.SecurityType; + import org.apache.ambari.server.state.Service; + import org.apache.ambari.server.state.ServiceComponentHost; + import org.apache.ambari.server.state.StackId; + import org.apache.ambari.server.state.UpgradeContext; + import org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptorContainer; + import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor; + import org.apache.ambari.server.state.kerberos.KerberosConfigurationDescriptor; + import org.apache.ambari.server.state.kerberos.KerberosDescriptor; + import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor; + import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; + import org.apache.ambari.server.state.kerberos.VariableReplacementHelper; + import org.apache.ambari.server.state.stack.upgrade.Direction; + import org.apache.commons.collections.CollectionUtils; + import org.apache.commons.collections.MapUtils; + import org.apache.commons.lang.StringUtils; + + import com.google.inject.Inject; + + /** + * PreconfigureKerberos updates existing service configurations with properties from service-level + * Kerberos descriptors, flagged for pre-configuring, during stack upgrades in order to prevent service + * restarts when the flagged services are installed. + */ + public class PreconfigureKerberosAction extends AbstractUpgradeServerAction { + static final String UPGRADE_DIRECTION_KEY = "upgrade_direction"; + + @Inject + private AmbariManagementController ambariManagementController; + + @Inject + private KerberosHelper kerberosHelper; + + @Inject + private ConfigHelper configHelper; + + @Inject + private VariableReplacementHelper variableReplacementHelper; + + @Override + public CommandReport execute(ConcurrentMap requestSharedDataContext) throws AmbariException, InterruptedException { + Map commandParameters = getCommandParameters(); + if (null == commandParameters || commandParameters.isEmpty()) { + return createCommandReport(0, HostRoleStatus.FAILED, "{}", "", + "Unable to change configuration values without command parameters"); + } + + if (!isDowngrade()) { + String clusterName = commandParameters.get("clusterName"); + Cluster cluster = m_clusters.getCluster(clusterName); + + if (cluster.getSecurityType() == SecurityType.KERBEROS) { + StackId stackId; + + try { + stackId = getTargetStackId(cluster); + } catch (AmbariException e) { + return createCommandReport(0, HostRoleStatus.FAILED, "{}", "", e.getLocalizedMessage()); + } + + if (stackId == null) { + return createCommandReport(0, HostRoleStatus.FAILED, "{}", "", + "The target stack Id was not specified."); + } + + KerberosDescriptor kerberosDescriptor = kerberosHelper.getKerberosDescriptor(KerberosHelper.KerberosDescriptorType.COMPOSITE, cluster, stackId, true); + + // Calculate the current host-specific configurations. These will be used to replace + // variables within the Kerberos descriptor data + Map> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, true, false); + + PreconfigureServiceType preconfigureServiceType = getPreconfigureServiceType(configurations); + + if (preconfigureServiceType != PreconfigureServiceType.NONE) { + Map> kerberosConfigurations = new HashMap<>(); + Map> propertiesToRemove = new HashMap<>(); + Map> propertiesToIgnore = new HashMap<>(); + + if (preconfigureServiceType == PreconfigureServiceType.ALL) { + // Force all services to be flagged for pre-configuration... + Map serviceDescriptors = kerberosDescriptor.getServices(); + if (serviceDescriptors != null) { + for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors.values()) { + serviceDescriptor.setPreconfigure(true); + } + } + } + + processServiceComponentHosts(cluster, kerberosDescriptor, configurations, kerberosConfigurations, propertiesToIgnore); + + // Calculate the set of configurations to update and replace any variables + // using the previously calculated Map of configurations for the host. + kerberosConfigurations = kerberosHelper.processPreconfiguredServiceConfigurations(kerberosConfigurations, configurations, cluster, kerberosDescriptor); + + Map> installedServices = calculateInstalledServices(cluster); + + kerberosHelper.applyStackAdvisorUpdates(cluster, installedServices.keySet(), configurations, kerberosConfigurations, + propertiesToIgnore, propertiesToRemove, true); + + kerberosHelper.setAuthToLocalRules(cluster, kerberosDescriptor, getDefaultRealm(configurations), installedServices, + configurations, kerberosConfigurations, true); + + processConfigurationChanges(cluster, stackId, kerberosDescriptor, kerberosConfigurations, propertiesToRemove, configurations); + } else { + actionLog.writeStdOut("Skipping: This facility is only available when kerberos-env/preconfigure_services is not \"NONE\""); + } + } else { + actionLog.writeStdOut("Skipping: This facility is only available when Kerberos is enabled"); + } + } else { + actionLog.writeStdOut("Skipping: This facility is only available during an upgrade"); + } + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", actionLog.getStdOut(), actionLog.getStdErr()); + } + + /** + * Given a Cluster object creates a map of service names to sets of the installed components for that + * service. + * + * @param cluster the cluster + * @return a map of (installed) service names to the relevant set of (installed) component names + */ + private Map> calculateInstalledServices(Cluster cluster) { + Map> installedServices = new HashMap<>(); + Map services = cluster.getServices(); + + for (Service service : services.values()) { + installedServices.put(service.getName(), service.getServiceComponents().keySet()); + } + + return installedServices; + } + + /** + * Safely retrieves the specified property from the specified configuration type from a map of + * configurations. + * + * @param configurations the existing configurations for the cluster + * @return the requested value or null if the configuration does not exist + */ + private String getValueFromConfiguration(Map> configurations, String configType, String propertyName) { + String value = null; + + if (configurations != null) { + Map kerberosEnv = configurations.get(configType); + + if (kerberosEnv != null) { + value = kerberosEnv.get(propertyName); + } + } + + return value; + } + + /** + * Safely retrieves the realm property of the kerberos-env configuration. + * + * @param configurations the existing configurations for the cluster + * @return the requested value or null if the configuration does not exist + * @see #getValueFromConfiguration(Map, String, String) + */ + private String getDefaultRealm(Map> configurations) { + return getValueFromConfiguration(configurations, KERBEROS_ENV, DEFAULT_REALM); + } + + /** + * Safely retrieves the preconfigure_services property of the kerberos-env configuration. + * + * @param configurations the existing configurations for the cluster + * @return the requested value or null if the configuration does not exist + * @see #getValueFromConfiguration(Map, String, String) + */ + private PreconfigureServiceType getPreconfigureServiceType(Map> configurations) { + String preconfigureServices = getValueFromConfiguration(configurations, KERBEROS_ENV, PRECONFIGURE_SERVICES); + + PreconfigureServiceType preconfigureServiceType = null; + if (!StringUtils.isEmpty(preconfigureServices)) { + try { + preconfigureServiceType = PreconfigureServiceType.valueOf(preconfigureServices.toUpperCase()); + } catch (Throwable t) { + preconfigureServiceType = PreconfigureServiceType.DEFAULT; + } + } + + return (preconfigureServiceType == null) ? PreconfigureServiceType.DEFAULT : preconfigureServiceType; + } + + /** + * Determines if upgrade direction is {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}. + * + * @return {@code true} if {@link Direction#DOWNGRADE}; {@code false} if {@link Direction#UPGRADE} + */ + private boolean isDowngrade() { + return Direction.DOWNGRADE.name().equalsIgnoreCase(getCommandParameterValue(UPGRADE_DIRECTION_KEY)); + } + + /** + * Retrieves the target stack ID for the stack upgrade or downgrade operation. + * + * @param cluster the cluster + * @return the target {@link StackId} + * @throws AmbariException if multiple stack id's are detected + */ + private StackId getTargetStackId(Cluster cluster) throws AmbariException { + UpgradeContext upgradeContext = getUpgradeContext(cluster); + + // !!! FIXME in a per-service view, what does this become? + Set stackIds = new HashSet<>(); + + for (Service service : cluster.getServices().values()) { + RepositoryVersionEntity targetRepoVersion = upgradeContext.getTargetRepositoryVersion(service.getName()); + StackId targetStackId = targetRepoVersion.getStackId(); + stackIds.add(targetStackId); + } + + if (1 != stackIds.size()) { + throw new AmbariException("Services are deployed from multiple stacks and cannot determine a unique one."); + } + + return stackIds.iterator().next(); + } + + /** + * Find and iterate through the {@link ServiceComponentHost} objects for the current {@link Cluster} + * to calculate property updates and auth-to-local rules. + * + * @param cluster the cluster + * @param kerberosDescriptor the Kerberos descriptor + * @param currentConfigurations the current configurations for the cluster + * @param kerberosConfigurations the (Kerberos-specific) configuration updates + * @param propertiesToBeIgnored a map to store properties that should be ignored by operations that update property values + * @throws AmbariException if an issue occurs + */ + private void processServiceComponentHosts(Cluster cluster, KerberosDescriptor kerberosDescriptor, + Map> currentConfigurations, + Map> kerberosConfigurations, + Map> propertiesToBeIgnored) + throws AmbariException { + + Collection hosts = cluster.getHosts(); + if (!hosts.isEmpty()) { + // Create the context to use for filtering Kerberos Identities based on the state of the cluster + Map filterContext = new HashMap<>(); + filterContext.put("configurations", currentConfigurations); + filterContext.put("services", cluster.getServices().keySet()); + + try { + Map> propertiesToIgnore = null; + + for (Host host : hosts) { + // Iterate over the components installed on the current host to get the service and + // component-level Kerberos descriptors in order to determine which principals, + // keytab files, and configurations need to be created or updated. + for (ServiceComponentHost sch : cluster.getServiceComponentHosts(host.getHostName())) { + String hostName = sch.getHostName(); + + String serviceName = sch.getServiceName(); + String componentName = sch.getServiceComponentName(); + + KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(serviceName); + + if (serviceDescriptor != null) { + List serviceIdentities = serviceDescriptor.getIdentities(true, filterContext); + + // Add service-level principals (and keytabs) + kerberosHelper.addIdentities(null, serviceIdentities, - null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, false); ++ null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations); + propertiesToIgnore = gatherPropertiesToIgnore(serviceIdentities, propertiesToIgnore); + + KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(componentName); + + if (componentDescriptor != null) { + List componentIdentities = componentDescriptor.getIdentities(true, filterContext); + + // Calculate the set of configurations to update and replace any variables + // using the previously calculated Map of configurations for the host. + kerberosHelper.mergeConfigurations(kerberosConfigurations, + componentDescriptor.getConfigurations(true), currentConfigurations, null); + + // Add component-level principals (and keytabs) + kerberosHelper.addIdentities(null, componentIdentities, - null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, false); ++ null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations); + propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore); + } + } + } + } + + // Add ambari-server identities only if 'kerberos-env.create_ambari_principal = true' + if (kerberosHelper.createAmbariIdentities(currentConfigurations.get(KERBEROS_ENV))) { + List ambariIdentities = kerberosHelper.getAmbariServerIdentities(kerberosDescriptor); + + for (KerberosIdentityDescriptor identity : ambariIdentities) { + // If the identity represents the ambari-server user, use the component name "AMBARI_SERVER_SELF" + // so it can be distinguished between other identities related to the AMBARI-SERVER + // component. + String componentName = KerberosHelper.AMBARI_SERVER_KERBEROS_IDENTITY_NAME.equals(identity.getName()) + ? "AMBARI_SERVER_SELF" + : "AMBARI_SERVER"; + + List componentIdentities = Collections.singletonList(identity); + kerberosHelper.addIdentities(null, componentIdentities, - null, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations, false); ++ null, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations); + propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore); + } + } + + if ((propertiesToBeIgnored != null) && (propertiesToIgnore != null)) { + propertiesToBeIgnored.putAll(propertiesToIgnore); + } + } catch (IOException e) { + throw new AmbariException(e.getMessage(), e); + } + } + } + + private Map> gatherPropertiesToIgnore(List identities, + Map> propertiesToIgnore) { + Map> identityConfigurations = kerberosHelper.getIdentityConfigurations(identities); + if (!MapUtils.isEmpty(identityConfigurations)) { + if (propertiesToIgnore == null) { + propertiesToIgnore = new HashMap<>(); + } + + for (Map.Entry> entry : identityConfigurations.entrySet()) { + String configType = entry.getKey(); + Map properties = entry.getValue(); + + if (MapUtils.isEmpty(properties)) { + Set propertyNames = propertiesToIgnore.get(configType); + if (propertyNames == null) { + propertyNames = new HashSet<>(); + propertiesToIgnore.put(configType, propertyNames); + } + propertyNames.addAll(properties.keySet()); + } + } + } + + return propertiesToIgnore; + } + + /** + * Processes configuration changes to determine if any work needs to be done. + *

+ * If work is to be done, a data file containing the details is created so it they changes may be + * processed in the appropriate stage. + * + * @param cluster the cluster + * @param targetStackId the target stack id + * @param kerberosConfigurations the Kerberos-specific configuration map + * @param propertiesToBeRemoved a map of properties to be removed from the current configuration, + * grouped by configuration type. + * @param variableReplaments replacement values to use when attempting to perform variable replacements on the property names + * @throws AmbariException if an issue is encountered + */ + private void processConfigurationChanges(Cluster cluster, StackId targetStackId, + KerberosDescriptor kerberosDescriptor, + Map> kerberosConfigurations, + Map> propertiesToBeRemoved, + Map> variableReplaments) + throws AmbariException { + actionLog.writeStdOut("Determining configuration changes"); + + if (!kerberosConfigurations.isEmpty()) { + Map installedServices = cluster.getServices(); + + // Build a map of configuration types to properties that indicate which properties should be altered + // This map should contain only properties defined in service-level Kerberos descriptors that + // have been flagged to be preconfigured and that have not yet been installed. + Map> propertyFilter = new HashMap<>(); + Map serviceDescriptors = kerberosDescriptor.getServices(); + if (serviceDescriptors != null) { + for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors.values()) { + if (!installedServices.containsKey(serviceDescriptor.getName()) && serviceDescriptor.shouldPreconfigure()) { + buildFilter(Collections.singleton(serviceDescriptor), propertyFilter, variableReplaments); + } + } + } + + // Add the auth-to-local rule configuration specifications to the filter + Map> authToLocalProperties = kerberosHelper.translateConfigurationSpecifications(kerberosDescriptor.getAllAuthToLocalProperties()); + if (!MapUtils.isEmpty(authToLocalProperties)) { + for (Map.Entry> entry : authToLocalProperties.entrySet()) { + Set properties = entry.getValue(); + + if (!CollectionUtils.isEmpty(properties)) { + String configurationType = entry.getKey(); + + Set propertyNames = propertyFilter.get(configurationType); + if (propertyNames == null) { + propertyNames = new HashSet<>(); + propertyFilter.put(configurationType, propertyNames); + } + + propertyNames.addAll(properties); + } + } + } + + Set visitedTypes = new HashSet<>(); + + for (Map.Entry> entry : kerberosConfigurations.entrySet()) { + String configType = entry.getKey(); + + String service = cluster.getServiceByConfigType(configType); + Set allowedProperties = propertyFilter.get(configType); + + // Update properties for services that are installed and not filtered out + if (installedServices.containsKey(service) && !CollectionUtils.isEmpty(allowedProperties)) { + Map propertiesToUpdate = entry.getValue(); + Set propertiesToRemove = (propertiesToBeRemoved == null) ? null : propertiesToBeRemoved.get(configType); + + // Filter the properties to update + if (propertiesToUpdate != null) { + Iterator> mapIterator = propertiesToUpdate.entrySet().iterator(); + while (mapIterator.hasNext()) { + Map.Entry mapEntry = mapIterator.next(); + + if (!allowedProperties.contains(mapEntry.getKey())) { + mapIterator.remove(); + } + } + } + + // Filter the properties to remove + if (propertiesToRemove != null) { + Iterator setIterator = propertiesToRemove.iterator(); + while (setIterator.hasNext()) { + String setEntry = setIterator.next(); + if (!allowedProperties.contains(setEntry)) { + setIterator.remove(); + } + } + } + + visitedTypes.add(configType); + + if (!MapUtils.isEmpty(propertiesToUpdate) || !CollectionUtils.isEmpty(propertiesToRemove)) { + if (!MapUtils.isEmpty(propertiesToUpdate)) { + for (Map.Entry property : propertiesToUpdate.entrySet()) { + actionLog.writeStdOut(String.format("Setting: %s/%s = %s", configType, property.getKey(), property.getValue())); + } + } + + if (!CollectionUtils.isEmpty(propertiesToRemove)) { + for (String property : propertiesToRemove) { + actionLog.writeStdOut(String.format("Removing: %s/%s", configType, property)); + } + } + + configHelper.updateConfigType(cluster, targetStackId, + ambariManagementController, configType, propertiesToUpdate, propertiesToRemove, + ambariManagementController.getAuthName(), "Preconfiguring for Kerberos during upgrade"); + } + } + } + + if (!MapUtils.isEmpty(propertiesToBeRemoved)) { + for (Map.Entry> entry : propertiesToBeRemoved.entrySet()) { + String configType = entry.getKey(); + + if (!visitedTypes.contains(configType)) { + Set propertiesToRemove = entry.getValue(); + + if (!CollectionUtils.isEmpty(propertiesToRemove)) { + for (String property : propertiesToRemove) { + actionLog.writeStdOut(String.format("Removing: %s/%s", configType, property)); + } + + configHelper.updateConfigType(cluster, targetStackId, + ambariManagementController, configType, null, entry.getValue(), + ambariManagementController.getAuthName(), "Preconfiguring for Kerberos during upgrade"); + } + } + } + } + } + } + + /** + * Adds entries to the property filter (propertyFilter) found in the {@link KerberosConfigurationDescriptor}s + * within the specified node of the Kerberos descriptor. + * + * @param containers the Kerberos descriptor containers to process + * @param propertyFilter the property filter map to update + * @param replacements replacement values to use when attempting to perform variable replacements on the property names + * @throws AmbariException if an issue occurs while replacing variables in the property names + */ + private void buildFilter(Collection containers, + Map> propertyFilter, + Map> replacements) + throws AmbariException { + if (containers != null) { + for (AbstractKerberosDescriptorContainer container : containers) { + Map configurationDescriptors = container.getConfigurations(false); + + if (!MapUtils.isEmpty(configurationDescriptors)) { + for (KerberosConfigurationDescriptor configurationDescriptor : configurationDescriptors.values()) { + Map properties = configurationDescriptor.getProperties(); + + if (!MapUtils.isEmpty(properties)) { + String configType = configurationDescriptor.getType(); + + Set propertyNames = propertyFilter.get(configType); + if (propertyNames == null) { + propertyNames = new HashSet<>(); + propertyFilter.put(configType, propertyNames); + } + + // Replace variables in the property name. For example ${knox-env/knox_user}. + for (String propertyName : properties.keySet()) { + propertyNames.add(variableReplacementHelper.replaceVariables(propertyName, replacements)); + } + } + } + } + + Collection childContainers = container.getChildContainers(); + if (childContainers != null) { + buildFilter(childContainers, propertyFilter, replacements); + } + } + } + } + } + http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java index f68570b,2992027..db9d178 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java @@@ -25,13 -29,17 +25,11 @@@ import java.util.concurrent.ExecutorSer import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.MetainfoDAO; - import org.apache.ambari.server.orm.entities.MetainfoEntity; +import org.apache.ambari.server.state.stack.LatestRepoCallable; import org.apache.ambari.server.state.stack.OsFamily; -import org.apache.ambari.server.state.stack.RepoUrlInfoCallable; -import org.apache.ambari.server.state.stack.RepoUrlInfoCallable.RepoUrlInfoResult; -import org.apache.ambari.server.state.stack.RepoVdfCallable; -import org.apache.commons.collections.MapUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Provides external functionality to the Stack framework. http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index d3ad351,6dc2b93..3688727 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@@ -1236,6 -1276,9 +1232,9 @@@ public class StackModule extends BaseMo RepositoryXml serviceRepoXml = ssd.getRepoFile(); if (null != serviceRepoXml) { repos.addAll(serviceRepoXml.getRepositories()); + if (null != serviceRepoXml.getLatestURI()) { - registerRepoUpdateTask(serviceRepoXml); ++ stackContext.registerRepoUpdateTask(serviceRepoXml.getLatestURI(), this); + } } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java index 3e04a87,90dd611..23f4078 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java @@@ -26,12 -26,8 +26,9 @@@ import java.util.Set import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.ClusterResponse; import org.apache.ambari.server.controller.ServiceConfigVersionResponse; +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData; import org.apache.ambari.server.events.ClusterConfigChangedEvent; import org.apache.ambari.server.metadata.RoleCommandOrder; - import org.apache.ambari.server.orm.entities.ClusterVersionEntity; - import org.apache.ambari.server.orm.entities.HostEntity; - import org.apache.ambari.server.orm.entities.HostVersionEntity; import org.apache.ambari.server.orm.entities.PrivilegeEntity; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.UpgradeEntity; http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/Clusters.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java index 31a00ca,8ab1fe9..d57b5d6 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java @@@ -29,9 -29,10 +29,8 @@@ public class RepositoryInfo private String osType; private String repoId; private String repoName; - private String distribution; - private String components; private String mirrorsList; private String defaultBaseUrl; - private String latestBaseUrl; private boolean repoSaved = false; private boolean unique = false; private boolean ambariManagedRepositories = true; @@@ -186,21 -191,23 +171,20 @@@ Objects.equal(osType, that.osType) && Objects.equal(repoId, that.repoId) && Objects.equal(repoName, that.repoName) && - Objects.equal(distribution, that.distribution) && - Objects.equal(components, that.components) && Objects.equal(mirrorsList, that.mirrorsList) && Objects.equal(defaultBaseUrl, that.defaultBaseUrl) && - Objects.equal(latestBaseUrl, that.latestBaseUrl) && Objects.equal(ambariManagedRepositories, that.ambariManagedRepositories); } @Override public int hashCode() { - return Objects.hashCode(baseUrl, osType, repoId, repoName, mirrorsList, defaultBaseUrl, latestBaseUrl, repoSaved, unique, ambariManagedRepositories); - return Objects.hashCode(baseUrl, osType, repoId, repoName, distribution, components, mirrorsList, defaultBaseUrl, - ambariManagedRepositories); ++ return Objects.hashCode(baseUrl, osType, repoId, repoName, mirrorsList, defaultBaseUrl, repoSaved, unique, ambariManagedRepositories); } public RepositoryResponse convertToResponse() { return new RepositoryResponse(getBaseUrl(), getOsType(), getRepoId(), - getRepoName(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl()); - getRepoName(), getDistribution(), getComponents(), getMirrorsList(), getDefaultBaseUrl()); ++ getRepoName(), getMirrorsList(), getDefaultBaseUrl()); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/Service.java index 91884f6,65189ca..b6203f9 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java @@@ -23,7 -22,7 +23,8 @@@ import java.util.Set import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.ServiceResponse; +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData; + import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; public interface Service { http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java index d755568,9fb2aba..ed19023 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java @@@ -23,7 -22,7 +23,8 @@@ import java.util.Set import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.ServiceComponentResponse; +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData; + import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; public interface ServiceComponent { @@@ -102,5 -102,20 +108,20 @@@ ServiceComponentHost addServiceComponentHost( String hostName) throws AmbariException; - void delete() throws AmbariException; + void delete(DeleteHostComponentStatusMetaData deleteMetaData); + + /** + * This method computes the state of the repository that's associated with the desired + * version. It is used, for example, when a host component reports its version and the + * state can be in flux. + * + * @param reportedVersion + * @throws AmbariException + */ + void updateRepositoryState(String reportedVersion) throws AmbariException; + + /** + * @return the repository state for the desired version + */ + RepositoryVersionState getRepositoryState(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java index 08f9f41,5ff9e37..575193f --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java @@@ -23,9 -23,8 +23,9 @@@ import java.util.Map import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.ServiceComponentHostResponse; +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData; import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity; - import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; + import org.apache.ambari.server.orm.entities.HostVersionEntity; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; @@@ -130,30 -107,9 +108,9 @@@ public interface ServiceComponentHost * * @param version component version (e.g. 2.2.0.0-2041) */ - void setVersion(String version); + void setVersion(String version) throws AmbariException; /** - * Gets the desired security state for this ServiceComponent - *

- * The returned SecurityState is a valid endpoint state where - * SecurityState.isEndpoint() == true. - * - * @return the desired SecurityState for this ServiceComponent - */ - SecurityState getDesiredSecurityState(); - - /** - * Sets the desired security state for this ServiceComponent - *

- * It is expected that the new SecurityState is a valid endpoint state such that - * SecurityState.isEndpoint() == true. - * - * @param securityState the desired SecurityState for this ServiceComponent - * @throws AmbariException if the new state is not an endpoint state - */ - void setDesiredSecurityState(SecurityState securityState) throws AmbariException; - - /** * @param upgradeState the upgrade state */ void setUpgradeState(UpgradeState upgradeState); http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java index 32ecbf8,22c97ed..74b592b --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java @@@ -19,10 -19,9 +19,11 @@@ package org.apache.ambari.server.state; import java.util.HashMap; +import java.util.HashSet; + import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReadWriteLock; @@@ -32,9 -31,10 +33,10 @@@ import org.apache.ambari.server.AmbariE import org.apache.ambari.server.ObjectNotFoundException; import org.apache.ambari.server.ServiceComponentHostNotFoundException; import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.controller.MaintenanceStateHelper; import org.apache.ambari.server.controller.ServiceComponentResponse; +import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData; import org.apache.ambari.server.events.ServiceComponentRecoveryChangedEvent; + import org.apache.ambari.server.events.listeners.upgrade.StackVersionListener; import org.apache.ambari.server.events.publishers.AmbariEventPublisher; import org.apache.ambari.server.orm.dao.ClusterServiceDAO; import org.apache.ambari.server.orm.dao.HostComponentDesiredStateDAO; @@@ -84,11 -92,15 +94,12 @@@ public class ServiceComponentImpl imple */ private final long desiredStateEntityId; - /** - * Data access object used for lookup up stacks. - */ - private final StackDAO stackDAO; + @Inject + private RepositoryVersionDAO repoVersionDAO; + + @Inject + private HostComponentStateDAO hostComponentDAO; - @Inject - private MaintenanceStateHelper maintenanceStateHelper; - @AssistedInject public ServiceComponentImpl(@Assisted Service service, @Assisted String componentName, AmbariMetaInfo ambariMetaInfo, http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java index c22bfb2,1104d19..8ba41ae --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java @@@ -18,14 -18,13 +18,16 @@@ package org.apache.ambari.server.state; + import java.util.ArrayList; + import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@@ -588,25 -577,22 +595,30 @@@ public class ServiceImpl implements Ser @Override @Transactional - public void delete() throws AmbariException { + public void delete(DeleteHostComponentStatusMetaData deleteMetaData) { + List components = getComponents(); // XXX temporal coupling, need to call this BEFORE deletingAllComponents - deleteAllComponents(); - deleteAllServiceConfigs(); + deleteAllComponents(deleteMetaData); + if (deleteMetaData.getAmbariException() != null) { + return; + } + try { + deleteAllServiceConfigs(); - StackId stackId = getDesiredStackId(); + removeEntities(); + } catch (AmbariException e) { + deleteMetaData.setAmbariException(e); + return; + } - removeEntities(); ++ StackId stackId = getDesiredStackId(); + // publish the service removed event - StackId stackId = cluster.getDesiredStackVersion(); + if (null == stackId) { + return; + } ServiceRemovedEvent event = new ServiceRemovedEvent(getClusterId(), stackId.getStackName(), - stackId.getStackVersion(), getName()); + stackId.getStackVersion(), getName(), components); eventPublisher.publish(event); } @@@ -635,16 -626,9 +652,12 @@@ @Override public MaintenanceState getMaintenanceState() { - return getServiceDesiredStateEntity().getMaintenanceState(); + if (maintenanceState.get() == null) { + maintenanceState.set(getServiceDesiredStateEntity().getMaintenanceState()); + } + return maintenanceState.get(); } - private ClusterServiceEntity getServiceEntity() { - return clusterServiceDAO.findByPK(serviceEntityPK); - } - private ClusterServiceEntityPK getServiceEntityPK(ClusterServiceEntity serviceEntity) { ClusterServiceEntityPK pk = new ClusterServiceEntityPK(); pk.setClusterId(serviceEntity.getClusterId()); http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java index 353dd86,a3886ab..6184b94 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java @@@ -33,6 -33,6 +33,7 @@@ import org.apache.ambari.server.control import org.apache.ambari.server.stack.Validable; import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.ConfigUpgradePack; ++import org.apache.ambari.server.state.stack.LatestRepoCallable; import org.apache.ambari.server.state.stack.RepositoryXml; import org.apache.ambari.server.state.stack.StackRoleCommandOrder; import org.apache.ambari.server.state.stack.UpgradePack; http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index 92e01c2,8f9d8e1..464cb41 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@@ -491,6 -553,51 +553,36 @@@ public class UpgradeHelper } /** + * Merges two service check groups when they have been orchestrated back-to-back. + * @param newHolder the "new" group holder, which was orchestrated after the "old" one + * @param oldHolder the "old" group holder, which is one that was already orchestrated + */ + @SuppressWarnings("unchecked") + private void mergeServiceChecks(UpgradeGroupHolder newHolder, UpgradeGroupHolder oldHolder) { + + LinkedHashSet priority = new LinkedHashSet<>(); + LinkedHashSet others = new LinkedHashSet<>(); + - Set extraKeys = new HashSet<>(); - LinkedHashSet extras = new LinkedHashSet<>(); - + for (List holderItems : new List[] { oldHolder.items, newHolder.items }) { + for (StageWrapper stageWrapper : holderItems) { - if (stageWrapper instanceof ServiceCheckStageWrapper) { - ServiceCheckStageWrapper wrapper = (ServiceCheckStageWrapper) stageWrapper; - if (wrapper.priority) { - priority.add(stageWrapper); - } else { - others.add(stageWrapper); - } ++ ServiceCheckStageWrapper wrapper = (ServiceCheckStageWrapper) stageWrapper; ++ ++ if (wrapper.priority) { ++ priority.add(stageWrapper); + } else { - // !!! It's a good chance that back-to-back service check groups are adding the - // same non-service-check wrappers. - // this should be "equal enough" to prevent them from duplicating on merge - String key = stageWrapper.toString(); - if (!extraKeys.contains(key)) { - extras.add(stageWrapper); - extraKeys.add(key); - } ++ others.add(stageWrapper); + } - + } + } + + // !!! remove duplicate wrappers that are now in the priority list + others = new LinkedHashSet<>(CollectionUtils.subtract(others, priority)); + + oldHolder.items = Lists.newLinkedList(priority); + oldHolder.items.addAll(others); - oldHolder.items.addAll(extras); + } + + /** * Walks through the UpgradeGroupHolder and updates titles and manual tasks, * replacing keyword tokens needed for display purposes * @@@ -768,8 -905,250 +890,241 @@@ serviceComponentHost.setVersion(StackVersionListener.UNKNOWN_VERSION); } } - serviceComponent.setDesiredVersion(desiredVersion); + // set component desired repo + serviceComponent.setDesiredRepositoryVersion(targetRepositoryVersion); + } + } + } + + /** + * Handles the creation or resetting of configurations based on whether an + * upgrade or downgrade is occurring. This method will not do anything when + * the service is not crossing major stack versions, since, by definition, no + * new configurations are automatically created when upgrading with the same + * stack (ie HDP 2.2.0.0 -> HDP 2.2.1.0). + *

+ * When upgrading or downgrade between stacks (HDP 2.2.0.0 -> HDP 2.3.0.0) + * then this will perform the following: + *

    + *
  • Upgrade: Create new configurations that are a merge between the source + * stack and the target stack. If a value has changed between stacks, then the + * target stack value should be taken unless the cluster's value differs from + * the old stack. This can occur if a property has been customized after - * installation. Read-only properties, however, are always taken from the new - * stack.
    • ++ * installation. + *
  • Downgrade: Reset the latest configurations from the service's original + * stack. The new configurations that were created on upgrade must be left + * intact until all components have been reverted, otherwise heartbeats will + * fail due to missing configurations.
    • + *
+ * + * @param upgradeContext + * the upgrade context (not {@code null}). + * @throws AmbariException + */ + private void processConfigurationsIfRequired(UpgradeContext upgradeContext) + throws AmbariException { + + AmbariManagementController controller = m_controllerProvider.get(); + + Cluster cluster = upgradeContext.getCluster(); + Direction direction = upgradeContext.getDirection(); + String userName = controller.getAuthName(); + Set servicesInUpgrade = upgradeContext.getSupportedServices(); + + Set clusterConfigTypes = new HashSet<>(); + Set processedClusterConfigTypes = new HashSet<>(); + + // merge or revert configurations for any service that needs it + for (String serviceName : servicesInUpgrade) { + RepositoryVersionEntity sourceRepositoryVersion = upgradeContext.getSourceRepositoryVersion(serviceName); + RepositoryVersionEntity targetRepositoryVersion = upgradeContext.getTargetRepositoryVersion(serviceName); + StackId sourceStackId = sourceRepositoryVersion.getStackId(); + StackId targetStackId = targetRepositoryVersion.getStackId(); + + // only work with configurations when crossing stacks + if (sourceStackId.equals(targetStackId)) { + RepositoryVersionEntity associatedRepositoryVersion = upgradeContext.getRepositoryVersion(); + LOG.info( + "The {} {} {} will not change stack configurations for {} since the source and target are both {}", + direction.getText(false), direction.getPreposition(), + associatedRepositoryVersion.getVersion(), serviceName, targetStackId); + + continue; + } + + ConfigHelper configHelper = m_configHelperProvider.get(); + + // downgrade is easy - just remove the new and make the old current + if (direction == Direction.DOWNGRADE) { + cluster.applyLatestConfigurations(targetStackId, serviceName); + continue; + } + - // the auto-merge must take read-only properties even if they have changed - // - if the properties was read-only in the source stack, then we must - // take the new stack's value - Map> readOnlyProperties = getReadOnlyProperties(sourceStackId, serviceName); - + // upgrade is a bit harder - we have to merge new stack configurations in + + // populate a map of default configurations for the service on the old + // stack (this is used when determining if a property has been + // customized and should be overriden with the new stack value) + Map> oldServiceDefaultConfigsByType = configHelper.getDefaultProperties( + sourceStackId, serviceName); + + // populate a map with default configurations from the new stack + Map> newServiceDefaultConfigsByType = configHelper.getDefaultProperties( + targetStackId, serviceName); + + if (null == oldServiceDefaultConfigsByType || null == newServiceDefaultConfigsByType) { + continue; + } + + Set foundConfigTypes = new HashSet<>(); + + // find the current, existing configurations for the service + List existingServiceConfigs = new ArrayList<>(); + List latestServiceConfigs = m_serviceConfigDAO.getLastServiceConfigsForService( + cluster.getClusterId(), serviceName); + + for (ServiceConfigEntity serviceConfig : latestServiceConfigs) { + List existingConfigurations = serviceConfig.getClusterConfigEntities(); + for (ClusterConfigEntity currentServiceConfig : existingConfigurations) { + String configurationType = currentServiceConfig.getType(); + + Config currentClusterConfigForService = cluster.getDesiredConfigByType(configurationType); + existingServiceConfigs.add(currentClusterConfigForService); + foundConfigTypes.add(configurationType); + } + } + + // !!! these are the types that come back from the config helper, but are not part of the service. + @SuppressWarnings("unchecked") + Set missingConfigTypes = new HashSet<>(CollectionUtils.subtract(oldServiceDefaultConfigsByType.keySet(), + foundConfigTypes)); + + for (String missingConfigType : missingConfigTypes) { + Config config = cluster.getDesiredConfigByType(missingConfigType); + if (null != config) { + existingServiceConfigs.add(config); + clusterConfigTypes.add(missingConfigType); + } + } + + // now that we have found, old, new, and existing confgs, overlay the + // existing on top of the new + for (Config existingServiceConfig : existingServiceConfigs) { + String configurationType = existingServiceConfig.getType(); + + // get current stack default configurations on install + Map oldServiceDefaultConfigs = oldServiceDefaultConfigsByType.get( + configurationType); + + // NPE sanity for current stack defaults + if (null == oldServiceDefaultConfigs) { + oldServiceDefaultConfigs = Collections.emptyMap(); + } + + // get the existing configurations + Map existingConfigurations = existingServiceConfig.getProperties(); + + // get the new configurations - Map newDefaultConfigurations = newServiceDefaultConfigsByType.get(configurationType); ++ Map newDefaultConfigurations = newServiceDefaultConfigsByType.get( ++ configurationType); + + // if the new stack configurations don't have the type, then simply add + // all of the existing in + if (null == newDefaultConfigurations) { + newServiceDefaultConfigsByType.put(configurationType, existingConfigurations); + continue; + } else { + // Remove any configs in the new stack whose value is NULL, unless + // they currently exist and the value is not NULL. + Iterator> iter = newDefaultConfigurations.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + if (entry.getValue() == null) { + iter.remove(); + } + } + } + - // process every existing configuration property for this configuration type ++ // process every existing configuration property for this configuration ++ // type + for (Map.Entry existingConfigurationEntry : existingConfigurations.entrySet()) { + String existingConfigurationKey = existingConfigurationEntry.getKey(); + String existingConfigurationValue = existingConfigurationEntry.getValue(); + + // if there is already an entry, we now have to try to determine if + // the value was customized after stack installation + if (newDefaultConfigurations.containsKey(existingConfigurationKey)) { + String newDefaultConfigurationValue = newDefaultConfigurations.get( + existingConfigurationKey); + + if (!StringUtils.equals(existingConfigurationValue, newDefaultConfigurationValue)) { + // the new default is different from the existing cluster value; + // only override the default value if the existing value differs + // from the original stack + String oldDefaultValue = oldServiceDefaultConfigs.get(existingConfigurationKey); + - // see if this property is a read-only property which means that - // we shouldn't care if it was changed - we should take the new - // stack's value - Set readOnlyPropertiesForType = readOnlyProperties.get(configurationType); - boolean readOnly = (null != readOnlyPropertiesForType - && readOnlyPropertiesForType.contains(existingConfigurationKey)); - - if (!readOnly && !StringUtils.equals(existingConfigurationValue, oldDefaultValue)) { - // at this point, we've determined that there is a difference - // between default values between stacks, but the value was also - // customized, so keep the customized value ++ if (!StringUtils.equals(existingConfigurationValue, oldDefaultValue)) { ++ // at this point, we've determined that there is a ++ // difference ++ // between default values between stacks, but the value was ++ // also customized, so keep the customized value + newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue); + } + } + } else { - // there is no entry in the map, so add the existing key/value pair ++ // there is no entry in the map, so add the existing key/value ++ // pair + newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue); + } + } + + /* + for every new configuration which does not exist in the existing + configurations, see if it was present in the current stack + + stack 2.x has foo-site/property (on-ambari-upgrade is false) + stack 2.y has foo-site/property + the current cluster (on 2.x) does not have it + + In this case, we should NOT add it back as clearly stack advisor has removed it + */ + Iterator> newDefaultConfigurationsIterator = newDefaultConfigurations.entrySet().iterator(); + while (newDefaultConfigurationsIterator.hasNext()) { + Map.Entry newConfigurationEntry = newDefaultConfigurationsIterator.next(); + String newConfigurationPropertyName = newConfigurationEntry.getKey(); + if (oldServiceDefaultConfigs.containsKey(newConfigurationPropertyName) + && !existingConfigurations.containsKey(newConfigurationPropertyName)) { + LOG.info( + "The property {}/{} exists in both {} and {} but is not part of the current set of configurations and will therefore not be included in the configuration merge", + configurationType, newConfigurationPropertyName, sourceStackId, targetStackId); + + // remove the property so it doesn't get merged in + newDefaultConfigurationsIterator.remove(); + } + } + } + + if (null != newServiceDefaultConfigsByType) { + + for (String clusterConfigType : clusterConfigTypes) { + if (processedClusterConfigTypes.contains(clusterConfigType)) { + newServiceDefaultConfigsByType.remove(clusterConfigType); + } else { + processedClusterConfigTypes.add(clusterConfigType); + } + + } + + Set configTypes = newServiceDefaultConfigsByType.keySet(); + LOG.warn("The upgrade will create the following configurations for stack {}: {}", + targetStackId, StringUtils.join(configTypes, ',')); + + String serviceVersionNote = String.format("%s %s %s", direction.getText(true), + direction.getPreposition(), upgradeContext.getRepositoryVersion().getVersion()); + + configHelper.createConfigTypes(cluster, targetStackId, controller, + newServiceDefaultConfigsByType, userName, serviceVersionNote); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinitionHash.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/ParameterizedSource.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Source.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 7147ce3,9c0b0ca..e6da65e --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@@ -145,14 -128,12 +132,16 @@@ import org.apache.ambari.server.state.U import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.state.configgroup.ConfigGroupFactory; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; + import org.apache.ambari.server.state.repository.ClusterVersionSummary; + import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.scheduler.RequestExecution; import org.apache.ambari.server.state.scheduler.RequestExecutionFactory; +import org.apache.ambari.server.state.stack.upgrade.Direction; +import org.apache.ambari.server.state.svccomphost.ServiceComponentHostSummary; +import org.apache.ambari.server.topology.TopologyDeleteFormer; import org.apache.ambari.server.topology.TopologyRequest; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -2315,15 -1545,10 +1569,15 @@@ public class ClusterImpl implements Clu serviceConfigEntity.setVersion(nextServiceConfigVersion); serviceConfigEntity.setUser(user); serviceConfigEntity.setNote(note); - serviceConfigEntity.setStack(clusterEntity.getDesiredStack()); + serviceConfigEntity.setStack(stackEntity); serviceConfigDAO.create(serviceConfigEntity); + List groupHostNames = null; if (configGroup != null) { + if (MapUtils.isNotEmpty(configGroup.getHosts())) { + groupHostNames = configGroup.getHosts().entrySet().stream().map(h -> h.getValue().getHostName()) + .collect(Collectors.toList()); + } serviceConfigEntity.setHostIds(new ArrayList<>(configGroup.getHosts().keySet())); serviceConfigEntity = serviceConfigDAO.merge(serviceConfigEntity); } @@@ -2585,19 -1806,31 +1837,32 @@@ throw new ObjectNotFoundException("Service config version with serviceName={} and version={} not found"); } + String configGroupName = null; // disable all configs related to service if (serviceConfigEntity.getGroupId() == null) { + // Here was fixed bug with entity changes revert. More you can find here AMBARI-21173. + // This issue reproduces only if you are changing same entity in first and second loop. + // In that case eclipselink will revert changes to cached, if entity has fluchGroup and it + // needs to be refreshed. Actually we don't need to change same antities in few steps, so i + // decided to filter out. duplicates and do not change them. It will be better for performance and bug will be fixed. Collection configTypes = serviceConfigTypes.get(serviceName); List enabledConfigs = clusterDAO.getEnabledConfigsByTypes(clusterId, configTypes); + List serviceConfigEntities = serviceConfigEntity.getClusterConfigEntities(); + ArrayList duplicatevalues = new ArrayList<>(serviceConfigEntities); + duplicatevalues.retainAll(enabledConfigs); + for (ClusterConfigEntity enabledConfig : enabledConfigs) { - enabledConfig.setSelected(false); - clusterDAO.merge(enabledConfig); + if (!duplicatevalues.contains(enabledConfig)) { + enabledConfig.setSelected(false); + clusterDAO.merge(enabledConfig); + } } - for (ClusterConfigEntity configEntity : serviceConfigEntity.getClusterConfigEntities()) { - configEntity.setSelected(true); - clusterDAO.merge(configEntity); + for (ClusterConfigEntity configEntity : serviceConfigEntities) { + if (!duplicatevalues.contains(configEntity)) { + configEntity.setSelected(true); + clusterDAO.merge(configEntity); + } } } else { Long configGroupId = serviceConfigEntity.getGroupId(); @@@ -3159,9 -2415,14 +2463,14 @@@ // since the entities which were modified came from the cluster entity's // list to begin with, we can just save them right back - no need for a // new collection since the entity instances were modified directly - clusterEntity = clusterDAO.merge(clusterEntity, true); + clusterEntity = clusterDAO.merge(clusterEntity); cacheConfigurations(); + + LOG.info( + "Applied latest configurations for {} on stack {}. The the following types were modified: {}", + serviceName, stackId, StringUtils.join(configTypesForService, ',')); + } finally { clusterGlobalLock.writeLock().unlock(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java index 702d776,5ac1ac3..8743f63 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java @@@ -46,13 -40,10 +46,11 @@@ import org.apache.ambari.server.control import org.apache.ambari.server.events.HostRegisteredEvent; import org.apache.ambari.server.events.HostsAddedEvent; import org.apache.ambari.server.events.HostsRemovedEvent; +import org.apache.ambari.server.events.TopologyUpdateEvent; import org.apache.ambari.server.events.publishers.AmbariEventPublisher; import org.apache.ambari.server.orm.dao.ClusterDAO; - import org.apache.ambari.server.orm.dao.ClusterVersionDAO; import org.apache.ambari.server.orm.dao.HostConfigMappingDAO; import org.apache.ambari.server.orm.dao.HostDAO; - import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; import org.apache.ambari.server.orm.dao.HostStateDAO; import org.apache.ambari.server.orm.dao.HostVersionDAO; import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO; @@@ -276,17 -243,9 +260,17 @@@ public class ClustersImpl implements Cl clusters.put(clusterName, cluster); clustersById.put(cluster.getClusterId(), cluster); clusterHostMap.put(clusterName, - Collections.newSetFromMap(new ConcurrentHashMap())); + Collections.newSetFromMap(new ConcurrentHashMap<>())); cluster.setCurrentStackVersion(stackId); + + TreeMap addedClusters = new TreeMap<>(); + TopologyCluster addedCluster = new TopologyCluster(); + addedClusters.put(Long.toString(cluster.getClusterId()), addedCluster); + TopologyUpdateEvent topologyUpdateEvent = new TopologyUpdateEvent(addedClusters, + TopologyUpdateEvent.EventType.UPDATE); + m_topologyHolder.get().updateData(topologyUpdateEvent); + m_metadataHolder.get().updateData(m_ambariManagementController.get().getClusterMetadata(cluster)); } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java index 0b54eec,274a425..2a9e69e --- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java @@@ -595,14 -580,9 +586,10 @@@ public class HostImpl implements Host writeLock.unlock(); } if (oldState != getState()) { + ambariEventPublisher.publish(new HostStateUpdateEvent(getHostName(), getState())); if (LOG.isDebugEnabled()) { - LOG.debug("Host transitioned to a new state" - + ", host=" + getHostName() - + ", oldState=" + oldState - + ", currentState=" + getState() - + ", eventType=" + event.getType().name() - + ", event=" + event); + LOG.debug("Host transitioned to a new state, host={}, oldState={}, currentState={}, eventType={}, event={}", + getHostName(), oldState, getState(), event.getType().name(), event); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java index a1b9e5c,0c7a9a9..9432f6c --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java @@@ -193,13 -201,17 +198,17 @@@ public class KerberosDescriptor extend } if (services == null) { - services = new TreeMap(); + services = new TreeMap<>(); } - services.put(name, service); - - // Set the service's parent to this KerberosDescriptor - service.setParent(this); + KerberosServiceDescriptor existing = services.get(name); + if (existing == null) { + services.put(name, service); + // Set the service's parent to this KerberosDescriptor + service.setParent(this); + } else { + existing.update(service); + } } } @@@ -418,4 -425,43 +422,43 @@@ return authToLocalProperties; } + + /** + * Get a map of principals, where the key is the principal path (SERVICE/COMPONENT/principal_name or SERVICE/principal_name) and the value is the principal. + *

+ * For example if the kerberos principal of the HISTORYSERVER is defined in the kerberos.json: + * "name": "history_server_jhs", + * "principal": { + * "value": "jhs/_HOST@${realm}", + * "type" : "service", + * }, + * Then "jhs/_HOST@EXAMPLE.COM" will be put into the map under the "MAPREDUCE2/HISTORYSERVER/history_server_jhs" key. + */ + public Map principals() throws AmbariException { + Map result = new HashMap<>(); + for (AbstractKerberosDescriptorContainer each : nullToEmpty(getChildContainers())) { + if ((each instanceof KerberosServiceDescriptor)) { + collectFromComponents(each.getName(), nullToEmpty(((KerberosServiceDescriptor) each).getComponents()).values(), result); + collectFromIdentities(each.getName(), "", nullToEmpty(each.getIdentities()), result); + } + } + return result; + } + + private static void collectFromComponents(String service, Collection components, Map result) { + for (KerberosComponentDescriptor each : components) { + collectFromIdentities(service, each.getName(), nullToEmpty(each.getIdentities()), result); + } + } + + private static void collectFromIdentities(String service, String component, Collection identities, Map result) { + for (KerberosIdentityDescriptor each : identities) { - if (each.getPrincipalDescriptor() != null && !each.getReferencedServiceName().isPresent()) { ++ if (each.getPrincipalDescriptor() != null && !each.getReferencedServiceName().isPresent() && !each.getName().startsWith("/")) { + String path = StringUtils.isBlank(component) + ? String.format("%s/%s", service, each.getName()) + : String.format("%s/%s/%s", service, component, each.getName()); + result.put(path, each.getPrincipalDescriptor().getName()); + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java index 68cafe5,51b7cd0..5da3399 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java @@@ -157,6 -150,8 +146,8 @@@ public class KerberosServiceDescriptor } } } + - setPreconfigure(getBooleanValue(data, KEY_PRECONFIGURE)); ++ setPreconfigure(getBooleanValue(data, "preconfigure")); } } @@@ -266,9 -279,13 +275,13 @@@ for (KerberosComponentDescriptor component : components.values()) { list.add(component.toMap()); } - map.put(KEY_COMPONENTS, list); + map.put(Type.COMPONENT.getDescriptorPluralName(), list); } + if (preconfigure != null) { - map.put(KEY_PRECONFIGURE, preconfigure.toString()); ++ map.put("preProcess", preconfigure.toString()); + } + return map; } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java index 078b4ae,c2209bb..03b3705 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java @@@ -146,7 -146,8 +146,6 @@@ public class RepositoryXml implements V private String mirrorslist = null; private String repoid = null; private String reponame = null; - private String latest = null; - private String distribution = null; - private String components = null; private boolean unique = false; private Repo() { @@@ -180,10 -181,13 +179,6 @@@ return reponame; } - public String getLatestUri() { - return latest; - public String getDistribution() { - return distribution; -- } -- - public String getComponents() { - return components; - } /** * @return true if version of HDP that change with each release */ @@@ -217,7 -221,8 +212,6 @@@ ri.setOsType(os.trim()); ri.setRepoId(r.getRepoId()); ri.setRepoName(r.getRepoName()); - ri.setLatestBaseUrl(r.getBaseUrl()); - ri.setDistribution(r.getDistribution()); - ri.setComponents(r.getComponents()); ri.setUnique(r.isUnique()); repos.add(ri); http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java index fdb7c8d,f540d8d..9524c09 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java @@@ -218,4 -246,98 +236,97 @@@ public class RepositoryVersionHelper throw new AmbariException("There were no suitable upgrade packs for stack " + stackName + " " + stackVersion + ((null != upgradeType) ? " and upgrade type " + upgradeType : "")); } + + /** + * Build the role parameters for an install command. + * + * @param amc the management controller. Tests don't use the same instance that gets injected. + * @param repoVersion the repository version + * @param osFamily the os family + * @param servicesOnHost the set of services to check for packages + * @return a Map to use in + */ + public Map buildRoleParams(AmbariManagementController amc, RepositoryVersionEntity repoVersion, String osFamily, Set servicesOnHost) + throws SystemException { + + StackId stackId = repoVersion.getStackId(); + + List packages = new ArrayList<>(); + + for (String serviceName : servicesOnHost) { + ServiceInfo info; + + try { + if (ami.get().isServiceRemovedInStack(stackId.getStackName(), stackId.getStackVersion(), serviceName)) { + LOG.info(String.format("%s has been removed from stack %s-%s. Skip calculating its installation packages", stackId.getStackName(), stackId.getStackVersion(), serviceName)); + continue; //No need to calculate install packages for removed services + } + + info = ami.get().getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); + } catch (AmbariException e) { + throw new SystemException(String.format("Cannot obtain stack information for %s-%s", stackId.getStackName(), stackId.getStackVersion()), e); + } + + List packagesForService = amc.getPackagesForServiceHost(info, + new HashMap<>(), osFamily); + + List blacklistedPackagePrefixes = configuration.get().getRollingUpgradeSkipPackagesPrefixes(); + + for (ServiceOsSpecific.Package aPackage : packagesForService) { + if (!aPackage.getSkipUpgrade()) { + boolean blacklisted = false; + for (String prefix : blacklistedPackagePrefixes) { + if (aPackage.getName().startsWith(prefix)) { + blacklisted = true; + break; + } + } + if (! blacklisted) { + packages.add(aPackage); + } + } + } + } + + Map roleParams = new HashMap<>(); + roleParams.put("stack_id", stackId.getStackId()); + // !!! TODO make roleParams so we don't have to do this awfulness. + roleParams.put(KeyNames.PACKAGE_LIST, gson.toJson(packages)); + + return roleParams; + } + + /** + * Adds a command repository to the action context + * @param context the context + * @param osFamily the OS family + * @param repoVersion the repository version entity + * @param repos the repository entities + */ + public void addCommandRepository(ActionExecutionContext context, + RepositoryVersionEntity repoVersion, OperatingSystemEntity osEntity) { + + final CommandRepository commandRepo = new CommandRepository(); + commandRepo.setRepositories(osEntity.getOsType(), osEntity.getRepositories()); + commandRepo.setRepositoryVersion(repoVersion.getVersion()); + commandRepo.setRepositoryVersionId(repoVersion.getId()); - commandRepo.setResolved(repoVersion.isResolved()); + commandRepo.setStackName(repoVersion.getStackId().getStackName()); + + if (!osEntity.isAmbariManagedRepos()) { + commandRepo.setNonManaged(); + } else { + commandRepo.setUniqueSuffix(String.format("-repo-%s", repoVersion.getId())); + } + + context.addVisitor(new ExecutionCommandVisitor() { + @Override + public void visit(ExecutionCommand command) { + if (null == command.getRepositoryFile()) { + command.setRepositoryFile(commandRepo); + } + } + }); + } + + } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java index 446ae8c,230b031..ecb616a --- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java @@@ -25,10 -24,8 +25,11 @@@ import java.util.HashSet import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@@ -93,8 -83,8 +95,9 @@@ import org.slf4j.Logger import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; + import com.google.common.util.concurrent.Striped; import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; import com.google.inject.persist.Transactional; @@@ -880,8 -869,8 +888,9 @@@ public class ServiceComponentHostImpl i } @Override + @Transactional public void setState(State state) { + State oldState = getState(); stateMachine.setCurrentState(state); HostComponentStateEntity stateEntity = getStateEntity(); if (stateEntity != null) { @@@ -913,7 -898,8 +922,8 @@@ } @Override + @Transactional - public void setVersion(String version) { + public void setVersion(String version) throws AmbariException { HostComponentStateEntity stateEntity = getStateEntity(); if (stateEntity != null) { stateEntity.setVersion(version);