Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E414F200D1F for ; Thu, 28 Sep 2017 15:24:59 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id E295A1609ED; Thu, 28 Sep 2017 13:24:59 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 17620160BD7 for ; Thu, 28 Sep 2017 15:24:57 +0200 (CEST) Received: (qmail 19023 invoked by uid 500); 28 Sep 2017 13:24:52 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 18681 invoked by uid 99); 28 Sep 2017 13:24:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Sep 2017 13:24:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5847CF5BBE; Thu, 28 Sep 2017 13:24:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aonishuk@apache.org To: commits@ambari.apache.org Date: Thu, 28 Sep 2017 13:25:35 -0000 Message-Id: <3ef57088266345f7b33639d52a08b766@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [46/50] [abbrv] ambari git commit: Merge remote-tracking branch 'remotes/origin/trunk' into branch-3.0-perf archived-at: Thu, 28 Sep 2017 13:25:00 -0000 http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java index c62b4ce,5373ffe..c36e107 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java @@@ -164,14 -147,9 +158,12 @@@ public class HostResourceProvider exten protected static final String FORCE_DELETE_COMPONENTS = "force_delete_components"; - private static Set pkPropertyIds = - new HashSet<>(Arrays.asList(new String[]{ - HOST_NAME_PROPERTY_ID})); + private static final Set PK_PROPERTY_IDS = ImmutableSet.of(HOST_HOST_NAME_PROPERTY_ID); @Inject + private MaintenanceStateHelper maintenanceStateHelper; + + @Inject private OsFamily osFamily; @Inject @@@ -542,41 -497,80 +525,106 @@@ clusterSet.add(hostRequest.getClusterName()); allClusterSet.add(hostRequest.getClusterName()); hostClustersMap.put(hostRequest.getHostname(), clusterSet); - if (hostRequest.getHostAttributes() != null) { - hostAttributes.put(hostRequest.getHostname(), hostRequest.getHostAttributes()); - } + Cluster cl = clusters.getCluster(hostRequest.getClusterName()); + String clusterId = Long.toString(cl.getClusterId()); + if (!addedTopologies.containsKey(clusterId)) { + addedTopologies.put(clusterId, new TopologyCluster()); + } + Host addedHost = clusters.getHost(hostRequest.getHostname()); + addedTopologies.get(clusterId).addTopologyHost(new TopologyHost(addedHost.getHostId(), + addedHost.getHostName(), + addedHost.getRackInfo(), + addedHost.getIPv4())); ++ //TODO fix repo info param + HostLevelParamsUpdateEvent hostLevelParamsUpdateEvent = new HostLevelParamsUpdateEvent(clusterId, new HostLevelParamsCluster( - ambariMetaInfo.getRepoInfo(cl, addedHost), ++ null,//ambariMetaInfo.getRepoInfo(cl, addedHost), + recoveryConfigHelper.getRecoveryConfig(clusters.getCluster(hostRequest.getClusterName()).getClusterName(), + addedHost.getHostName()) + )); + hostLevelParamsUpdateEvent.setHostName(addedHost.getHostName()); + hostLevelParamsUpdateEvents.add(hostLevelParamsUpdateEvent); } } clusters.updateHostWithClusterAndAttributes(hostClustersMap, hostAttributes); - for (String clusterName : allClusterSet) { - clusters.getCluster(clusterName).recalculateAllClusterVersionStates(); - } ++ // TODO add rack change to topology update + updateHostRackInfoIfChanged(clusters, hostRequests); + + TopologyUpdateEvent topologyUpdateEvent = + new TopologyUpdateEvent(addedTopologies, TopologyUpdateEvent.EventType.UPDATE); + topologyHolder.updateData(topologyUpdateEvent); + for (HostLevelParamsUpdateEvent hostLevelParamsUpdateEvent : hostLevelParamsUpdateEvents) { + hostLevelParamsHolder.updateData(hostLevelParamsUpdateEvent); + } } + /** + * Iterates through the provided host request and checks if there is rack info provided. + * If the rack info differs from the rack info of the host than updates it with the value from + * the host request. + * @param clusters - * @param hostRequests ++ * @param hostRequests + * @throws AmbariException + * @throws AuthorizationException + */ + private void updateHostRackInfoIfChanged(Clusters clusters, Set hostRequests) + throws AmbariException, AuthorizationException { + + HashSet rackChangeAffectedClusters = new HashSet<>(); + + for (HostRequest hostRequest : hostRequests) { + String clusterName = hostRequest.getClusterName(); + + if (StringUtils.isNotBlank(clusterName)) { + Cluster cluster = clusters.getCluster(clusterName); + Host host = clusters.getHost(hostRequest.getHostname()); + + if (updateHostRackInfoIfChanged(cluster, host, hostRequest)) + rackChangeAffectedClusters.add(clusterName); + } + } - ++ // TODO rack change topology update + for (String clusterName: rackChangeAffectedClusters) { + getManagementController().registerRackChange(clusterName); + } + } + + + + /** + * If the rack info provided in the request differs from the rack info of the host + * update the rack info of the host with the value from the host request + * + * @param cluster The cluster to check user privileges against. User is required + * to have {@link RoleAuthorization#HOST_ADD_DELETE_HOSTS} rights on the cluster. + * @param host The host of which rack information is to be updated + * @param hostRequest + * @return true is host was updated otherwise false + * @throws AmbariException + * @throws AuthorizationException + */ + private boolean updateHostRackInfoIfChanged(Cluster cluster, Host host, HostRequest hostRequest) + throws AmbariException, AuthorizationException { + + Long resourceId = cluster.getResourceId(); + + String hostRackInfo = host.getRackInfo(); + String requestRackInfo = hostRequest.getRackInfo(); + + boolean rackChange = requestRackInfo != null && !requestRackInfo.equals(hostRackInfo); + + if (rackChange) { + if(!AuthorizationHelper.isAuthorized(ResourceType.CLUSTER, resourceId, RoleAuthorization.HOST_ADD_DELETE_HOSTS)) { + throw new AuthorizationException("The authenticated user is not authorized to update host rack information"); + } ++ //TODO topology update + + host.setRackInfo(requestRackInfo); + } + + return rackChange; + } + private void createHostResource(Clusters clusters, Set duplicates, Set unknowns, Set allHosts, HostRequest request) @@@ -751,14 -753,10 +807,12 @@@ } } + TreeMap topologyUpdates = new TreeMap<>(); for (HostRequest request : requests) { if (LOG.isDebugEnabled()) { - LOG.debug("Received an updateHost request" - + ", hostname=" + request.getHostname() - + ", request=" + request); + LOG.debug("Received an updateHost request, hostname={}, request={}", request.getHostname(), request); } + TopologyHost topologyHost = new TopologyHost(); Host host = clusters.getHost(request.getHostname()); @@@ -855,21 -835,11 +893,18 @@@ } } - if (clusterName != null && !clusterName.isEmpty()) { - clusters.getCluster(clusterName).recalculateAllClusterVersionStates(); - if (rackChange) { - // Authorization check for this update was performed before we got to this point. - controller.registerRackChange(clusterName); - } + if (StringUtils.isNotBlank(clusterName) && rackChange) { + // Authorization check for this update was performed before we got to this point. + controller.registerRackChange(clusterName); } + if (!topologyUpdates.containsKey(clusterId.toString())) { + topologyUpdates.put(clusterId.toString(), new TopologyCluster()); + } + topologyUpdates.get(clusterId.toString()).addTopologyHost(topologyHost); + TopologyUpdateEvent topologyUpdateEvent = new TopologyUpdateEvent(topologyUpdates, + TopologyUpdateEvent.EventType.UPDATE); + topologyHolder.updateData(topologyUpdateEvent); //todo: if attempt was made to update a property other than those //todo: that are allowed above, should throw exception } @@@ -1000,12 -960,6 +1035,9 @@@ } } clusters.publishHostsDeletion(allClustersWithHosts, hostNames); - for (String clustername : hostsClusters) { - clusters.getCluster(clustername).recalculateAllClusterVersionStates(); - } + TopologyUpdateEvent topologyUpdateEvent = new TopologyUpdateEvent(topologyUpdates, + TopologyUpdateEvent.EventType.DELETE); + topologyHolder.updateData(topologyUpdateEvent); } private void validateHostInDeleteFriendlyState(HostRequest hostRequest, Clusters clusters, boolean forceDelete) throws AmbariException { http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java index d2f3f8d,6455805..44ef9f7 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java @@@ -86,6 -86,8 +86,7 @@@ public class RepositoryVersionResourceP public static final String REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "stack_version"); public static final String REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "repository_version"); public static final String REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "display_name"); + public static final String REPOSITORY_VERSION_HIDDEN_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "hidden"); - public static final String REPOSITORY_VERSION_RESOLVED_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "resolved"); public static final String SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID = new OperatingSystemResourceDefinition().getPluralName(); public static final String SUBRESOURCE_REPOSITORIES_PROPERTY_ID = new RepositoryResourceDefinition().getPluralName(); @@@ -123,16 -122,15 +121,14 @@@ REPOSITORY_VERSION_PARENT_ID, REPOSITORY_VERSION_HAS_CHILDREN, REPOSITORY_VERSION_AVAILABLE_SERVICES, - REPOSITORY_VERSION_STACK_SERVICES, - REPOSITORY_VERSION_RESOLVED_PROPERTY_ID); + REPOSITORY_VERSION_STACK_SERVICES); @SuppressWarnings("serial") - public static Map keyPropertyIds = new HashMap() { - { - put(Type.Stack, REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID); - put(Type.StackVersion, REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID); - put(Type.RepositoryVersion, REPOSITORY_VERSION_ID_PROPERTY_ID); - } - }; + public static Map keyPropertyIds = new ImmutableMap.Builder() + .put(Type.Stack, REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID) + .put(Type.StackVersion, REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID) + .put(Type.RepositoryVersion, REPOSITORY_VERSION_ID_PROPERTY_ID) + .build(); @Inject private Gson gson; @@@ -261,9 -256,10 +254,10 @@@ setResourceProperty(resource, REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID, entity.getStackName(), requestedIds); setResourceProperty(resource, REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID, entity.getStackVersion(), requestedIds); setResourceProperty(resource, REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID, entity.getDisplayName(), requestedIds); + setResourceProperty(resource, REPOSITORY_VERSION_HIDDEN_PROPERTY_ID, entity.isHidden(), requestedIds); setResourceProperty(resource, REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID, entity.getVersion(), requestedIds); setResourceProperty(resource, REPOSITORY_VERSION_TYPE_PROPERTY_ID, entity.getType(), requestedIds); - setResourceProperty(resource, REPOSITORY_VERSION_RESOLVED_PROPERTY_ID, entity.isResolved(), requestedIds); + setResourceProperty(resource, REPOSITORY_VERSION_PARENT_ID, entity.getParentId(), requestedIds); List children = entity.getChildren(); http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java index 81dc340,81f283c..5a5d27c --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java @@@ -737,7 -790,37 +790,36 @@@ public class RequestResourceProvider ex setResourceProperty(resource, REQUEST_SOURCE_SCHEDULE, null, requestedPropertyIds); } - final CalculatedStatus status = CalculatedStatus.statusFromRequest(s_hostRoleCommandDAO, topologyManager, entity.getRequestId()); - + Map summary = s_hostRoleCommandDAO.findAggregateCounts(entity.getRequestId()); + + // get summaries from TopologyManager for logical requests + summary.putAll(topologyManager.getStageSummaries(entity.getRequestId())); + + // summary might be empty due to delete host have cleared all + // HostRoleCommands or due to hosts haven't registered yet with the cluster + // when the cluster is provisioned with a Blueprint + final CalculatedStatus status; + LogicalRequest logicalRequest = topologyManager.getRequest(entity.getRequestId()); + if (summary.isEmpty() && null != logicalRequest) { + // In this case, it appears that there are no tasks but this is a logical + // topology request, so it's a matter of hosts simply not registering yet + // for tasks to be created ==> status = PENDING. + // For a new LogicalRequest there should be at least one HostRequest, + // while if they were removed already ==> status = COMPLETED. + if (logicalRequest.getHostRequests().isEmpty()) { + status = CalculatedStatus.COMPLETED; + } else { + status = CalculatedStatus.PENDING; + } + } else { + // there are either tasks or this is not a logical request, so do normal + // status calculations + status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet()); + } + + if (null != logicalRequest) { + setResourceProperty(resource, REQUEST_PENDING_HOST_REQUEST_COUNT_ID, logicalRequest.getPendingHostRequestCount(), requestedPropertyIds); + } setResourceProperty(resource, REQUEST_STATUS_PROPERTY_ID, status.getStatus().toString(), requestedPropertyIds); setResourceProperty(resource, REQUEST_PROGRESS_PERCENT_ID, status.getPercent(), requestedPropertyIds); http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java index 33cab7c,76a4547..3f8b6e9 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java @@@ -72,7 -75,7 +75,8 @@@ import org.apache.ambari.server.state.S import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.State; +import org.apache.ambari.server.topology.TopologyDeleteFormer; + import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; @@@ -124,9 -166,11 +167,14 @@@ public class ServiceResourceProvider ex @Inject private KerberosHelper kerberosHelper; + @Inject + private TopologyDeleteFormer topologyDeleteFormer; + + /** + * Used to lookup the repository when creating services. + */ + private final RepositoryVersionDAO repositoryVersionDAO; + // ----- Constructors ---------------------------------------------------- /** http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index b657f03,52f66bc..0ff21a2 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@@ -851,22 -731,12 +731,11 @@@ public class UpgradeResourceProvider ex itemEntity.setText(wrapper.getText()); itemEntity.setTasks(wrapper.getTasksJson()); itemEntity.setHosts(wrapper.getHostsJson()); + itemEntities.add(itemEntity); - // At this point, need to change the effective Stack Id so that subsequent tasks run on the newer value. - if (upgradeType == UpgradeType.NON_ROLLING && UpdateStackGrouping.class.equals(group.groupClass)) { - if (direction.isUpgrade()) { - upgradeContext.setEffectiveStackId(upgradeContext.getTargetStackId()); - } else { - upgradeContext.setEffectiveStackId(upgradeContext.getOriginalStackId()); - } - } else if (UpdateStackGrouping.class.equals(group.groupClass)) { - upgradeContext.setEffectiveStackId(upgradeContext.getTargetStackId()); - } - injectVariables(configHelper, cluster, itemEntity); - makeServerSideStage(upgradeContext, req, itemEntity, (ServerSideActionTask) task, - skippable, supportsAutoSkipOnFailure, allowRetry, pack, configUpgradePack); - if (makeServerSideStage(group, upgradeContext, effectiveRepositoryVersion, req, - itemEntity, (ServerSideActionTask) task, configUpgradePack)) { - itemEntities.add(itemEntity); - } ++ makeServerSideStage(group, upgradeContext, effectiveRepositoryVersion, req, ++ itemEntity, (ServerSideActionTask) task, configUpgradePack); } } } else { @@@ -1511,13 -1185,13 +1184,13 @@@ * upgrade * @throws AmbariException */ - private void makeServerSideStage(UpgradeContext context, RequestStageContainer request, - UpgradeItemEntity entity, ServerSideActionTask task, boolean skippable, - boolean supportsAutoSkipOnFailure, boolean allowRetry, - UpgradePack upgradePack, ConfigUpgradePack configUpgradePack) - throws AmbariException { - private boolean makeServerSideStage(UpgradeGroupHolder group, UpgradeContext context, ++ private void makeServerSideStage(UpgradeGroupHolder group, UpgradeContext context, + RepositoryVersionEntity effectiveRepositoryVersion, RequestStageContainer request, + UpgradeItemEntity entity, ServerSideActionTask task, ConfigUpgradePack configUpgradePack) + throws AmbariException { Cluster cluster = context.getCluster(); + UpgradePack upgradePack = context.getUpgradePack(); Map commandParams = getNewParameterMap(request, context); commandParams.put(UpgradeContext.COMMAND_PARAM_UPGRADE_PACK, upgradePack.getName()); @@@ -1593,8 -1276,12 +1266,8 @@@ break; } - if (!process) { - return false; - } - ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(), - Role.AMBARI_SERVER_ACTION.toString(), Collections. emptyList(), + Role.AMBARI_SERVER_ACTION.toString(), Collections.emptyList(), commandParams); actionContext.setTimeout(Short.valueOf((short) -1)); http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/events/AlertReceivedEvent.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/events/ClusterConfigFinishedEvent.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/alerts/AlertAggregateListener.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/alerts/AlertReceivedListener.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/tasks/TaskStatusListener.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/events/listeners/tasks/TaskStatusListener.java index c40969e,230b943..1dd8a5b --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/tasks/TaskStatusListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/tasks/TaskStatusListener.java @@@ -26,8 -26,8 +26,9 @@@ import java.util.Iterator import java.util.List; import java.util.Map; import java.util.Set; + import java.util.concurrent.ConcurrentHashMap; +import org.apache.ambari.server.ClusterNotFoundException; import org.apache.ambari.server.EagerSingleton; import org.apache.ambari.server.Role; import org.apache.ambari.server.actionmanager.HostRoleCommand; @@@ -51,8 -44,6 +52,7 @@@ import org.apache.ambari.server.orm.ent import org.apache.ambari.server.orm.entities.RoleSuccessCriteriaEntity; import org.apache.ambari.server.orm.entities.StageEntity; import org.apache.ambari.server.orm.entities.StageEntityPK; +import org.apache.ambari.server.topology.TopologyManager; - import org.jboss.netty.util.internal.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java index cd82957,1944761..aa29894 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java @@@ -107,55 -103,39 +103,31 @@@ public class DistributeRepositoriesActi "Distribute repositories did not complete, will set all INSTALLING versions for host {} to INSTALL_FAILED.", event.getHostname()); } else { - // Parse structured output + + DistributeRepositoriesStructuredOutput structuredOutput = null; try { + structuredOutput = gson.fromJson(event.getCommandReport().getStructuredOut(), + DistributeRepositoriesStructuredOutput.class); + } catch (JsonSyntaxException e) { + LOG.error("Cannot parse structured output %s", e); + } + + if (null == structuredOutput || null == structuredOutput.repositoryVersionId) { + LOG.error("Received an installation reponse, but it did not contain a repository version id"); + } else { newHostState = RepositoryVersionState.INSTALLED; - DistributeRepositoriesStructuredOutput structuredOutput = StageUtils.getGson().fromJson( - event.getCommandReport().getStructuredOut(), - DistributeRepositoriesStructuredOutput.class); - - repositoryVersion = structuredOutput.getInstalledRepositoryVersion(); - - // Handle the case in which the version to install did not contain the build number, - // but the structured output does contain the build number. - if (null != structuredOutput.getActualVersion() && !structuredOutput.getActualVersion().isEmpty() && - null != structuredOutput.getInstalledRepositoryVersion() && !structuredOutput.getInstalledRepositoryVersion().isEmpty() && - null != structuredOutput.getStackId() && !structuredOutput.getStackId().isEmpty() && - !structuredOutput.getActualVersion().equals(structuredOutput.getInstalledRepositoryVersion())) { - - // !!! getInstalledRepositoryVersion() from the agent is the one - // entered in the UI. getActualVersion() is computed. - - StackId stackId = new StackId(structuredOutput.getStackId()); - RepositoryVersionEntity version = repoVersionDAO.findByStackAndVersion( - stackId, structuredOutput.getInstalledRepositoryVersion()); - - if (null != version) { - LOG.info("Repository version {} was found, but {} is the actual value", - structuredOutput.getInstalledRepositoryVersion(), - structuredOutput.getActualVersion()); - // !!! the entered version is not correct - version.setVersion(structuredOutput.getActualVersion()); - repoVersionDAO.merge(version); - repositoryVersion = structuredOutput.getActualVersion(); - } else { - // !!! extra check that the actual version is correct - stackId = new StackId(structuredOutput.getStackId()); - version = repoVersionDAO.findByStackAndVersion(stackId, - structuredOutput.getActualVersion()); - - LOG.debug("Repository version {} was not found, check for {}. Found={}", - structuredOutput.getInstalledRepositoryVersion(), - structuredOutput.getActualVersion(), - Boolean.valueOf(null != version)); - - if (null != version) { - repositoryVersion = structuredOutput.getActualVersion(); - } + + String actualVersion = structuredOutput.actualVersion; + + RepositoryVersionEntity repoVersion = repoVersionDAO.findByPK(structuredOutput.repositoryVersionId); + + if (null != repoVersion && StringUtils.isNotBlank(actualVersion)) { + if (!StringUtils.equals(repoVersion.getVersion(), actualVersion)) { + repoVersion.setVersion(actualVersion); - repoVersion.setResolved(true); + repoVersionDAO.merge(repoVersion); + repositoryVersion = actualVersion; - } else { - // the reported versions are the same - we should ensure that the - // repo is resolved - if (!repoVersion.isResolved()) { - repoVersion.setResolved(true); - repoVersionDAO.merge(repoVersion); - } } } - } catch (JsonSyntaxException e) { - LOG.error("Cannot parse structured output %s", e); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java index a35957d,e5298dc..394e9f9 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java @@@ -108,8 -98,16 +98,8 @@@ public class StackVersionListener if (null != rve) { String currentRepoVersion = rve.getVersion(); if (!StringUtils.equals(currentRepoVersion, newVersion)) { -- rve.setVersion(newVersion); - rve.setResolved(true); -- repositoryVersionDAO.merge(rve); - } else { - // the reported versions are the same - we should ensure that the repo - // is resolved - if (!rve.isResolved()) { - rve.setResolved(true); ++ rve.setVersion(newVersion); + repositoryVersionDAO.merge(rve); - } } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java index ff874a6,096a395..1ef415a --- a/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java @@@ -36,9 -37,10 +36,9 @@@ import com.google.inject.Injector * RoleCommandOrderProvider which caches RoleCommandOrder objects for a cluster to avoid the cost of construction of * RoleCommandOrder objects each time. */ -@Singleton public class CachedRoleCommandOrderProvider implements RoleCommandOrderProvider { - private static Logger LOG = LoggerFactory.getLogger(CachedRoleCommandOrderProvider.class); + private static final Logger LOG = LoggerFactory.getLogger(CachedRoleCommandOrderProvider.class); @Inject private Injector injector; http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertDefinitionDAO.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertSummaryDTO.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java index 6b34575,010ccec..f634d89 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java @@@ -1007,6 -1027,34 +1008,6 @@@ public class HostRoleCommandDAO taskIds.addAll(daoUtils.selectList(hostRoleCommandQuery)); } - return taskIds; + return Sets.newHashSet(taskIds); } - - /** - * A simple DTO for storing the most recent service check time for a given - * {@link Role}. - */ - public static class LastServiceCheckDTO { - - /** - * The role. - */ - public final String role; - - /** - * The time that the service check ended. - */ - public final long endTime; - - /** - * Constructor. - * - * @param role - * @param endTime - */ - public LastServiceCheckDTO(String role, long endTime) { - this.role = role; - this.endTime = endTime; - } - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/RequestDAO.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigDAO.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigDAO.java index de2d25a,6bfad54..94f5fa5 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigDAO.java @@@ -159,29 -145,9 +159,29 @@@ public class ServiceConfigDAO } /** + * Gets the latest service config versions of default config group for a service + * @param clusterId + * the cluster (not {@code null}). + * @param serviceName + * Name of the service whose latest service config versions needs to be retrieved . + * @return all default group service configurations for the cluster and service. + */ + @RequiresSession + public ServiceConfigEntity getLastServiceConfigForServiceDefaultGroup(Long clusterId, String serviceName) { + TypedQuery query = entityManagerProvider.get().createNamedQuery( + "ServiceConfigEntity.findLatestServiceConfigsByServiceDefaultGroup", + ServiceConfigEntity.class); + + query.setParameter("clusterId", clusterId); + query.setParameter("serviceName", serviceName); + + return daoUtils.selectOne(query); + } + + /** - * Get all service configurations for the specified cluster and stack. This - * will return different versions of the same configuration (HDFS v1 and v2) - * if they exist. + * Get service configurations for the specified cluster and stack. This will + * return different versions of the same configuration (HDFS v1 and v2) if + * they exist. * * @param clusterId * the cluster (not {@code null}). http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/AlertDefinitionEntity.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java index 772b151,2b56b11..bf9e678 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java @@@ -130,6 -144,19 +144,9 @@@ public class RepositoryVersionEntity @Column(name="version_xsd", insertable = true, updatable = true) private String versionXsd; + @Column(name = "hidden", nullable = false, insertable = true, updatable = true) + private short isHidden = 0; + - /** - * Repositories can't be trusted until they have been deployed and we've - * detected their actual version. Most of the time, things match up, but - * editing a VDF could causes the version to be misrepresented. Once we have - * received the correct version of the repository (normally after it's been - * installed), then we can set this flag to {@code true}. - */ - @Column(name = "resolved", nullable = false) - private short resolved = 0; - @ManyToOne @JoinColumn(name = "parent_id") private RepositoryVersionEntity parent; @@@ -433,4 -455,45 +438,25 @@@ return null == parent ? null : parent.getId(); } + /** + * Gets whether this repository is hidden. + * + * @return + */ + public boolean isHidden() { + return isHidden != 0; + } + + /** + * Sets whether this repository is hidden. A repository can be hidden for + * several reasons, including if it has been removed (but needs to be kept + * around for foreign key relationships) or if it just is not longer desired + * to see it. + * + * @param isHidden + */ + public void setHidden(boolean isHidden) { + this.isHidden = (short) (isHidden ? 1 : 0); + } + - /** - * Gets whether this repository has been installed and has reported back its - * actual version. - * - * @return {@code true} if the version for this repository can be trusted, - * {@code false} otherwise. - */ - public boolean isResolved() { - return resolved == 1; - } - - /** - * Sets whether this repository has been installed and has reported back its - * actual version. - * - * @param resolved - */ - public void setResolved(boolean resolved) { - this.resolved = resolved ? (short) 1 : (short) 0; - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigEntity.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigEntity.java index 50dcd1c,b1409ed..6df260d --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigEntity.java @@@ -46,13 -46,24 +46,27 @@@ import javax.persistence.TableGenerator , initialValue = 1 ) @NamedQueries({ - @NamedQuery(name = "ServiceConfigEntity.findAll", query = "SELECT serviceConfig FROM ServiceConfigEntity serviceConfig WHERE serviceConfig.clusterId=:clusterId ORDER BY serviceConfig.version DESC"), - @NamedQuery(name = "ServiceConfigEntity.findNextServiceConfigVersion", query = "SELECT COALESCE(MAX(serviceConfig.version), 0) + 1 AS nextVersion FROM ServiceConfigEntity serviceConfig WHERE serviceConfig.serviceName=:serviceName AND serviceConfig.clusterId=:clusterId"), - @NamedQuery(name = "ServiceConfigEntity.findAllServiceConfigsByStack", query = "SELECT serviceConfig FROM ServiceConfigEntity serviceConfig WHERE serviceConfig.clusterId=:clusterId AND serviceConfig.stack=:stack"), - @NamedQuery(name = "ServiceConfigEntity.findLatestServiceConfigsByStack", query = "SELECT serviceConfig FROM ServiceConfigEntity serviceConfig WHERE serviceConfig.clusterId = :clusterId AND (serviceConfig.groupId = null OR serviceConfig.groupId IN (SELECT cg.groupId from ConfigGroupEntity cg)) AND serviceConfig.version = (SELECT MAX(serviceConfig2.version) FROM ServiceConfigEntity serviceConfig2 WHERE serviceConfig2.clusterId= :clusterId AND serviceConfig2.stack = :stack AND serviceConfig2.serviceName = serviceConfig.serviceName)"), - @NamedQuery(name = "ServiceConfigEntity.findLatestServiceConfigsByService", query = "SELECT scv FROM ServiceConfigEntity scv WHERE scv.clusterId = :clusterId AND scv.serviceName = :serviceName AND (scv.groupId = null OR scv.groupId IN (SELECT cg.groupId from ConfigGroupEntity cg)) AND scv.version = (SELECT MAX(scv2.version) FROM ServiceConfigEntity scv2 WHERE (scv2.serviceName = :serviceName AND scv2.clusterId = :clusterId) AND (scv2.groupId = scv.groupId OR (scv2.groupId IS NULL AND scv.groupId IS NULL)))"), - @NamedQuery(name = "ServiceConfigEntity.findLatestServiceConfigsByServiceDefaultGroup", query = "SELECT scv FROM ServiceConfigEntity scv WHERE scv.clusterId = :clusterId AND scv.serviceName = :serviceName AND scv.groupId = null AND scv.version = (SELECT MAX(scv2.version) FROM ServiceConfigEntity scv2 WHERE (scv2.serviceName = :serviceName AND scv2.clusterId = :clusterId) AND scv2.groupId IS NULL)"), - @NamedQuery(name = "ServiceConfigEntity.findLatestServiceConfigsByCluster", query = "SELECT scv FROM ServiceConfigEntity scv WHERE scv.clusterId = :clusterId AND scv.serviceConfigId IN (SELECT MAX(scv1.serviceConfigId) FROM ServiceConfigEntity scv1 WHERE (scv1.clusterId = :clusterId) AND (scv1.groupId IS NULL) GROUP BY scv1.serviceName)")}) + @NamedQuery( + name = "ServiceConfigEntity.findAll", + query = "SELECT serviceConfig FROM ServiceConfigEntity serviceConfig WHERE serviceConfig.clusterId=:clusterId ORDER BY serviceConfig.version DESC"), + @NamedQuery( + name = "ServiceConfigEntity.findNextServiceConfigVersion", + query = "SELECT COALESCE(MAX(serviceConfig.version), 0) + 1 AS nextVersion FROM ServiceConfigEntity serviceConfig WHERE serviceConfig.serviceName=:serviceName AND serviceConfig.clusterId=:clusterId"), + @NamedQuery( - name = "ServiceConfigEntity.findServiceConfigsByStack", ++ name = "ServiceConfigEntity.findAllServiceConfigsByStack", + query = "SELECT serviceConfig FROM ServiceConfigEntity serviceConfig WHERE serviceConfig.clusterId=:clusterId AND serviceConfig.stack=:stack AND serviceConfig.serviceName=:serviceName"), + @NamedQuery( + name = "ServiceConfigEntity.findLatestServiceConfigsByStack", + query = "SELECT serviceConfig FROM ServiceConfigEntity serviceConfig WHERE serviceConfig.clusterId = :clusterId AND (serviceConfig.groupId = null OR serviceConfig.groupId IN (SELECT cg.groupId from ConfigGroupEntity cg)) AND serviceConfig.version = (SELECT MAX(serviceConfig2.version) FROM ServiceConfigEntity serviceConfig2 WHERE serviceConfig2.clusterId= :clusterId AND serviceConfig2.stack = :stack AND serviceConfig2.serviceName = serviceConfig.serviceName)"), + @NamedQuery( + name = "ServiceConfigEntity.findLatestServiceConfigsByService", + query = "SELECT scv FROM ServiceConfigEntity scv WHERE scv.clusterId = :clusterId AND scv.serviceName = :serviceName AND (scv.groupId = null OR scv.groupId IN (SELECT cg.groupId from ConfigGroupEntity cg)) AND scv.version = (SELECT MAX(scv2.version) FROM ServiceConfigEntity scv2 WHERE (scv2.serviceName = :serviceName AND scv2.clusterId = :clusterId) AND (scv2.groupId = scv.groupId OR (scv2.groupId IS NULL AND scv.groupId IS NULL)))"), + @NamedQuery( ++ name = "ServiceConfigEntity.findLatestServiceConfigsByServiceDefaultGroup", ++ query = "SELECT scv FROM ServiceConfigEntity scv WHERE scv.clusterId = :clusterId AND scv.serviceName = :serviceName AND scv.groupId = null AND scv.version = (SELECT MAX(scv2.version) FROM ServiceConfigEntity scv2 WHERE (scv2.serviceName = :serviceName AND scv2.clusterId = :clusterId) AND scv2.groupId IS NULL)"), ++ @NamedQuery( + name = "ServiceConfigEntity.findLatestServiceConfigsByCluster", + query = "SELECT scv FROM ServiceConfigEntity scv WHERE scv.clusterId = :clusterId AND scv.serviceConfigId IN (SELECT MAX(scv1.serviceConfigId) FROM ServiceConfigEntity scv1 WHERE (scv1.clusterId = :clusterId) AND (scv1.groupId IS NULL) GROUP BY scv1.serviceName)") }) public class ServiceConfigEntity { @Id @Column(name = "service_config_id") http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeItemEntity.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java index cab467f,e804961..95fc973 --- a/ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java @@@ -664,7 -666,7 +663,7 @@@ public class ExecutionScheduleManager } protected BatchRequestResponse performApiGetRequest(String relativeUri, boolean queryAllFields) { - WebResource webResource = ambariWebResource.path(relativeUri); - WebResource webResource = extendApiResource(ambariWebResource, relativeUri); ++ WebResource webResource = ambariWebResource.path(completeRelativeUri(relativeUri)); if (queryAllFields) { webResource = webResource.queryParam("fields", "*"); } @@@ -680,7 -682,8 +679,7 @@@ protected BatchRequestResponse performApiRequest(String relativeUri, String body, String method, Integer userId) { ClientResponse response; try { - response = ambariWebResource.path(relativeUri).header(USER_ID_HEADER, userId).method(method, ClientResponse.class, body); - response = extendApiResource(ambariWebResource, relativeUri) - .header(USER_ID_HEADER, userId).method(method, ClientResponse.class, body); ++ response = ambariWebResource.path(completeRelativeUri(relativeUri)).header(USER_ID_HEADER, userId).method(method, ClientResponse.class, body); } catch (UniformInterfaceException e) { response = e.getResponse(); } @@@ -796,5 -799,19 +795,16 @@@ requestExecution.updateStatus(RequestExecution.Status.COMPLETED); } } + - /** - * Returns the absolute web resource with {@link #DEFAULT_API_PATH} - * @param webResource Ambari WebResource as provided by the client {@link #ambariWebResource} - * @param relativeUri relative request URI - * @return Extended WebResource - */ - protected WebResource extendApiResource(WebResource webResource, String relativeUri) { - WebResource result = webResource; - if (StringUtils.isNotEmpty(relativeUri) && !CONTAINS_API_VERSION_PATTERN.matcher(relativeUri).matches()) { - result = webResource.path(DEFAULT_API_PATH); ++ private String completeRelativeUri(String relativeUri){ ++ if (StringUtils.isNotEmpty(relativeUri) ++ && !(relativeUri.startsWith("api/v1") || relativeUri.startsWith("/api/v1"))){ ++ if (relativeUri.charAt(0) != '/') { ++ relativeUri = '/' + relativeUri; ++ } ++ return "api/v1" + relativeUri; + } - return result.path(relativeUri); ++ return relativeUri; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/security/authentication/AmbariDelegatingAuthenticationFilter.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/AmbariUserAuthentication.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/AbstractServerAction.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java index 7aac346,3db844a..d6b8ffc --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java @@@ -75,7 -76,23 +76,21 @@@ public abstract class AbstractPrepareKe Map> currentConfigurations, Map> kerberosConfigurations, boolean includeAmbariIdentity, - Map> propertiesToBeIgnored, - boolean excludeHeadless) throws AmbariException { + Map> propertiesToBeIgnored) throws AmbariException { + List components = new ArrayList<>(); + for (ServiceComponentHost each : schToProcess) { + components.add(Component.fromServiceComponentHost(each)); + } - processServiceComponents(cluster, kerberosDescriptor, components, identityFilter, dataDirectory, currentConfigurations, kerberosConfigurations, includeAmbariIdentity, propertiesToBeIgnored, excludeHeadless); ++ processServiceComponents(cluster, kerberosDescriptor, components, identityFilter, dataDirectory, currentConfigurations, kerberosConfigurations, includeAmbariIdentity, propertiesToBeIgnored); + } + + protected void processServiceComponents(Cluster cluster, KerberosDescriptor kerberosDescriptor, + List schToProcess, + Collection identityFilter, String dataDirectory, + Map> currentConfigurations, + Map> kerberosConfigurations, + boolean includeAmbariIdentity, - Map> propertiesToBeIgnored, - boolean excludeHeadless) throws AmbariException { ++ Map> propertiesToBeIgnored) throws AmbariException { actionLog.writeStdOut("Processing Kerberos identities and configurations"); http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java index 5b65833,f56e946..4e63f4a --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java @@@ -111,29 -105,25 +105,25 @@@ public class PrepareDisableKerberosServ // Calculate the current host-specific configurations. These will be used to replace // variables within the Kerberos descriptor data - Map> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptorProperties); + Map> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false); processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, - configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore, false); + configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore); // Add auth-to-local configurations to the set of changes - Set authToLocalProperties = kerberosDescriptor.getAllAuthToLocalProperties(); + Map> authToLocalProperties = kerberosHelper.translateConfigurationSpecifications(kerberosDescriptor.getAllAuthToLocalProperties()); if (authToLocalProperties != null) { - for (String authToLocalProperty : authToLocalProperties) { - Matcher m = KerberosDescriptor.AUTH_TO_LOCAL_PROPERTY_SPECIFICATION_PATTERN.matcher(authToLocalProperty); - - if (m.matches()) { - String configType = m.group(1); - String propertyName = m.group(2); - - if (configType == null) { - configType = ""; - } - - // Add existing auth_to_local configuration, if set - Map configuration = kerberosConfigurations.get(configType); - if (configuration != null) { - configuration.put(propertyName, "DEFAULT"); + for (Map.Entry> entry : authToLocalProperties.entrySet()) { + String configType = entry.getKey(); + Set propertyNames = entry.getValue(); + + if (!CollectionUtils.isEmpty(propertyNames)) { + for (String propertyName : propertyNames) { + // Add existing auth_to_local configuration, if set + Map configuration = kerberosConfigurations.get(configType); + if (configuration != null) { + configuration.put(propertyName, "DEFAULT"); + } } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java index ca15695,3ec84fa..e13f033 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java @@@ -87,11 -114,15 +114,15 @@@ public class PrepareEnableKerberosServe // Calculate the current host-specific configurations. These will be used to replace // variables within the Kerberos descriptor data - Map> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptorProperties); + Map> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false); processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, - configurations, kerberosConfigurations, true, propertiesToIgnore, false); + configurations, kerberosConfigurations, true, propertiesToIgnore); + // Calculate the set of configurations to update and replace any variables + // using the previously calculated Map of configurations for the host. + kerberosConfigurations = kerberosHelper.processPreconfiguredServiceConfigurations(kerberosConfigurations, configurations, cluster, kerberosDescriptor); + kerberosHelper.applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations, propertiesToIgnore, propertiesToRemove, true); http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java index f239cff,49828cb..00c82a5 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java @@@ -92,16 -92,17 +91,17 @@@ public class PrepareKerberosIdentitiesS // Calculate the current host-specific configurations. These will be used to replace // variables within the Kerberos descriptor data - Map> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptorProperties); + Map> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false); processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, - configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore, !CollectionUtils.isEmpty(getHostFilter())); + configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore); kerberosHelper.applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations, - propertiesToIgnore, propertiesToRemove, true); + propertiesToIgnore, propertiesToRemove, true); if ("true".equalsIgnoreCase(getCommandParameterValue(commandParameters, UPDATE_CONFIGURATIONS))) { - processAuthToLocalRules(cluster, kerberosDescriptor, schToProcess, kerberosConfigurations, getDefaultRealm(commandParameters)); + Map> calculatedConfigurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false); + processAuthToLocalRules(cluster, calculatedConfigurations, kerberosDescriptor, schToProcess, kerberosConfigurations, getDefaultRealm(commandParameters), false); processConfigurationChanges(dataDirectory, kerberosConfigurations, propertiesToRemove); } http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java index 086055d,451f802..5ec0692 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java @@@ -300,12 -198,22 +198,17 @@@ public class FinalizeUpgradeAction exte } } - // Impacts all hosts that have a version - outSB.append( - String.format("Finalizing the version for %d host(s).\n", hostVersionsAllowed.size())); - cluster.mapHostVersions(hostsToUpdate, upgradingClusterVersion, RepositoryVersionState.CURRENT); + // move host versions from CURRENT to INSTALLED if their repos are no + // longer used + finalizeHostRepositoryVersions(cluster); + - if (upgradeContext.getOrchestrationType() == RepositoryType.STANDARD) { - outSB.append(String.format("Finalizing the version for cluster %s.\n", cluster.getClusterName())); - cluster.setCurrentStackVersion(cluster.getDesiredStackVersion()); - } - + // mark revertable + if (repositoryType.isRevertable() && direction == Direction.UPGRADE) { + UpgradeEntity upgrade = cluster.getUpgradeInProgress(); + upgrade.setRevertAllowed(true); + upgrade = m_upgradeDAO.merge(upgrade); + } - versionEventPublisher.publish(new StackUpgradeFinishEvent(cluster)); // Reset upgrade state cluster.setUpgradeEntity(null);