Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 2390A200BBD for ; Tue, 8 Nov 2016 21:36:50 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 2215D160B0A; Tue, 8 Nov 2016 20:36:50 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 94EBD160AD0 for ; Tue, 8 Nov 2016 21:36:48 +0100 (CET) Received: (qmail 2309 invoked by uid 500); 8 Nov 2016 20:36:47 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 2295 invoked by uid 99); 8 Nov 2016 20:36:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Nov 2016 20:36:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8EAD5E186F; Tue, 8 Nov 2016 20:36:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ncole@apache.org To: commits@ambari.apache.org Date: Tue, 08 Nov 2016 20:36:47 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/5] ambari git commit: AMBARI-18634 - Add a cluster version directly as INSTALLED (jonathanhurley) archived-at: Tue, 08 Nov 2016 20:36:50 -0000 Repository: ambari Updated Branches: refs/heads/trunk fa7704dfa -> d838763dc AMBARI-18634 - Add a cluster version directly as INSTALLED (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a396b7c0 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a396b7c0 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a396b7c0 Branch: refs/heads/trunk Commit: a396b7c0c8f4fe27d757e5785010bf3ba21a4ced Parents: 343b4d8 Author: Jonathan Hurley Authored: Fri Nov 4 13:03:32 2016 -0400 Committer: Jonathan Hurley Committed: Fri Nov 4 14:45:15 2016 -0400 ---------------------------------------------------------------------- .../ClusterStackVersionResourceProvider.java | 221 +++++++++++-------- .../org/apache/ambari/server/state/Cluster.java | 6 +- .../org/apache/ambari/server/state/Host.java | 14 ++ .../server/state/cluster/ClusterImpl.java | 17 +- .../ambari/server/state/host/HostImpl.java | 27 +++ ...ClusterStackVersionResourceProviderTest.java | 184 +++++++++++++++ .../server/state/cluster/ClusterTest.java | 33 ++- 7 files changed, 389 insertions(+), 113 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a396b7c0/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index 4e9fd6b..42aae84 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -48,6 +48,7 @@ import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.ActionExecutionContext; import org.apache.ambari.server.controller.AmbariActionExecutionHelper; import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; import org.apache.ambari.server.controller.spi.NoSuchResourceException; import org.apache.ambari.server.controller.spi.Predicate; @@ -88,6 +89,8 @@ import org.apache.ambari.server.utils.VersionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Injector; @@ -110,6 +113,16 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou protected static final String CLUSTER_STACK_VERSION_HOST_STATES_PROPERTY_ID = PropertyHelper.getPropertyId("ClusterStackVersions", "host_states"); protected static final String CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID = PropertyHelper.getPropertyId("ClusterStackVersions", "repository_version"); protected static final String CLUSTER_STACK_VERSION_STAGE_SUCCESS_FACTOR = PropertyHelper.getPropertyId("ClusterStackVersions", "success_factor"); + + /** + * Forces the {@link HostVersionEntity}s to a specific + * {@link RepositoryVersionState}. When used during the creation of + * {@link HostVersionEntity}s, this will set the state to + * {@link RepositoryVersionState#INSTALLED}. When used during the update of a + * cluster stack version, this will force all entities to + * {@link RepositoryVersionState#CURRENT}. + * + */ protected static final String CLUSTER_STACK_VERSION_FORCE = "ClusterStackVersions/force"; protected static final String INSTALL_PACKAGES_ACTION = "install_packages"; @@ -127,43 +140,25 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou */ private static final float INSTALL_PACKAGES_SUCCESS_FACTOR = 0.85f; - @SuppressWarnings("serial") - private static Set pkPropertyIds = new HashSet() { - { - add(CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_ID_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_STACK_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_STATE_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID); - } - }; - - @SuppressWarnings("serial") - private static Set propertyIds = new HashSet() { - { - add(CLUSTER_STACK_VERSION_ID_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_STACK_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_HOST_STATES_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_STATE_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID); - add(CLUSTER_STACK_VERSION_STAGE_SUCCESS_FACTOR); - add(CLUSTER_STACK_VERSION_FORCE); - } - }; - - @SuppressWarnings("serial") - private static Map keyPropertyIds = new HashMap() { - { - put(Type.Cluster, CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID); - put(Type.ClusterStackVersion, CLUSTER_STACK_VERSION_ID_PROPERTY_ID); - put(Type.Stack, CLUSTER_STACK_VERSION_STACK_PROPERTY_ID); - put(Type.StackVersion, CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID); - put(Type.RepositoryVersion, CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID); - } - }; + private static Set pkPropertyIds = Sets.newHashSet( + CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID, CLUSTER_STACK_VERSION_ID_PROPERTY_ID, + CLUSTER_STACK_VERSION_STACK_PROPERTY_ID, CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID, + CLUSTER_STACK_VERSION_STATE_PROPERTY_ID, + CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID); + + private static Set propertyIds = Sets.newHashSet(CLUSTER_STACK_VERSION_ID_PROPERTY_ID, + CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID, CLUSTER_STACK_VERSION_STACK_PROPERTY_ID, + CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID, CLUSTER_STACK_VERSION_HOST_STATES_PROPERTY_ID, + CLUSTER_STACK_VERSION_STATE_PROPERTY_ID, CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID, + CLUSTER_STACK_VERSION_STAGE_SUCCESS_FACTOR, CLUSTER_STACK_VERSION_FORCE); + + private static Map keyPropertyIds = ImmutableMap. builder() + .put(Type.Cluster, CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID) + .put(Type.ClusterStackVersion, CLUSTER_STACK_VERSION_ID_PROPERTY_ID) + .put(Type.Stack, CLUSTER_STACK_VERSION_STACK_PROPERTY_ID) + .put(Type.StackVersion, CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID) + .put(Type.RepositoryVersion, CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID) + .build(); @Inject private static ClusterVersionDAO clusterVersionDAO; @@ -177,7 +172,8 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou @Inject private static HostRoleCommandFactory hostRoleCommandFactory; - private static Gson gson = StageUtils.getGson(); + @Inject + private static Gson gson; @Inject private static Provider actionExecutionHelper; @@ -338,9 +334,6 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou cluster.getClusterName(), entity.getDirection().getText(false))); } - // get all of the host eligible for stack distribution - List hosts = getHostsForStackDistribution(cluster); - final StackId stackId; if (propertyMap.containsKey(CLUSTER_STACK_VERSION_STACK_PROPERTY_ID) && propertyMap.containsKey(CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID)) { @@ -356,14 +349,6 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou stackId = currentStackVersion; } - - // why does the JSON body parser convert JSON primitives into strings!? - Float successFactor = INSTALL_PACKAGES_SUCCESS_FACTOR; - String successFactorProperty = (String) propertyMap.get(CLUSTER_STACK_VERSION_STAGE_SUCCESS_FACTOR); - if (StringUtils.isNotBlank(successFactorProperty)) { - successFactor = Float.valueOf(successFactorProperty); - } - RepositoryVersionEntity repoVersionEnt = repositoryVersionDAO.findByStackAndVersion( stackId, desiredRepoVersion); @@ -381,6 +366,9 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou String.format("Version %s is backed by a version definition, but it could not be parsed", desiredRepoVersion), e); } + // get all of the host eligible for stack distribution + List hosts = getHostsForStackDistribution(cluster); + /* If there is a repository that is already ATTEMPTED to be installed and the version is GREATER than the one trying to install, we must fail (until we can support that via Patch Upgrades) @@ -427,6 +415,80 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou } } + // if true, then we need to force all new host versions into the INSTALLED state + boolean forceInstalled = Boolean.parseBoolean((String)propertyMap.get( + CLUSTER_STACK_VERSION_FORCE)); + + final RequestStatusResponse response; + + try { + if (forceInstalled) { + createHostVersions(cluster, hosts, stackId, desiredRepoVersion, RepositoryVersionState.INSTALLED); + response = null; + } else { + createHostVersions(cluster, hosts, stackId, desiredRepoVersion, + RepositoryVersionState.INSTALLING); + + RequestStageContainer installRequest = createOrchestration(cluster, stackId, hosts, + repoVersionEnt, propertyMap); + + response = installRequest.getRequestStatusResponse(); + } + } catch (AmbariException e) { + throw new SystemException("Can not persist request", e); + } + + return getRequestStatus(response); + } + + @Transactional + private void createHostVersions(Cluster cluster, List hosts, StackId stackId, + String desiredRepoVersion, RepositoryVersionState repoState) + throws AmbariException, SystemException { + final String clusterName = cluster.getClusterName(); + final String authName = getManagementController().getAuthName(); + + ClusterVersionEntity clusterVersionEntity = clusterVersionDAO.findByClusterAndStackAndVersion( + clusterName, stackId, desiredRepoVersion); + + if (clusterVersionEntity == null) { + try { + // Create/persist new cluster stack version + cluster.createClusterVersion(stackId, desiredRepoVersion, authName, repoState); + + clusterVersionEntity = clusterVersionDAO.findByClusterAndStackAndVersion(clusterName, + stackId, desiredRepoVersion); + } catch (AmbariException e) { + throw new SystemException( + String.format("Can not create cluster stack version %s for cluster %s", + desiredRepoVersion, clusterName), e); + } + } else { + // Move cluster version into the specified state (retry installation) + cluster.transitionClusterVersion(stackId, desiredRepoVersion, repoState); + } + + // Will also initialize all Host Versions to the specified state state. + cluster.transitionHosts(clusterVersionEntity, repoState); + + // Directly transition host versions to NOT_REQUIRED for hosts that don't + // have versionable components + for (Host host : hosts) { + if (!host.hasComponentsAdvertisingVersions(stackId)) { + transitionHostVersionToNotRequired(host, cluster, + clusterVersionEntity.getRepositoryVersion()); + } + } + } + + @Transactional + private RequestStageContainer createOrchestration(Cluster cluster, StackId stackId, + List hosts, RepositoryVersionEntity repoVersionEnt, Map propertyMap) + throws AmbariException, SystemException { + final AmbariManagementController managementController = getManagementController(); + final AmbariMetaInfo ami = managementController.getAmbariMetaInfo(); + + // build the list of OS repos List operatingSystems = repoVersionEnt.getOperatingSystems(); Map> perOsRepos = new HashMap>(); for (OperatingSystemEntity operatingSystem : operatingSystems) { @@ -434,9 +496,8 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou if (operatingSystem.isAmbariManagedRepos()) { perOsRepos.put(operatingSystem.getOsType(), operatingSystem.getRepositories()); } else { - perOsRepos.put(operatingSystem.getOsType(), Collections.emptyList()); + perOsRepos.put(operatingSystem.getOsType(), Collections. emptyList()); } - } RequestStageContainer req = createRequest(); @@ -459,13 +520,18 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou int hostCount = hosts.size(); int batchCount = (int) (Math.ceil((double)hostCount / maxTasks)); - ArrayList directTransitions = new ArrayList(); - long stageId = req.getLastStageId() + 1; if (0L == stageId) { stageId = 1L; } + // why does the JSON body parser convert JSON primitives into strings!? + Float successFactor = INSTALL_PACKAGES_SUCCESS_FACTOR; + String successFactorProperty = (String) propertyMap.get(CLUSTER_STACK_VERSION_STAGE_SUCCESS_FACTOR); + if (StringUtils.isNotBlank(successFactorProperty)) { + successFactor = Float.valueOf(successFactorProperty); + } + boolean hasStage = false; ArrayList stages = new ArrayList(batchCount); @@ -515,59 +581,20 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou throw new SystemException("Cannot modify stage", e); } } - } else { - directTransitions.add(host); } } } if (!hasStage) { - throw new SystemException(String.format("There are no hosts that have components to install repository %s", - desiredRepoVersion)); + throw new SystemException( + String.format("There are no hosts that have components to install for repository %s", + repoVersionEnt.getDisplayName())); } req.addStages(stages); + req.persist(); - try { - ClusterVersionEntity clusterVersionEntity = clusterVersionDAO.findByClusterAndStackAndVersion( - clName, stackId, desiredRepoVersion); - - if (clusterVersionEntity == null) { - try { - // Create/persist new cluster stack version - cluster.createClusterVersion(stackId, - desiredRepoVersion, managementController.getAuthName(), - RepositoryVersionState.INSTALLING); - - clusterVersionEntity = clusterVersionDAO.findByClusterAndStackAndVersion( - clName, stackId, desiredRepoVersion); - } catch (AmbariException e) { - throw new SystemException( - String.format( - "Can not create cluster stack version %s for cluster %s", - desiredRepoVersion, clName), e); - } - } else { - // Move cluster version into INSTALLING state (retry installation) - cluster.transitionClusterVersion(stackId, - desiredRepoVersion, RepositoryVersionState.INSTALLING); - } - - // Will also initialize all Host Versions in an INSTALLING state. - cluster.transitionHostsToInstalling(clusterVersionEntity); - - // Directly transition host versions to NOT_REQUIRED for hosts that don't have - // versionable components - for(Host host : directTransitions) { - transitionHostVersionToNotRequired(host, cluster, - clusterVersionEntity.getRepositoryVersion()); - } - - req.persist(); - } catch (AmbariException e) { - throw new SystemException("Can not persist request", e); - } - return getRequestStatus(req.getRequestStatusResponse()); + return req; } private ActionExecutionContext getHostVersionInstallCommand(RepositoryVersionEntity repoVersion, http://git-wip-us.apache.org/repos/asf/ambari/blob/a396b7c0/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java index b1958ef..c6ae050 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java @@ -228,7 +228,7 @@ public interface Cluster { /** * Creates or updates host versions for all of the hosts within a cluster * based on state of cluster stack version. This is used to transition all - * hosts into the {@link RepositoryVersionState#INSTALLING} state. + * hosts into the specified state. *

* The difference between this method compared to * {@link Cluster#mapHostVersions} is that it affects all hosts (not only @@ -241,9 +241,11 @@ public interface Cluster { * cluster version to be queried for a stack name/version info and * desired RepositoryVersionState. The only valid state of a cluster * version is {@link RepositoryVersionState#INSTALLING} + * @param state + * the state to transition the cluster's hosts to. * @throws AmbariException */ - void transitionHostsToInstalling(ClusterVersionEntity sourceClusterVersion) + void transitionHosts(ClusterVersionEntity sourceClusterVersion, RepositoryVersionState state) throws AmbariException; /** http://git-wip-us.apache.org/repos/asf/ambari/blob/a396b7c0/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java index bd6cc0d..04b2104 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java @@ -388,4 +388,18 @@ public interface Host extends Comparable { * @return */ List getAllHostVersions(); + + /** + * Gets whether this host has components which advertise their version. + * + * @param stackId + * the version of the stack to use when checking version + * advertise-ability. + * @return {@code true} if at least 1 component on this host advertises its + * version. + * @throws AmbariException + * if there is a problem retrieving the component from the stack. + * @see ComponentInfo#isVersionAdvertised() + */ + boolean hasComponentsAdvertisingVersions(StackId stackId) throws AmbariException; } http://git-wip-us.apache.org/repos/asf/ambari/blob/a396b7c0/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index bb46c5c..8b157c7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -167,7 +167,8 @@ public class ClusterImpl implements Cluster { */ private static final String CLUSTER_SESSION_ATTRIBUTES_PREFIX = "cluster_session_attributes:"; private static final Set ALLOWED_REPOSITORY_STATES = - EnumSet.of(RepositoryVersionState.INIT, RepositoryVersionState.INSTALLING); + EnumSet.of(RepositoryVersionState.INIT, RepositoryVersionState.INSTALLING, + RepositoryVersionState.INSTALLED); @Inject private Clusters clusters; @@ -1179,15 +1180,16 @@ public class ClusterImpl implements Cluster { * {@inheritDoc} */ @Override - public void transitionHostsToInstalling(ClusterVersionEntity sourceClusterVersion) throws AmbariException { + public void transitionHosts(ClusterVersionEntity sourceClusterVersion, + RepositoryVersionState state) throws AmbariException { + if (sourceClusterVersion == null) { throw new AmbariException("Could not find current stack version of cluster " + getClusterName()); } - if (RepositoryVersionState.INSTALLING != sourceClusterVersion.getState()) { - throw new AmbariException("Unable to transition cluster hosts into " - + RepositoryVersionState.INSTALLING - + ". The only valid state is INSTALLING"); + if (state != sourceClusterVersion.getState()) { + throw new AmbariException("Unable to transition cluster hosts into " + state + + ". The only valid state is " + sourceClusterVersion.getState()); } Map hosts = clusters.getHostsForCluster(getClusterName()); @@ -1215,8 +1217,7 @@ public class ClusterImpl implements Cluster { hosts.keySet(), existingHostsWithClusterStackAndVersion); createOrUpdateHostVersionToState(sourceClusterVersion, hosts, - existingHostStackVersions, hostsMissingRepoVersion, - RepositoryVersionState.INSTALLING); + existingHostStackVersions, hostsMissingRepoVersion, state); } finally { clusterGlobalLock.writeLock().unlock(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/a396b7c0/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java index 513d924..e66759e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java @@ -34,6 +34,7 @@ import org.apache.ambari.server.agent.AgentEnv; import org.apache.ambari.server.agent.DiskInfo; import org.apache.ambari.server.agent.HostInfo; import org.apache.ambari.server.agent.RecoveryReport; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.HostResponse; import org.apache.ambari.server.events.MaintenanceModeEvent; import org.apache.ambari.server.events.publishers.AmbariEventPublisher; @@ -45,12 +46,14 @@ import org.apache.ambari.server.orm.dao.HostDAO; import org.apache.ambari.server.orm.dao.HostStateDAO; import org.apache.ambari.server.orm.dao.HostVersionDAO; import org.apache.ambari.server.orm.entities.ClusterEntity; +import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.HostStateEntity; import org.apache.ambari.server.orm.entities.HostVersionEntity; import org.apache.ambari.server.state.AgentVersion; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.Host; @@ -61,6 +64,7 @@ import org.apache.ambari.server.state.HostHealthStatus; import org.apache.ambari.server.state.HostHealthStatus.HealthStatus; import org.apache.ambari.server.state.HostState; import org.apache.ambari.server.state.MaintenanceState; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; import org.apache.ambari.server.state.fsm.SingleArcTransition; @@ -128,6 +132,9 @@ public class HostImpl implements Host { @Inject private HostConfigMappingDAO hostConfigMappingDAO; + @Inject + private AmbariMetaInfo ambariMetaInfo; + /** * The ID of the host which is to retrieve it from JPA. */ @@ -1146,6 +1153,26 @@ public class HostImpl implements Host { public HostStateEntity getHostStateEntity() { return hostStateDAO.findByHostId(hostId); } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasComponentsAdvertisingVersions(StackId stackId) throws AmbariException { + HostEntity hostEntity = getHostEntity(); + + for (HostComponentStateEntity componentState : hostEntity.getHostComponentStateEntities()) { + ComponentInfo component = ambariMetaInfo.getComponent(stackId.getStackName(), + stackId.getStackVersion(), componentState.getServiceName(), + componentState.getComponentName()); + + if (component.isVersionAdvertised()) { + return true; + } + } + + return false; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/a396b7c0/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java index 9837b0f..c157258 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java @@ -1611,6 +1611,190 @@ public class ClusterStackVersionResourceProviderTest { testCreateResourcesExistingUpgrade(TestAuthenticationFactory.createClusterOperator()); } + /** + * Tests that forcing the host versions into + * {@link RepositoryVersionState#INSTALLED} + * + * @throws Exception + */ + @Test + public void testCreateResourcesInInstalledState() throws Exception { + Resource.Type type = Resource.Type.ClusterStackVersion; + + AmbariManagementController managementController = createMock(AmbariManagementController.class); + Clusters clusters = createNiceMock(Clusters.class); + Cluster cluster = createNiceMock(Cluster.class); + StackId stackId = new StackId("HDP", "2.2.0"); + String repoVersion = "2.2.0.1-885"; + + File f = new File("src/test/resources/hbase_version_test.xml"); + + RepositoryVersionEntity repoVersionEntity = new RepositoryVersionEntity(); + repoVersionEntity.setId(1l); + repoVersionEntity.setOperatingSystems(OS_JSON); + repoVersionEntity.setVersionXml(IOUtils.toString(new FileInputStream(f))); + repoVersionEntity.setVersionXsd("version_definition.xsd"); + repoVersionEntity.setType(RepositoryType.STANDARD); + + Map hostsForCluster = new HashMap(); + List hostVersionEntitiesMergedWithNotRequired = new ArrayList<>(); + int hostCount = 10; + + for (int i = 0; i < hostCount; i++) { + String hostname = "host" + i; + Host host = createNiceMock(hostname, Host.class); + expect(host.getHostName()).andReturn(hostname).anyTimes(); + expect(host.getOsFamily()).andReturn("redhat6").anyTimes(); + expect(host.getMaintenanceState(EasyMock.anyLong())).andReturn(MaintenanceState.OFF).anyTimes(); + + // ensure that 2 hosts don't have versionable components so they + // transition correct into the not required state + if (i < hostCount - 2) { + expect(host.hasComponentsAdvertisingVersions(eq(stackId))).andReturn(true).atLeastOnce(); + } else { + expect(host.hasComponentsAdvertisingVersions(eq(stackId))).andReturn(false).atLeastOnce(); + + // mock out the host versions so that we can test hosts being + // transitioned into NOT_REQUIRED + HostVersionEntity hostVersionEntity = EasyMock.createNiceMock(HostVersionEntity.class); + expect(hostVersionEntity.getRepositoryVersion()).andReturn(repoVersionEntity).atLeastOnce(); + replay(hostVersionEntity); + + hostVersionEntitiesMergedWithNotRequired.add(hostVersionEntity); + expect(host.getAllHostVersions()).andReturn(hostVersionEntitiesMergedWithNotRequired).anyTimes(); + } + + replay(host); + + hostsForCluster.put(hostname, host); + } + + Service hdfsService = createNiceMock(Service.class); + expect(hdfsService.getName()).andReturn("HDFS").anyTimes(); + expect(hdfsService.getServiceComponents()).andReturn(new HashMap()); + + Map serviceMap = new HashMap<>(); + serviceMap.put("HDFS", hdfsService); + + final ServiceComponentHost schDatanode = createMock(ServiceComponentHost.class); + expect(schDatanode.getServiceName()).andReturn("HDFS").anyTimes(); + expect(schDatanode.getServiceComponentName()).andReturn("DATANODE").anyTimes(); + + final List serviceComponentHosts = Arrays.asList(schDatanode); + + ServiceOsSpecific.Package hdfsPackage = new ServiceOsSpecific.Package(); + hdfsPackage.setName("hdfs"); + + List packages = Collections.singletonList(hdfsPackage); + + RequestStatusResponse response = createNiceMock(RequestStatusResponse.class); + ResourceProviderFactory resourceProviderFactory = createNiceMock(ResourceProviderFactory.class); + ResourceProvider csvResourceProvider = createNiceMock( + ClusterStackVersionResourceProvider.class); + + AbstractControllerResourceProvider.init(resourceProviderFactory); + + expect(managementController.getClusters()).andReturn(clusters).anyTimes(); + expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(managementController.getAuthName()).andReturn("admin").anyTimes(); + expect(managementController.getJdkResourceUrl()).andReturn("/JdkResourceUrl").anyTimes(); + expect(managementController.getPackagesForServiceHost(anyObject(ServiceInfo.class), + (Map) anyObject(List.class), anyObject(String.class))).andReturn( + packages).anyTimes(); // only one host has the versionable component + + expect(resourceProviderFactory.getHostResourceProvider(anyObject(Set.class), + anyObject(Map.class), eq(managementController))).andReturn(csvResourceProvider).anyTimes(); + + expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster); + expect(clusters.getHostsForCluster(anyObject(String.class))).andReturn( + hostsForCluster).anyTimes(); + + String clusterName = "Cluster100"; + expect(cluster.getClusterId()).andReturn(1L).anyTimes(); + expect(cluster.getClusterName()).andReturn(clusterName).atLeastOnce(); + expect(cluster.getHosts()).andReturn(hostsForCluster.values()).atLeastOnce(); + expect(cluster.getServices()).andReturn(serviceMap).anyTimes(); + expect(cluster.getServiceComponentHosts(anyObject(String.class))).andReturn( + serviceComponentHosts).anyTimes(); + + expect(repositoryVersionDAOMock.findByStackAndVersion(anyObject(StackId.class), + anyObject(String.class))).andReturn(repoVersionEntity); + + expect(clusterVersionDAO.findByCluster(anyObject(String.class))).andReturn( + Collections. emptyList()).once(); + + ClusterEntity clusterEntity = new ClusterEntity(); + clusterEntity.setClusterId(1l); + clusterEntity.setClusterName(clusterName); + + ClusterVersionEntity cve = new ClusterVersionEntity(clusterEntity, repoVersionEntity, + RepositoryVersionState.INSTALL_FAILED, 0, ""); + + // first expect back a null to make the code think it needs to create one, + // then return the real one it's going to use + expect(clusterVersionDAO.findByClusterAndStackAndVersion(anyObject(String.class), + anyObject(StackId.class), anyObject(String.class))).andReturn(null).once(); + expect(clusterVersionDAO.findByClusterAndStackAndVersion(anyObject(String.class), + anyObject(StackId.class), anyObject(String.class))).andReturn(cve).once(); + + // now the important expectations - that the cluster transition methods were + // called correctly + cluster.transitionHosts(cve, RepositoryVersionState.INSTALLED); + for (HostVersionEntity hostVersionEntity : hostVersionEntitiesMergedWithNotRequired) { + expect(hostVersionDAO.merge(hostVersionEntity)).andReturn(hostVersionEntity).once(); + } + + // replay + replay(managementController, response, clusters, hdfsService, resourceProviderFactory, + csvResourceProvider, cluster, repositoryVersionDAOMock, configHelper, schDatanode, + stageFactory, clusterVersionDAO, hostVersionDAO); + + ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider(type, + PropertyHelper.getPropertyIds(type), PropertyHelper.getKeyPropertyIds(type), + managementController); + + injector.injectMembers(provider); + + // add the property map to a set for the request. add more maps for multiple + // creates + Set> propertySet = new LinkedHashSet>(); + + Map properties = new LinkedHashMap(); + + // add properties to the request map + properties.put( + ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID, + clusterName); + + properties.put( + ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID, + repoVersion); + + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_STACK_PROPERTY_ID, + stackId.getStackName()); + + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID, + stackId.getStackVersion()); + + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_FORCE, "true"); + + propertySet.add(properties); + + // set the security auth + SecurityContextHolder.getContext().setAuthentication( + TestAuthenticationFactory.createAdministrator()); + + // create the request + Request request = PropertyHelper.getCreateRequest(propertySet, null); + + RequestStatus status = provider.createResources(request); + Assert.assertNotNull(status); + + // verify + verify(managementController, response, clusters, cluster, hostVersionDAO); + } + + private void testCreateResourcesExistingUpgrade(Authentication authentication) throws Exception { Resource.Type type = Resource.Type.ClusterStackVersion; http://git-wip-us.apache.org/repos/asf/ambari/blob/a396b7c0/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java index dcbc435..90a3d02 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java @@ -67,7 +67,6 @@ import org.apache.ambari.server.orm.dao.HostComponentStateDAO; import org.apache.ambari.server.orm.dao.HostDAO; import org.apache.ambari.server.orm.dao.HostVersionDAO; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; -import org.apache.ambari.server.orm.dao.ResourceTypeDAO; import org.apache.ambari.server.orm.dao.StackDAO; import org.apache.ambari.server.orm.entities.ClusterConfigEntity; import org.apache.ambari.server.orm.entities.ClusterConfigMappingEntity; @@ -108,6 +107,7 @@ import org.apache.ambari.server.state.configgroup.ConfigGroupFactory; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; import org.apache.ambari.server.state.host.HostHealthyHeartbeatEvent; import org.apache.ambari.server.state.host.HostRegistrationRequestEvent; +import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.apache.ambari.server.utils.EventBusSynchronizer; import org.apache.commons.lang.StringUtils; import org.junit.After; @@ -146,7 +146,6 @@ public class ClusterTest { private ConfigGroupFactory configGroupFactory; private OrmTestHelper helper; private StackDAO stackDAO; - private ResourceTypeDAO resourceTypeDAO; private ClusterDAO clusterDAO; private HostDAO hostDAO; private ClusterVersionDAO clusterVersionDAO; @@ -204,7 +203,6 @@ public class ClusterTest { metaInfo = injector.getInstance(AmbariMetaInfo.class); helper = injector.getInstance(OrmTestHelper.class); stackDAO = injector.getInstance(StackDAO.class); - resourceTypeDAO = injector.getInstance(ResourceTypeDAO.class); clusterDAO = injector.getInstance(ClusterDAO.class); hostDAO = injector.getInstance(HostDAO.class); clusterVersionDAO = injector.getInstance(ClusterVersionDAO.class); @@ -1720,7 +1718,7 @@ public class ClusterTest { List hostVersionsH1Before = hostVersionDAO.findByClusterAndHost("c1", "h1"); assertEquals(1, hostVersionsH1Before.size()); - c1.transitionHostsToInstalling(entityHDP2); + c1.transitionHosts(entityHDP2, RepositoryVersionState.INSTALLING); List hostVersionsH1After = hostVersionDAO.findByClusterAndHost("c1", "h1"); assertEquals(2, hostVersionsH1After.size()); @@ -1739,7 +1737,7 @@ public class ClusterTest { assertTrue(checked); // Test for update of existing host stack version - c1.transitionHostsToInstalling(entityHDP2); + c1.transitionHosts(entityHDP2, RepositoryVersionState.INSTALLING); hostVersionsH1After = hostVersionDAO.findByClusterAndHost("c1", "h1"); assertEquals(2, hostVersionsH1After.size()); @@ -1778,7 +1776,7 @@ public class ClusterTest { hostInMaintenanceMode.setMaintenanceState(c1.getClusterId(), MaintenanceState.ON); // transition host versions to INSTALLING - c1.transitionHostsToInstalling(entityHDP2); + c1.transitionHosts(entityHDP2, RepositoryVersionState.INSTALLING); List hostInMaintModeVersions = hostVersionDAO.findByClusterAndHost("c1", hostInMaintenanceMode.getHostName()); @@ -2721,4 +2719,27 @@ public class ClusterTest { assertFalse(((ClusterImpl) cluster).isClusterPropertyCached("foo")); } + + /** + * Tests that the {@link ClusterVersionEntity} can be created initially with a + * state of {@link RepositoryVersionState#INSTALLED}. This state is needed for + * {@link UpgradeType#HOST_ORDERED}. + * + * @throws Exception + */ + @Test + public void testClusterVersionCreationWithInstalledState() throws Exception { + StackId stackId = new StackId("HDP", "0.1"); + StackEntity stackEntity = stackDAO.find(stackId.getStackName(), stackId.getStackVersion()); + org.junit.Assert.assertNotNull(stackEntity); + + String clusterName = "c1"; + clusters.addCluster(clusterName, stackId); + c1 = clusters.getCluster(clusterName); + + helper.getOrCreateRepositoryVersion(stackId, stackId.getStackVersion()); + + c1.createClusterVersion(stackId, stackId.getStackVersion(), "admin", + RepositoryVersionState.INSTALLED); + } }