Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 313CE200CDE for ; Mon, 24 Jul 2017 17:26:33 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 30101165521; Mon, 24 Jul 2017 15:26:33 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D12361654FE for ; Mon, 24 Jul 2017 17:26:31 +0200 (CEST) Received: (qmail 84009 invoked by uid 500); 24 Jul 2017 15:26:30 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 82386 invoked by uid 99); 24 Jul 2017 15:26:29 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 24 Jul 2017 15:26:29 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B2ACEF3275; Mon, 24 Jul 2017 15:26:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ncole@apache.org To: commits@ambari.apache.org Date: Mon, 24 Jul 2017 15:27:01 -0000 Message-Id: <266210b8384e46eb81cd53dbf247670c@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [35/50] [abbrv] ambari git commit: AMBARI-21526 - ZKFC Doesn't Update its Version On Some Upgrades (jonathanhurley) archived-at: Mon, 24 Jul 2017 15:26:33 -0000 AMBARI-21526 - ZKFC Doesn't Update its Version On Some Upgrades (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4fdca575 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4fdca575 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4fdca575 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 4fdca575bcf61e5d8d85a4ff6b7fc08d61dfb5e4 Parents: 6b4d093 Author: Jonathan Hurley Authored: Wed Jul 19 16:49:01 2017 -0400 Committer: Jonathan Hurley Committed: Wed Jul 19 20:30:27 2017 -0400 ---------------------------------------------------------------------- .../libraries/functions/stack_features.py | 2 +- .../listeners/upgrade/StackVersionListener.java | 33 +++++++++-- .../2.1.0.2.0/package/scripts/zkfc_slave.py | 8 +-- .../upgrade/StackVersionListenerTest.java | 58 ++++++++++++-------- 4 files changed, 65 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4fdca575/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index 24201dd..b241288 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -123,7 +123,7 @@ def get_stack_feature_version(config): # if this is not an upgrade, then we take the simple path if upgrade_direction is None: Logger.info( - "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}-> {4}".format( + "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3} -> {4}".format( stack_version, current_cluster_version, command_stack, command_version, version_for_stack_feature_checks)) return version_for_stack_feature_checks http://git-wip-us.apache.org/repos/asf/ambari/blob/4fdca575/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java index f8d5a5f..b812476 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java @@ -19,13 +19,16 @@ package org.apache.ambari.server.events.listeners.upgrade; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.EagerSingleton; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent; import org.apache.ambari.server.events.publishers.VersionEventPublisher; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.State; import org.apache.ambari.server.state.UpgradeState; import org.apache.commons.lang.StringUtils; @@ -34,6 +37,7 @@ import org.slf4j.LoggerFactory; import com.google.common.eventbus.Subscribe; import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.Singleton; /** @@ -55,6 +59,13 @@ public class StackVersionListener { private RepositoryVersionDAO repositoryVersionDAO; /** + * Used for looking up a component's advertising version status given a stack + * and name. + */ + @Inject + private Provider ambariMetaInfoProvider; + + /** * Constructor. * * @param eventPublisher the publisher @@ -95,20 +106,30 @@ public class StackVersionListener { // Update host component version value if needed try { - ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent( - sch.getServiceComponentName()); + // get the component information for the desired stack; if a component + // moves from UNKNOWN to providing a version, we must do the version + // advertised check against the target stack + StackId desiredStackId = sch.getDesiredStackId(); + + AmbariMetaInfo ambariMetaInfo = ambariMetaInfoProvider.get(); + ComponentInfo componentInfo = ambariMetaInfo.getComponent(desiredStackId.getStackName(), + desiredStackId.getStackVersion(), sch.getServiceName(), sch.getServiceComponentName()); // not advertising a version, do nothing - if (!sc.isVersionAdvertised()) { + if (!componentInfo.isVersionAdvertised()) { // that's odd; a version came back - log it and still do nothing if (!StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, newVersion)) { - LOG.debug( + LOG.warn( "ServiceComponent {} doesn't advertise version, however ServiceHostComponent {} on host {} advertised version as {}. Skipping version update", - sc.getName(), sch.getServiceComponentName(), sch.getHostName(), newVersion); + sch.getServiceComponentName(), sch.getServiceComponentName(), sch.getHostName(), + newVersion); } return; } + ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent( + sch.getServiceComponentName()); + // proces the UNKNOWN version if (StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, newVersion)) { processUnknownDesiredVersion(cluster, sc, sch, newVersion); @@ -215,4 +236,4 @@ public class StackVersionListener { sch.setUpgradeState(upgradeState); sch.recalculateHostVersionState(); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4fdca575/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py index ca5f605..cd47109 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py @@ -30,11 +30,6 @@ from resource_management.core import shell from resource_management.libraries.functions import conf_select, stack_select from resource_management.libraries.functions.constants import StackFeature from resource_management.libraries.functions.check_process_status import check_process_status -from resource_management.libraries.functions.security_commons import build_expectations -from resource_management.libraries.functions.security_commons import cached_kinit_executor -from resource_management.libraries.functions.security_commons import get_params_from_filesystem -from resource_management.libraries.functions.security_commons import validate_security_config_properties -from resource_management.libraries.functions.security_commons import FILE_TYPE_XML from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.script import Script from resource_management.core.resources.zkmigrator import ZkMigrator @@ -144,8 +139,7 @@ class ZkfcSlaveDefault(ZkfcSlave): Logger.info("Executing Stack Upgrade pre-restart") import params env.set_params(params) - if params.version and check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version) \ - and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + if check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version_for_stack_feature_checks): conf_select.select(params.stack_name, "hadoop", params.version) stack_select.select("hadoop-hdfs-zkfc", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/4fdca575/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java index efe594a..ff6fab9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java @@ -65,16 +65,15 @@ public class StackVersionListenerTest extends EasyMockSupport { private static final RepositoryVersionEntity DUMMY_REPOSITORY_VERSION_ENTITY = new RepositoryVersionEntity(); private static final HostVersionEntity DUMMY_HOST_VERSION_ENTITY = new HostVersionEntity(); private static final UpgradeEntity DUMMY_UPGRADE_ENTITY = new UpgradeEntity(); - public static final String STACK_NAME = "HDP-2.4.0.0"; - public static final String STACK_VERSION = "2.4.0.0"; + public static final String STACK_NAME = "HDP"; + public static final String STACK_VERSION = "2.4"; private Cluster cluster; private ServiceComponentHost sch; private Service service; private ServiceComponent serviceComponent; private VersionEventPublisher publisher = new VersionEventPublisher(); - private ComponentInfo componentInfo; - private StackId stackId; + private StackId stackId = new StackId(STACK_NAME, STACK_VERSION); @TestSubject private StackVersionListener listener = new StackVersionListener(publisher); @@ -82,6 +81,12 @@ public class StackVersionListenerTest extends EasyMockSupport { @Mock private Provider ambariMetaInfoProvider; + @Mock + private ComponentInfo componentInfo; + + @Mock + private AmbariMetaInfo ambariMetaInfo; + @Before public void setup() throws Exception { cluster = createNiceMock(Cluster.class); @@ -89,20 +94,27 @@ public class StackVersionListenerTest extends EasyMockSupport { service = createNiceMock(Service.class); serviceComponent = createNiceMock(ServiceComponent.class); componentInfo = createNiceMock(ComponentInfo.class); - stackId = createNiceMock(StackId.class); expect(cluster.getClusterId()).andReturn(CLUSTER_ID); - expect(cluster.getService(SERVICE_NAME)).andReturn(service).atLeastOnce(); - expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).atLeastOnce(); + expect(cluster.getService(SERVICE_NAME)).andReturn(service).anyTimes(); + expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn( + serviceComponent).anyTimes(); + expect(sch.getDesiredStackId()).andReturn(stackId).atLeastOnce(); expect(sch.getServiceName()).andReturn(SERVICE_NAME).atLeastOnce(); expect(sch.getServiceComponentName()).andReturn(SERVICE_COMPONENT_NAME).atLeastOnce(); + + expect(ambariMetaInfoProvider.get()).andReturn(ambariMetaInfo).atLeastOnce(); + expect(ambariMetaInfo.getComponent(STACK_NAME, STACK_VERSION, SERVICE_NAME, + SERVICE_COMPONENT_NAME)).andReturn(componentInfo).atLeastOnce(); + + injectMocks(listener); } @Test public void testRecalculateHostVersionStateWhenVersionIsNullAndNewVersionIsNotBlank() throws AmbariException { expect(sch.getVersion()).andReturn(null); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setVersion(INVALID_NEW_VERSION); expectLastCall().once(); expect(sch.recalculateHostVersionState()).andReturn(null).once(); @@ -115,7 +127,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateHostVersionStateWhenVersionIsUnknownAndNewVersionIsNotBlank() throws AmbariException { expect(sch.getVersion()).andReturn(UNKNOWN_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setVersion(INVALID_NEW_VERSION); expectLastCall().once(); expect(sch.recalculateHostVersionState()).andReturn(null).once(); @@ -128,7 +140,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateClusterVersionStateWhenVersionIsNullAndNewVersionIsValid() throws AmbariException { expect(sch.getVersion()).andReturn(null); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setVersion(VALID_NEW_VERSION); expectLastCall().once(); expect(sch.recalculateHostVersionState()).andReturn(DUMMY_HOST_VERSION_ENTITY).once(); @@ -141,7 +153,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateClusterVersionStateWhenVersionIsUnknownAndNewVersionIsValid() throws AmbariException { expect(sch.getVersion()).andReturn(UNKNOWN_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setVersion(VALID_NEW_VERSION); expectLastCall().once(); expect(sch.recalculateHostVersionState()).andReturn(DUMMY_HOST_VERSION_ENTITY).once(); @@ -154,7 +166,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateHostVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsNotValid() throws AmbariException { expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setUpgradeState(UpgradeState.NONE); expectLastCall().once(); sch.setVersion(INVALID_NEW_VERSION); @@ -169,7 +181,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateClusterVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsValid() throws AmbariException { expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setUpgradeState(UpgradeState.NONE); expectLastCall().once(); sch.setVersion(VALID_NEW_VERSION); @@ -183,7 +195,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateClusterVersionStateWhenVersionNotAdvertised() throws AmbariException { - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.FALSE); + expect(componentInfo.isVersionAdvertised()).andReturn(false).once(); replayAll(); sendEventAndVerify(VALID_NEW_VERSION); } @@ -191,7 +203,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testNoActionTakenOnNullVersion() { - expect(serviceComponent.isVersionAdvertised()).andReturn(true); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); resetAll(); replayAll(); @@ -204,7 +216,7 @@ public class StackVersionListenerTest extends EasyMockSupport { expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION); expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setUpgradeState(UpgradeState.COMPLETE); expectLastCall().once(); @@ -218,7 +230,7 @@ public class StackVersionListenerTest extends EasyMockSupport { public void testSetUpgradeStateToNoneWhenNoUpgradeAndNewVersionIsEqualToComponentDesiredVersion() { expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION); expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setUpgradeState(UpgradeState.NONE); expectLastCall().once(); @@ -232,7 +244,7 @@ public class StackVersionListenerTest extends EasyMockSupport { public void testSetUpgradeStateToVersionMismatchWhenUpgradeIsInProgressAndNewVersionIsNotEqualToComponentDesiredVersion() { expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION); expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setUpgradeState(UpgradeState.VERSION_MISMATCH); expectLastCall().once(); @@ -251,7 +263,7 @@ public class StackVersionListenerTest extends EasyMockSupport { expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH); expect(cluster.getUpgradeInProgress()).andReturn(DUMMY_UPGRADE_ENTITY); expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setUpgradeState(UpgradeState.COMPLETE); expectLastCall().once(); @@ -265,7 +277,7 @@ public class StackVersionListenerTest extends EasyMockSupport { expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION); expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH); expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setUpgradeState(UpgradeState.NONE); expectLastCall().once(); @@ -277,7 +289,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testSetUpgradeStateToVersionMismatchByDefaultWhenHostAndNewVersionsAreValid() { expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); sch.setUpgradeState(UpgradeState.VERSION_MISMATCH); expectLastCall().once(); @@ -289,7 +301,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testSetRepositoryVersion() throws Exception { expect(sch.getVersion()).andReturn(UNKNOWN_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE); + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); RepositoryVersionDAO dao = createNiceMock(RepositoryVersionDAO.class); RepositoryVersionEntity entity = createNiceMock(RepositoryVersionEntity.class); @@ -321,6 +333,8 @@ public class StackVersionListenerTest extends EasyMockSupport { */ @Test public void testRepositoryVersionNotSetDuringUpgrade() throws Exception { + expect(componentInfo.isVersionAdvertised()).andReturn(true).once(); + // this call will make it seem like there is an upgrade in progress expect(cluster.getUpgradeInProgress()).andReturn(createNiceMock(UpgradeEntity.class));