Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A2F651744A for ; Thu, 27 Aug 2015 20:36:27 +0000 (UTC) Received: (qmail 29533 invoked by uid 500); 27 Aug 2015 20:36:27 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 29438 invoked by uid 500); 27 Aug 2015 20:36:27 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 29427 invoked by uid 99); 27 Aug 2015 20:36:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 27 Aug 2015 20:36:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4A03DE7E8E; Thu, 27 Aug 2015 20:36:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jonathanhurley@apache.org To: commits@ambari.apache.org Date: Thu, 27 Aug 2015 20:36:28 -0000 Message-Id: <1031caa8227446c9ac144ae0b0cf8824@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/2] ambari git commit: AMBARI-12903 - Pre-Req Checks Should Only Warn On Unhealthy Slave Hosts Not In MM (jonathanhurley) AMBARI-12903 - Pre-Req Checks Should Only Warn On Unhealthy Slave Hosts Not In MM (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/59dd207c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/59dd207c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/59dd207c Branch: refs/heads/trunk Commit: 59dd207c39968df161341568f7538b9e3a990de2 Parents: 3b5efe4 Author: Jonathan Hurley Authored: Thu Aug 27 12:04:24 2015 -0400 Committer: Jonathan Hurley Committed: Thu Aug 27 15:22:58 2015 -0400 ---------------------------------------------------------------------- .../ambari/server/checks/CheckDescription.java | 4 +- .../server/checks/HostsHeartbeatCheck.java | 58 +++++++++++++--- .../checks/HostsRepositoryVersionCheck.java | 73 +++++++++++--------- .../server/checks/HostsHeartbeatCheckTest.java | 27 ++++++-- 4 files changed, 111 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/59dd207c/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java index 5cfbb47..7151b0e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java @@ -41,7 +41,9 @@ public enum CheckDescription { "All hosts must be heartbeating with the Ambari Server unless they are in Maintenance Mode", new HashMap() {{ put(AbstractCheckDescriptor.DEFAULT, - "The following hosts must be heartbeating to the Ambari Server: {{fails}}."); + "The following hosts must be heartbeating to the Ambari Server or be put into maintenance mode."); + put(HostsHeartbeatCheck.KEY_HOSTS_IN_MM_WARNING, + "The following hosts are in maintenance mode and will not be a part of the upgrade."); }}), HOSTS_MASTER_MAINTENANCE(PrereqCheckType.HOST, http://git-wip-us.apache.org/repos/asf/ambari/blob/59dd207c/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsHeartbeatCheck.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsHeartbeatCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsHeartbeatCheck.java index 6076a32..a8600c4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsHeartbeatCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsHeartbeatCheck.java @@ -17,13 +17,13 @@ */ package org.apache.ambari.server.checks; -import java.util.Map; +import java.util.Collection; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.PrereqCheckRequest; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Host; -import org.apache.ambari.server.state.HostHealthStatus; +import org.apache.ambari.server.state.HostHealthStatus.HealthStatus; import org.apache.ambari.server.state.MaintenanceState; import org.apache.ambari.server.state.stack.PrereqCheckStatus; import org.apache.ambari.server.state.stack.PrerequisiteCheck; @@ -31,12 +31,23 @@ import org.apache.ambari.server.state.stack.PrerequisiteCheck; import com.google.inject.Singleton; /** - * Checks that all hosts are either in maintenance mode or heartbeating with the server. + * Checks that all hosts are heartbeating with the Ambari Server. If there is a + * host which is not heartbeating, then it must be in maintenance mode to + * prevent a failure of this check. + *

+ * Hosts that are in maintenance mode will be added to a warning that they will + * not be included in the upgrade. + *

+ * This check will return {@link PrereqCheckStatus#FAIL} if there are hosts not + * heartbeating and not in maintenance mode. Otherwise, it will return + * {@link PrereqCheckStatus#WARNING} for any hosts in maintenance mode. */ @Singleton @UpgradeCheck(group = UpgradeCheckGroup.LIVELINESS, order = 1.0f) public class HostsHeartbeatCheck extends AbstractCheckDescriptor { + static final String KEY_HOSTS_IN_MM_WARNING = "key.hosts.in.mm.warning"; + /** * Constructor. */ @@ -45,19 +56,48 @@ public class HostsHeartbeatCheck extends AbstractCheckDescriptor { } @Override - public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException { + public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) + throws AmbariException { final String clusterName = request.getClusterName(); final Cluster cluster = clustersProvider.get().getCluster(clusterName); - final Map clusterHosts = clustersProvider.get().getHostsForCluster(clusterName); - for (Map.Entry hostEntry : clusterHosts.entrySet()) { - final Host host = hostEntry.getValue(); - if (host.getHealthStatus().getHealthStatus() == HostHealthStatus.HealthStatus.UNKNOWN && host.getMaintenanceState(cluster.getClusterId()) == MaintenanceState.OFF) { - prerequisiteCheck.getFailedOn().add(host.getHostName()); + Collection hosts = cluster.getHosts(); + + for (Host host : hosts) { + HealthStatus hostHealth = host.getHealthStatus().getHealthStatus(); + MaintenanceState maintenanceState = host.getMaintenanceState(cluster.getClusterId()); + switch (hostHealth) { + case UNHEALTHY: + case UNKNOWN: + if (maintenanceState == MaintenanceState.OFF) { + prerequisiteCheck.getFailedOn().add(host.getHostName()); + } + break; + default: + break; + } } + + // for any hosts unhealthy and NOT in MM mode, fail this check if (!prerequisiteCheck.getFailedOn().isEmpty()) { prerequisiteCheck.setStatus(PrereqCheckStatus.FAIL); prerequisiteCheck.setFailReason(getFailReason(prerequisiteCheck, request)); + return; + } + + // no failues so far, check to see if any hosts are in MM so that this check + // will produce a warning + for (Host host : hosts) { + MaintenanceState maintenanceState = host.getMaintenanceState(cluster.getClusterId()); + if (maintenanceState != MaintenanceState.OFF) { + prerequisiteCheck.getFailedOn().add(host.getHostName()); + } + } + + if (!prerequisiteCheck.getFailedOn().isEmpty()) { + prerequisiteCheck.setStatus(PrereqCheckStatus.WARNING); + prerequisiteCheck.setFailReason( + getFailReason(KEY_HOSTS_IN_MM_WARNING, prerequisiteCheck, request)); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/59dd207c/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java index 6ebf8e1..eaa0096 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java @@ -35,7 +35,10 @@ import org.apache.ambari.server.state.stack.PrerequisiteCheck; import com.google.inject.Singleton; /** - * Checks that all hosts have particular repository version. + * Checks that all hosts have particular repository version. Hosts that are in + * maintenance mode will be skipped and will not report a warning. Even if they + * do not have the repo version, they will not be included in the upgrade + * orchstration, so no warning is required. */ @Singleton @UpgradeCheck(group = UpgradeCheckGroup.REPOSITORY_VERSION) @@ -60,52 +63,54 @@ public class HostsRepositoryVersionCheck extends AbstractCheckDescriptor { } @Override - public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException { + public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) + throws AmbariException { final String clusterName = request.getClusterName(); final Cluster cluster = clustersProvider.get().getCluster(clusterName); final Map clusterHosts = clustersProvider.get().getHostsForCluster(clusterName); final StackId stackId = cluster.getDesiredStackVersion(); for (Host host : clusterHosts.values()) { - if (host.getMaintenanceState(cluster.getClusterId()) == MaintenanceState.OFF) { - - if (null != request.getRepositoryVersion()) { - boolean found = false; - for (HostVersionEntity hve : hostVersionDaoProvider.get().findByHost(host.getHostName())) { + // hosts in MM will produce a warning if they do not have the repo version + MaintenanceState maintenanceState = host.getMaintenanceState(cluster.getClusterId()); + if (maintenanceState != MaintenanceState.OFF) { + continue; + } - if (hve.getRepositoryVersion().getVersion().equals(request.getRepositoryVersion()) && - hve.getState() == RepositoryVersionState.INSTALLED) { - found = true; - break; - } - } + if (null != request.getRepositoryVersion()) { + boolean found = false; + for (HostVersionEntity hve : hostVersionDaoProvider.get().findByHost(host.getHostName())) { - if (!found) { - prerequisiteCheck.getFailedOn().add(host.getHostName()); - } - } else { - final RepositoryVersionEntity repositoryVersion = repositoryVersionDaoProvider.get().findByStackAndVersion( - stackId, request.getRepositoryVersion()); - if (repositoryVersion == null) { - prerequisiteCheck.setStatus(PrereqCheckStatus.FAIL); - prerequisiteCheck.setFailReason(getFailReason(KEY_NO_REPO_VERSION, prerequisiteCheck, request)); - prerequisiteCheck.getFailedOn().addAll(clusterHosts.keySet()); - return; + if (hve.getRepositoryVersion().getVersion().equals(request.getRepositoryVersion()) + && hve.getState() == RepositoryVersionState.INSTALLED) { + found = true; + break; } + } - StackEntity repositoryStackEntity = repositoryVersion.getStack(); - StackId repositoryStackId = new StackId( - repositoryStackEntity.getStackName(), - repositoryStackEntity.getStackVersion()); + if (!found) { + prerequisiteCheck.getFailedOn().add(host.getHostName()); + } + } else { + final RepositoryVersionEntity repositoryVersion = repositoryVersionDaoProvider.get().findByStackAndVersion( + stackId, request.getRepositoryVersion()); + if (repositoryVersion == null) { + prerequisiteCheck.setStatus(PrereqCheckStatus.FAIL); + prerequisiteCheck.setFailReason( + getFailReason(KEY_NO_REPO_VERSION, prerequisiteCheck, request)); + prerequisiteCheck.getFailedOn().addAll(clusterHosts.keySet()); + return; + } - final HostVersionEntity hostVersion = hostVersionDaoProvider.get().findByClusterStackVersionAndHost( - clusterName, repositoryStackId, repositoryVersion.getVersion(), - host.getHostName()); + StackEntity repositoryStackEntity = repositoryVersion.getStack(); + StackId repositoryStackId = new StackId(repositoryStackEntity.getStackName(), + repositoryStackEntity.getStackVersion()); - if (hostVersion == null || hostVersion.getState() != RepositoryVersionState.INSTALLED) { - prerequisiteCheck.getFailedOn().add(host.getHostName()); - } + final HostVersionEntity hostVersion = hostVersionDaoProvider.get().findByClusterStackVersionAndHost( + clusterName, repositoryStackId, repositoryVersion.getVersion(), host.getHostName()); + if (hostVersion == null || hostVersion.getState() != RepositoryVersionState.INSTALLED) { + prerequisiteCheck.getFailedOn().add(host.getHostName()); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/59dd207c/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsHeartbeatCheckTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsHeartbeatCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsHeartbeatCheckTest.java index 5f6cae9..847027c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsHeartbeatCheckTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsHeartbeatCheckTest.java @@ -17,8 +17,8 @@ */ package org.apache.ambari.server.checks; -import java.util.HashMap; -import java.util.Map; +import java.util.ArrayList; +import java.util.List; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.PrereqCheckRequest; @@ -74,7 +74,7 @@ public class HostsHeartbeatCheckTest { Mockito.when(cluster.getClusterId()).thenReturn(1L); Mockito.when(cluster.getCurrentStackVersion()).thenReturn(new StackId("HDP", "2.2")); Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); - final Map hosts = new HashMap(); + final List hosts = new ArrayList<>(); final Host host1 = Mockito.mock(Host.class); final Host host2 = Mockito.mock(Host.class); final Host host3 = Mockito.mock(Host.class); @@ -90,17 +90,30 @@ public class HostsHeartbeatCheckTest { Mockito.when(status1.getHealthStatus()).thenReturn(HealthStatus.HEALTHY); Mockito.when(status2.getHealthStatus()).thenReturn(HealthStatus.HEALTHY); Mockito.when(status3.getHealthStatus()).thenReturn(HealthStatus.UNKNOWN); - hosts.put("host1", host1); - hosts.put("host2", host2); - hosts.put("host3", host3); - Mockito.when(clusters.getHostsForCluster("cluster")).thenReturn(hosts); + hosts.add(host1); + hosts.add(host2); + hosts.add(host3); + Mockito.when(cluster.getHosts()).thenReturn(hosts); PrerequisiteCheck check = new PrerequisiteCheck(null, null); hostHeartbeatCheck.perform(check, new PrereqCheckRequest("cluster")); Assert.assertEquals(PrereqCheckStatus.FAIL, check.getStatus()); + // put the unhealthy host into MM to now produce a warning + check = new PrerequisiteCheck(null, null); + Mockito.when(host3.getMaintenanceState(1L)).thenReturn(MaintenanceState.ON); + hostHeartbeatCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus()); + + // make it's status healthy, but keep in MM to still produce a warning + check = new PrerequisiteCheck(null, null); Mockito.when(status3.getHealthStatus()).thenReturn(HealthStatus.HEALTHY); + hostHeartbeatCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus()); + + // take it out our MM to allow the check to pass check = new PrerequisiteCheck(null, null); + Mockito.when(host3.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF); hostHeartbeatCheck.perform(check, new PrereqCheckRequest("cluster")); Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus());