cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wid...@apache.org
Subject [10/50] [abbrv] git commit: updated refs/heads/reporter to 3e1816d
Date Mon, 27 Jul 2015 08:42:58 GMT
CLOUDSTACK-8628: Send an alert when fencing a KVM host failed

Also change the logging a bit so that you get useful logs when
not running on DEBUG level

Signed-off-by: Wido den Hollander <wido@widodh.nl>

This closes #580


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/fbe3b04a
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/fbe3b04a
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/fbe3b04a

Branch: refs/heads/reporter
Commit: fbe3b04a43ff7380a6874a2c331f372aa59ccf3d
Parents: d1f76a2
Author: Wido den Hollander <wido@widodh.nl>
Authored: Thu Jul 16 16:15:20 2015 +0200
Committer: Wido den Hollander <wido@widodh.nl>
Committed: Fri Jul 17 12:33:11 2015 +0200

----------------------------------------------------------------------
 server/src/com/cloud/ha/KVMFencer.java      | 26 +++++++++++++++---------
 server/test/com/cloud/ha/KVMFencerTest.java | 22 ++++++++++++++++++++
 2 files changed, 38 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/fbe3b04a/server/src/com/cloud/ha/KVMFencer.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/KVMFencer.java b/server/src/com/cloud/ha/KVMFencer.java
index 7392dff..b5834ef 100644
--- a/server/src/com/cloud/ha/KVMFencer.java
+++ b/server/src/com/cloud/ha/KVMFencer.java
@@ -26,6 +26,7 @@ import javax.naming.ConfigurationException;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
+import com.cloud.alert.AlertManager;
 import com.cloud.agent.api.FenceAnswer;
 import com.cloud.agent.api.FenceCommand;
 import com.cloud.exception.AgentUnavailableException;
@@ -48,6 +49,8 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
     @Inject
     AgentManager _agentMgr;
     @Inject
+    AlertManager _alertMgr;
+    @Inject
     ResourceManager _resourceMgr;
 
     @Override
@@ -75,18 +78,22 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
     @Override
     public Boolean fenceOff(VirtualMachine vm, Host host) {
         if (host.getHypervisorType() != HypervisorType.KVM && host.getHypervisorType()
!= HypervisorType.LXC) {
-            s_logger.debug("Don't know how to fence non kvm hosts " + host.getHypervisorType());
+            s_logger.warn("Don't know how to fence non kvm hosts " + host.getHypervisorType());
             return null;
         }
 
         List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(host.getClusterId());
         FenceCommand fence = new FenceCommand(vm, host);
 
+        int i = 0;
         for (HostVO h : hosts) {
             if (h.getHypervisorType() == HypervisorType.KVM || h.getHypervisorType() == HypervisorType.LXC)
{
                 if (h.getStatus() != Status.Up) {
                     continue;
                 }
+
+                i++;
+
                 if (h.getId() == host.getId()) {
                     continue;
                 }
@@ -94,14 +101,10 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
                 try {
                     answer = (FenceAnswer)_agentMgr.send(h.getId(), fence);
                 } catch (AgentUnavailableException e) {
-                    if (s_logger.isDebugEnabled()) {
-                        s_logger.debug("Moving on to the next host because " + h.toString()
+ " is unavailable");
-                    }
+                    s_logger.info("Moving on to the next host because " + h.toString() +
" is unavailable");
                     continue;
                 } catch (OperationTimedoutException e) {
-                    if (s_logger.isDebugEnabled()) {
-                        s_logger.debug("Moving on to the next host because " + h.toString()
+ " is unavailable");
-                    }
+                    s_logger.info("Moving on to the next host because " + h.toString() +
" is unavailable");
                     continue;
                 }
                 if (answer != null && answer.getResult()) {
@@ -110,9 +113,12 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
             }
         }
 
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Unable to fence off " + vm.toString() + " on " + host.toString());
-        }
+        _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(),
host.getPodId(),
+                            "Unable to fence off host: " + host.getId(),
+                            "Fencing off host " + host.getId() + " did not succeed after
asking " + i + " hosts. " +
+                            "Check Agent logs for more information.");
+
+        s_logger.error("Unable to fence off " + vm.toString() + " on " + host.toString());
 
         return false;
     }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/fbe3b04a/server/test/com/cloud/ha/KVMFencerTest.java
----------------------------------------------------------------------
diff --git a/server/test/com/cloud/ha/KVMFencerTest.java b/server/test/com/cloud/ha/KVMFencerTest.java
index cdd13b6..da120af 100644
--- a/server/test/com/cloud/ha/KVMFencerTest.java
+++ b/server/test/com/cloud/ha/KVMFencerTest.java
@@ -31,6 +31,7 @@ import org.mockito.Mockito;
 import org.mockito.runners.MockitoJUnitRunner;
 
 import com.cloud.agent.AgentManager;
+import com.cloud.alert.AlertManager;
 import com.cloud.agent.api.FenceAnswer;
 import com.cloud.agent.api.FenceCommand;
 import com.cloud.exception.AgentUnavailableException;
@@ -50,6 +51,8 @@ public class KVMFencerTest {
     @Mock
     AgentManager agentManager;
     @Mock
+    AlertManager alertMgr;
+    @Mock
     ResourceManager resourceManager;
 
     KVMFencer fencer;
@@ -58,6 +61,7 @@ public class KVMFencerTest {
     public void setup() {
         fencer = new KVMFencer();
         fencer._agentMgr = agentManager;
+        fencer._alertMgr = alertMgr;
         fencer._hostDao = hostDao;
         fencer._resourceMgr = resourceManager;
     }
@@ -67,6 +71,8 @@ public class KVMFencerTest {
         HostVO host = Mockito.mock(HostVO.class);
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getStatus()).thenReturn(Status.Up);
         Mockito.when(host.getId()).thenReturn(1l);
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -80,6 +86,8 @@ public class KVMFencerTest {
         HostVO host = Mockito.mock(HostVO.class);
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getStatus()).thenReturn(Status.Down);
         Mockito.when(host.getId()).thenReturn(1l);
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -94,12 +102,16 @@ public class KVMFencerTest {
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(host.getStatus()).thenReturn(Status.Up);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(1l);
 
         HostVO secondHost = Mockito.mock(HostVO.class);
         Mockito.when(secondHost.getClusterId()).thenReturn(1l);
         Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+        Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+        Mockito.when(secondHost.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(2l);
 
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -118,12 +130,16 @@ public class KVMFencerTest {
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(host.getStatus()).thenReturn(Status.Up);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(1l);
 
         HostVO secondHost = Mockito.mock(HostVO.class);
         Mockito.when(secondHost.getClusterId()).thenReturn(1l);
         Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+        Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+        Mockito.when(secondHost.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(2l);
 
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -141,12 +157,16 @@ public class KVMFencerTest {
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(host.getStatus()).thenReturn(Status.Up);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(1l);
 
         HostVO secondHost = Mockito.mock(HostVO.class);
         Mockito.when(secondHost.getClusterId()).thenReturn(1l);
         Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+        Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+        Mockito.when(secondHost.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(2l);
 
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -165,6 +185,8 @@ public class KVMFencerTest {
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.Any);
         Mockito.when(host.getStatus()).thenReturn(Status.Down);
         Mockito.when(host.getId()).thenReturn(1l);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
 
         Mockito.when(resourceManager.listAllHostsInCluster(1l)).thenReturn(Collections.singletonList(host));


Mime
View raw message