cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject git commit: updated refs/heads/master to 3907bc6
Date Fri, 14 Nov 2014 21:18:35 GMT
Repository: cloudstack
Updated Branches:
  refs/heads/master a49e24095 -> 3907bc6f9


CLOUDSTACK-7916: Generate Alerts if System VMs cannot be started.


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/3907bc6f
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/3907bc6f
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/3907bc6f

Branch: refs/heads/master
Commit: 3907bc6f9d2b4a61a4d0b32e5c1a15d95210531e
Parents: a49e240
Author: Nitin Mehta <nitin.mehta@citrix.com>
Authored: Fri Nov 14 13:18:27 2014 -0800
Committer: Nitin Mehta <nitin.mehta@citrix.com>
Committed: Fri Nov 14 13:18:27 2014 -0800

----------------------------------------------------------------------
 .../cloud/alert/ConsoleProxyAlertAdapter.java   |  12 +-
 .../alert/SecondaryStorageVmAlertAdapter.java   |  11 +-
 .../consoleproxy/ConsoleProxyManagerImpl.java   |  74 +++++++------
 .../SecondaryStorageManagerImpl.java            | 110 ++++++++++---------
 4 files changed, 112 insertions(+), 95 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3907bc6f/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java b/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java
index df4f5ba..4ca5f4a 100644
--- a/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java
+++ b/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java
@@ -53,7 +53,8 @@ public class ConsoleProxyAlertAdapter extends AdapterBase implements AlertAdapte
 
         DataCenterVO dc = _dcDao.findById(args.getZoneId());
         ConsoleProxyVO proxy = args.getProxy();
-        if (proxy == null)
+        //FIXME - Proxy can be null in case of creation failure. Have a better fix than checking
for != 0
+        if (proxy == null && args.getProxyId() != 0)
             proxy = _consoleProxyDao.findById(args.getProxyId());
 
         switch (args.getType()) {
@@ -98,12 +99,9 @@ public class ConsoleProxyAlertAdapter extends AdapterBase implements AlertAdapte
 
             case ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE:
                 if (s_logger.isDebugEnabled())
-                    s_logger.debug("Console proxy creation failure, zone: " + dc.getName()
+ ", proxy: " + proxy.getHostName() + ", public IP: " +
-                        proxy.getPublicIpAddress() + ", private IP: " + (proxy.getPrivateIpAddress()
== null ? "N/A" : proxy.getPrivateIpAddress()));
-
-                _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY, args.getZoneId(),
proxy.getPodIdToDeployIn(),
-                    "Console proxy creation failure. zone: " + dc.getName() + ", proxy: "
+ proxy.getHostName() + ", public IP: " + proxy.getPublicIpAddress() +
-                        ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" :
proxy.getPrivateIpAddress()) + ", error details: " + args.getMessage(),
+                    s_logger.debug("Console proxy creation failure, zone: " + dc.getName());
+                _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY, args.getZoneId(),
null,
+                    "Console proxy creation failure. zone: " + dc.getName() + ", error details:
" + args.getMessage(),
                     "Console proxy creation failure (zone " + dc.getName() + ")");
                 break;
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3907bc6f/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java b/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java
index b7834bb..3bfedce 100644
--- a/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java
+++ b/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java
@@ -53,7 +53,7 @@ public class SecondaryStorageVmAlertAdapter extends AdapterBase implements
Alert
 
         DataCenterVO dc = _dcDao.findById(args.getZoneId());
         SecondaryStorageVmVO secStorageVm = args.getSecStorageVm();
-        if (secStorageVm == null)
+        if (secStorageVm == null && args.getSecStorageVmId() != 0)
             secStorageVm = _ssvmDao.findById(args.getSecStorageVmId());
 
         switch (args.getType()) {
@@ -103,13 +103,10 @@ public class SecondaryStorageVmAlertAdapter extends AdapterBase implements
Alert
 
             case SecStorageVmAlertEventArgs.SSVM_CREATE_FAILURE:
                 if (s_logger.isDebugEnabled())
-                    s_logger.debug("Secondary Storage Vm creation failure, zone: " + dc.getName()
+ ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " +
-                        secStorageVm.getPublicIpAddress() + ", private IP: " + (secStorageVm.getPrivateIpAddress()
== null ? "N/A" : secStorageVm.getPrivateIpAddress()));
+                    s_logger.debug("Secondary Storage Vm creation failure, zone: " + dc.getName());
 
-                _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SSVM, args.getZoneId(),
secStorageVm.getPodIdToDeployIn(),
-                    "Secondary Storage Vm creation failure. zone: " +
-                        dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() +
", public IP: " + secStorageVm.getPublicIpAddress() + ", private IP: " +
-                        (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress())
+ ", error details: " + args.getMessage(),
+                _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SSVM, args.getZoneId(),
null,
+                    "Secondary Storage Vm creation failure. zone: " + dc.getName() + ", error
details: " + args.getMessage(),
                     "Secondary Storage Vm creation failure (zone " + dc.getName() + ")");
                 break;
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3907bc6f/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
index 14d782a..4ccaa98 100755
--- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
+++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
@@ -539,6 +539,7 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
             if (proxy.getState() == VirtualMachine.State.Stopped) {
                 _itMgr.advanceStart(proxy.getUuid(), null, null);
                 proxy = _consoleProxyDao.findById(proxy.getId());
+                return proxy;
             }
 
             // For VMs that are in Stopping, Starting, Migrating state, let client to wait
by returning null
@@ -668,9 +669,6 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
             if (s_logger.isDebugEnabled()) {
                 s_logger.debug("Unable to allocate console proxy storage, remove the console
proxy record from DB, proxy id: " + proxyVmId);
             }
-
-            SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT,
this,
-                new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE,
dataCenterId, proxyVmId, null, "Unable to allocate storage"));
         }
         return null;
     }
@@ -887,43 +885,57 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements
ConsoleProxy
         }
 
         ConsoleProxyVO proxy = null;
-        if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
-            try {
-                proxy = assignProxyFromStoppedPool(dataCenterId);
-                if (proxy == null) {
-                    if (s_logger.isInfoEnabled()) {
-                        s_logger.info("No stopped console proxy is available, need to allocate
a new console proxy");
-                    }
+        String errorString = null;
+        try{
+            if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
+                try {
+                    proxy = assignProxyFromStoppedPool(dataCenterId);
+                    if (proxy == null) {
+                        if (s_logger.isInfoEnabled()) {
+                            s_logger.info("No stopped console proxy is available, need to
allocate a new console proxy");
+                        }
 
-                    try {
-                        proxy = startNew(dataCenterId);
-                    } catch (ConcurrentOperationException e) {
-                        s_logger.info("Concurrent Operation caught " + e);
-                    }
-                } else {
-                    if (s_logger.isInfoEnabled()) {
-                        s_logger.info("Found a stopped console proxy, bring it up to running
pool. proxy vm id : " + proxy.getId());
+                        try {
+                            proxy = startNew(dataCenterId);
+                        } catch (ConcurrentOperationException e) {
+                            s_logger.info("Concurrent Operation caught " + e);
+                        }
+                    } else {
+                        if (s_logger.isInfoEnabled()) {
+                            s_logger.info("Found a stopped console proxy, bring it up to
running pool. proxy vm id : " + proxy.getId());
+                        }
                     }
+                } finally {
+                    _allocProxyLock.unlock();
+                }
+            } else {
+                if (s_logger.isInfoEnabled()) {
+                    s_logger.info("Unable to acquire proxy allocation lock, skip for next
time");
                 }
-            } finally {
-                _allocProxyLock.unlock();
-            }
-        } else {
-            if (s_logger.isInfoEnabled()) {
-                s_logger.info("Unable to acquire proxy allocation lock, skip for next time");
             }
-        }
-
-        if (proxy != null) {
-            long proxyVmId = proxy.getId();
-            proxy = startProxy(proxyVmId);
 
             if (proxy != null) {
-                if (s_logger.isInfoEnabled()) {
-                    s_logger.info("Console proxy " + proxy.getHostName() + " is started");
+                long proxyVmId = proxy.getId();
+                proxy = startProxy(proxyVmId);
+
+                if (proxy != null) {
+                    if (s_logger.isInfoEnabled()) {
+                        s_logger.info("Console proxy " + proxy.getHostName() + " is started");
+                    }
+                    SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT,
this,
+                        new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_UP,
dataCenterId, proxy.getId(), proxy, null));
                 }
             }
+        }catch (Exception e){
+           errorString = e.getMessage();
+        }finally {
+            // TODO - For now put all the alerts as creation failure. Distinguish between
creation vs start failure in future.
+            // Also add failure reason since startvm masks some of them.
+            if(proxy == null || proxy.getState() != State.Running)
+                SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT,
this,
+                    new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE,
dataCenterId, 0l, null, errorString));
         }
+
     }
 
     public boolean isZoneReady(Map<Long, ZoneHostInfo> zoneHostInfoMap, long dataCenterId)
{

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3907bc6f/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
index 8069050..c996c0b 100755
--- a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
+++ b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
@@ -506,9 +506,6 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements
Secondar
                 s_logger.debug("Unable to allocate secondary storage vm storage, remove the
secondary storage vm record from DB, secondary storage vm id: " +
                     secStorageVmId);
             }
-
-            SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this,
-                new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_CREATE_FAILURE,
dataCenterId, secStorageVmId, null, "Unable to allocate storage"));
         }
         return null;
     }
@@ -654,70 +651,83 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements
Secondar
             }
             return;
         }
-
-        boolean secStorageVmFromStoppedPool = false;
-        SecondaryStorageVmVO secStorageVm = assignSecStorageVmFromStoppedPool(dataCenterId,
role);
-        if (secStorageVm == null) {
-            if (s_logger.isInfoEnabled()) {
-                s_logger.info("No stopped secondary storage vm is available, need to allocate
a new secondary storage vm");
-            }
-
-            if (_allocLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
-                try {
-                    secStorageVm = startNew(dataCenterId, role);
-                    for (UploadVO upload : _uploadDao.listAll()) {
-                        _uploadDao.expunge(upload.getId());
-                    }
-                } finally {
-                    _allocLock.unlock();
-                }
-            } else {
+        SecondaryStorageVmVO secStorageVm = null;
+        String errorString = null;
+        try{
+            boolean secStorageVmFromStoppedPool = false;
+            secStorageVm = assignSecStorageVmFromStoppedPool(dataCenterId, role);
+            if (secStorageVm == null) {
                 if (s_logger.isInfoEnabled()) {
-                    s_logger.info("Unable to acquire synchronization lock to allocate secStorageVm
resource for standby capacity, wait for next scan");
+                    s_logger.info("No stopped secondary storage vm is available, need to
allocate a new secondary storage vm");
                 }
-                return;
-            }
-        } else {
-            if (s_logger.isInfoEnabled()) {
-                s_logger.info("Found a stopped secondary storage vm, bring it up to running
pool. secStorageVm vm id : " + secStorageVm.getId());
-            }
-            secStorageVmFromStoppedPool = true;
-        }
 
-        if (secStorageVm != null) {
-            long secStorageVmId = secStorageVm.getId();
-            GlobalLock secStorageVmLock = GlobalLock.getInternLock(getSecStorageVmLockName(secStorageVmId));
-            try {
-                if (secStorageVmLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
+                if (_allocLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
                     try {
-                        secStorageVm = startSecStorageVm(secStorageVmId);
+                        secStorageVm = startNew(dataCenterId, role);
+                        for (UploadVO upload : _uploadDao.listAll()) {
+                            _uploadDao.expunge(upload.getId());
+                        }
                     } finally {
-                        secStorageVmLock.unlock();
+                        _allocLock.unlock();
                     }
                 } else {
                     if (s_logger.isInfoEnabled()) {
-                        s_logger.info("Unable to acquire synchronization lock to start secStorageVm
for standby capacity, secStorageVm vm id : " + secStorageVm.getId());
+                        s_logger.info("Unable to acquire synchronization lock to allocate
secStorageVm resource for standby capacity, wait for next scan");
                     }
                     return;
                 }
-            } finally {
-                secStorageVmLock.releaseRef();
-            }
-
-            if (secStorageVm == null) {
+            } else {
                 if (s_logger.isInfoEnabled()) {
-                    s_logger.info("Unable to start secondary storage vm for standby capacity,
secStorageVm vm Id : " + secStorageVmId +
-                        ", will recycle it and start a new one");
+                    s_logger.info("Found a stopped secondary storage vm, bring it up to running
pool. secStorageVm vm id : " + secStorageVm.getId());
                 }
+                secStorageVmFromStoppedPool = true;
+            }
 
-                if (secStorageVmFromStoppedPool) {
-                    destroySecStorageVm(secStorageVmId);
+            if (secStorageVm != null) {
+                long secStorageVmId = secStorageVm.getId();
+                GlobalLock secStorageVmLock = GlobalLock.getInternLock(getSecStorageVmLockName(secStorageVmId));
+                try {
+                    if (secStorageVmLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
+                        try {
+                            secStorageVm = startSecStorageVm(secStorageVmId);
+                        } finally {
+                            secStorageVmLock.unlock();
+                        }
+                    } else {
+                        if (s_logger.isInfoEnabled()) {
+                            s_logger.info("Unable to acquire synchronization lock to start
secStorageVm for standby capacity, secStorageVm vm id : " + secStorageVm.getId());
+                        }
+                        return;
+                    }
+                } finally {
+                    secStorageVmLock.releaseRef();
                 }
-            } else {
-                if (s_logger.isInfoEnabled()) {
-                    s_logger.info("Secondary storage vm " + secStorageVm.getHostName() +
" is started");
+
+                if (secStorageVm == null) {
+                    if (s_logger.isInfoEnabled()) {
+                        s_logger.info("Unable to start secondary storage vm for standby capacity,
secStorageVm vm Id : " + secStorageVmId +
+                            ", will recycle it and start a new one");
+                    }
+
+                    if (secStorageVmFromStoppedPool) {
+                        destroySecStorageVm(secStorageVmId);
+                    }
+                } else {
+                    SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this,
+                            new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_UP,
dataCenterId, secStorageVmId, secStorageVm, null));
+                    if (s_logger.isInfoEnabled()) {
+                        s_logger.info("Secondary storage vm " + secStorageVm.getHostName()
+ " is started");
+                    }
                 }
             }
+        }catch (Exception e){
+            errorString = e.getMessage();
+        }finally{
+            // TODO - For now put all the alerts as creation failure. Distinguish between
creation vs start failure in future.
+            // Also add failure reason since startvm masks some of them.
+            if(secStorageVm == null || secStorageVm.getState() != State.Running)
+                SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this,
+                        new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_CREATE_FAILURE,
dataCenterId, 0l, null, errorString));
         }
     }
 


Mime
View raw message