Return-Path: X-Original-To: apmail-cloudstack-commits-archive@www.apache.org Delivered-To: apmail-cloudstack-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DF16EF519 for ; Fri, 29 Mar 2013 12:46:40 +0000 (UTC) Received: (qmail 63015 invoked by uid 500); 29 Mar 2013 12:46:37 -0000 Delivered-To: apmail-cloudstack-commits-archive@cloudstack.apache.org Received: (qmail 62940 invoked by uid 500); 29 Mar 2013 12:46:37 -0000 Mailing-List: contact commits-help@cloudstack.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cloudstack.apache.org Delivered-To: mailing list commits@cloudstack.apache.org Received: (qmail 61472 invoked by uid 99); 29 Mar 2013 12:46:03 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 29 Mar 2013 12:46:03 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id B1C09833D53; Fri, 29 Mar 2013 12:46:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tsp@apache.org To: commits@cloudstack.apache.org Date: Fri, 29 Mar 2013 12:46:16 -0000 Message-Id: <0b1ff47ced064bb4ab20c7218bf719e5@git.apache.org> In-Reply-To: <55b3eba091994edabc8fb3e231f210e9@git.apache.org> References: <55b3eba091994edabc8fb3e231f210e9@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [14/25] CLOUDSTACK-1156: Limit Primary and Secondary storage for domain/accounts http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/server/src/com/cloud/storage/download/DownloadMonitorImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java index 0bc89e3..7ed6f3f 100755 --- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java @@ -64,6 +64,7 @@ import com.cloud.event.UsageEventUtils; import com.cloud.event.dao.UsageEventDao; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.StorageUnavailableException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -97,7 +98,9 @@ import com.cloud.storage.template.TemplateConstants; import com.cloud.storage.template.TemplateInfo; import com.cloud.template.TemplateManager; import com.cloud.user.Account; +import com.cloud.user.AccountManager; import com.cloud.user.ResourceLimitService; +import com.cloud.utils.UriUtils; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.db.DB; import com.cloud.utils.db.JoinBuilder; @@ -177,6 +180,8 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor protected ResourceLimitService _resourceLimitMgr; @Inject protected UserVmDao _userVmDao; + @Inject + protected AccountManager _accountMgr; private Boolean _sslCopy = new Boolean(false); private String _copyAuthPasswd; @@ -289,7 +294,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor s_logger.warn("There is no secondary storage VM for secondary storage host " + destServer.getName()); return false; } - DownloadListener dl = new DownloadListener(ssAhost, destServer, template, _timer, _vmTemplateHostDao, destTmpltHost.getId(), this, dcmd, _templateDao); + DownloadListener dl = new DownloadListener(ssAhost, destServer, template, _timer, _vmTemplateHostDao, destTmpltHost.getId(), this, dcmd, _templateDao, _resourceLimitMgr, _alertMgr, _accountMgr); if (downloadJobExists) { dl.setCurrState(destTmpltHost.getDownloadState()); } @@ -376,7 +381,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor s_logger.warn("There is no secondary storage VM for secondary storage host " + sserver.getName()); return; } - DownloadListener dl = new DownloadListener(ssAhost, sserver, template, _timer, _vmTemplateHostDao, vmTemplateHost.getId(), this, dcmd, _templateDao); + DownloadListener dl = new DownloadListener(ssAhost, sserver, template, _timer, _vmTemplateHostDao, vmTemplateHost.getId(), this, dcmd, _templateDao, _resourceLimitMgr, _alertMgr, _accountMgr); if (downloadJobExists) { dl.setCurrState(vmTemplateHost.getDownloadState()); } @@ -465,7 +470,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor return; } DownloadListener dl = new DownloadListener(ssvm, sserver, volume, _timer, _volumeHostDao, volumeHost.getId(), - this, dcmd, _volumeDao, _storageMgr); + this, dcmd, _volumeDao, _storageMgr, _resourceLimitMgr, _alertMgr, _accountMgr); if (downloadJobExists) { dl.setCurrState(volumeHost.getDownloadState()); @@ -561,18 +566,21 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor else{ s_logger.warn("Failed to get size for volume" + volume.getName()); } - String eventType = EventTypes.EVENT_VOLUME_UPLOAD; + String eventType = EventTypes.EVENT_VOLUME_UPLOAD; if(volume.getAccountId() != Account.ACCOUNT_ID_SYSTEM){ UsageEventUtils.publishUsageEvent(eventType, volume.getAccountId(), host.getDataCenterId(), volume.getId(), volume.getName(), null, 0l, size, volume.getClass().getName(), volume.getUuid()); } }else if (dnldStatus == Status.DOWNLOAD_ERROR || dnldStatus == Status.ABANDONED || dnldStatus == Status.UNKNOWN){ - //Decrement the volume count - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), com.cloud.configuration.Resource.ResourceType.volume); + //Decrement the volume and secondary storage space count + _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), + com.cloud.configuration.Resource.ResourceType.volume); + _resourceLimitMgr.recalculateResourceCount(volume.getAccountId(), volume.getDomainId(), + com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal()); } - txn.commit(); + txn.commit(); } - + @Override public void handleSysTemplateDownload(HostVO host) { List hypers = _resourceMgr.listAvailHypervisorInZone(host.getId(), host.getDataCenterId()); @@ -748,6 +756,22 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor volume.setSize(volInfo.getSize()); _volumeDao.update(volumeHost.getVolumeId(), volume); } + + if (volInfo.getSize() > 0) { + try { + String url = _volumeHostDao.findByVolumeId(volume.getId()).getDownloadUrl(); + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(volume.getAccountId()), + com.cloud.configuration.Resource.ResourceType.secondary_storage, + volInfo.getSize() - UriUtils.getRemoteSize(url)); + } catch (ResourceAllocationException e) { + s_logger.warn(e.getMessage()); + _alertMgr.sendAlert(_alertMgr.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, volume.getDataCenterId(), + volume.getPodId(), e.getMessage(), e.getMessage()); + } finally { + _resourceLimitMgr.recalculateResourceCount(volume.getAccountId(), volume.getDomainId(), + com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal()); + } + } } continue; } @@ -857,6 +881,22 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor tmpltHost.setSize(tmpltInfo.getSize()); tmpltHost.setPhysicalSize(tmpltInfo.getPhysicalSize()); tmpltHost.setLastUpdated(new Date()); + + if (tmpltInfo.getSize() > 0) { + long accountId = tmplt.getAccountId(); + try { + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId), + com.cloud.configuration.Resource.ResourceType.secondary_storage, + tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl())); + } catch (ResourceAllocationException e) { + s_logger.warn(e.getMessage()); + _alertMgr.sendAlert(_alertMgr.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, ssHost.getDataCenterId(), + null, e.getMessage(), e.getMessage()); + } finally { + _resourceLimitMgr.recalculateResourceCount(accountId, _accountMgr.getAccount(accountId).getDomainId(), + com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal()); + } + } } _vmTemplateHostDao.update(tmpltHost.getId(), tmpltHost); } else { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java index bacca01..3a453d5 100755 --- a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -123,20 +123,8 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.UserVmDao; - -import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; -import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; -import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; -import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; -import org.apache.log4j.Logger; - -import javax.ejb.Local; -import javax.naming.ConfigurationException; -import java.util.*; -import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.dao.VMSnapshotDao; - @Component @Local(value = { SnapshotManager.class, SnapshotService.class }) public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, SnapshotService { @@ -209,6 +197,7 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, private int _totalRetries; private int _pauseInterval; private int _backupsnapshotwait; + Boolean backup; protected SearchBuilder PolicySnapshotSearch; protected SearchBuilder PoliciesForSnapSearch; @@ -307,16 +296,21 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); } - _resourceLimitMgr.incrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); + _resourceLimitMgr.incrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); } - - Boolean backup = Boolean.parseBoolean(this._configDao.getValue(Config.BackupSnapshotAferTakingSnapshot.toString())); if (backup) { this.backupSnapshot(snapshotId); } } catch(Exception e) { - s_logger.debug("Failed to create snapshot", e); - throw new CloudRuntimeException("Failed to create snapshot", e); + s_logger.debug("Failed to create snapshot", e); + if (backup) { + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, + new Long(volume.getSize())); + } else { + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.primary_storage, + new Long(volume.getSize())); + } + throw new CloudRuntimeException("Failed to create snapshot", e); } return snapshot; @@ -539,7 +533,9 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, snapshotCheck.getDataCenterId(), snapshotId, snapshotCheck.getName(), null, null, 0L, snapshotCheck.getClass().getName(), snapshotCheck.getUuid()); } - _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.snapshot); + _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.snapshot); + _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.secondary_storage, + new Long(snapshotCheck.getSize())); } return result; } catch (Exception e) { @@ -745,6 +741,8 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, if (_snapshotDao.expunge(snapshot.getId())) { if (snapshot.getRecurringType() == Type.MANUAL) { _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.snapshot); + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, + new Long(snapshot.getSize())); } // Log event after successful deletion @@ -995,23 +993,29 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, ClusterVO cluster = _clusterDao.findById(storagePoolVO.getClusterId()); if (cluster != null && cluster.getHypervisorType() == HypervisorType.Ovm) { - throw new InvalidParameterValueException("Ovm won't support taking snapshot"); + throw new InvalidParameterValueException("Ovm won't support taking snapshot"); } - + // Verify permissions _accountMgr.checkAccess(caller, null, true, volume); Type snapshotType = getSnapshotType(policyId); Account owner = _accountMgr.getAccount(volume.getAccountId()); + try{ - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.snapshot); - } catch (ResourceAllocationException e){ - if (snapshotType != Type.MANUAL){ - String msg = "Snapshot resource limit exceeded for account id : " + owner.getId() + ". Failed to create recurring snapshots"; - s_logger.warn(msg); - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, - "Snapshot resource limit exceeded for account id : " + owner.getId() + ". Failed to create recurring snapshots; please use updateResourceLimit to increase the limit"); - } - throw e; + _resourceLimitMgr.checkResourceLimit(owner, ResourceType.snapshot); + if (backup) { + _resourceLimitMgr.checkResourceLimit(owner, ResourceType.secondary_storage, new Long(volume.getSize())); + } else { + _resourceLimitMgr.checkResourceLimit(owner, ResourceType.primary_storage, new Long(volume.getSize())); + } + } catch (ResourceAllocationException e) { + if (snapshotType != Type.MANUAL){ + String msg = "Snapshot resource limit exceeded for account id : " + owner.getId() + ". Failed to create recurring snapshots"; + s_logger.warn(msg); + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, + "Snapshot resource limit exceeded for account id : " + owner.getId() + ". Failed to create recurring snapshots; please use updateResourceLimit to increase the limit"); + } + throw e; } // Determine the name for this snapshot @@ -1034,6 +1038,13 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, if (snapshot == null) { throw new CloudRuntimeException("Failed to create snapshot for volume: "+volumeId); } + if (backup) { + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, + new Long(volume.getSize())); + } else { + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, + new Long(volume.getSize())); + } return snapshot; } @@ -1042,6 +1053,7 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, String value = _configDao.getValue(Config.BackupSnapshotWait.toString()); _backupsnapshotwait = NumbersUtil.parseInt(value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue())); + backup = Boolean.parseBoolean(this._configDao.getValue(Config.BackupSnapshotAferTakingSnapshot.toString())); Type.HOURLY.setMax(NumbersUtil.parseInt(_configDao.getValue("snapshot.max.hourly"), HOURLYMAX)); Type.DAILY.setMax(NumbersUtil.parseInt(_configDao.getValue("snapshot.max.daily"), DAILYMAX)); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/server/src/com/cloud/template/HypervisorTemplateAdapter.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java index 1426421..a0444e5 100755 --- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java @@ -39,7 +39,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ImageDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.ImageService; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -61,20 +60,11 @@ import com.cloud.storage.VMTemplateZoneVO; import com.cloud.storage.download.DownloadMonitor; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.user.Account; +import com.cloud.utils.UriUtils; import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.UserVmVO; -import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; -import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; -import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; -import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; -import org.apache.log4j.Logger; - -import javax.ejb.Local; -import java.net.*; -import java.util.List; - @Local(value=TemplateAdapter.class) public class HypervisorTemplateAdapter extends TemplateAdapterBase implements TemplateAdapter { private final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class); @@ -134,6 +124,9 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase implements Te } profile.setUrl(validateUrl(url)); + // Check that the resource limit for secondary storage won't be exceeded + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), + ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); return profile; } @@ -160,6 +153,9 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase implements Te } profile.setUrl(validateUrl(url)); + // Check that the resource limit for secondary storage won't be exceeded + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), + ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); return profile; } @@ -183,10 +179,11 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase implements Te s_logger.debug("create template Failed", e); throw new CloudRuntimeException("create template Failed", e); } - _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template); - + _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template); + _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.secondary_storage, + UriUtils.getRemoteSize(profile.getUrl())); return template; - } + } @Override @DB public boolean delete(TemplateProfile profile) { @@ -294,8 +291,10 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase implements Te s_logger.debug("Failed to acquire lock when deleting template with ID: " + templateId); success = false; } else if (_tmpltDao.remove(templateId)) { - // Decrement the number of templates + // Decrement the number of templates and total secondary storage space used by the account _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template); + _resourceLimitMgr.recalculateResourceCount(accountId, account.getDomainId(), + ResourceType.secondary_storage.getOrdinal()); } } finally { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/server/src/com/cloud/template/TemplateManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index d843dbc..2892e00 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -893,16 +893,19 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override @DB public boolean copy(long userId, VMTemplateVO template, HostVO srcSecHost, DataCenterVO srcZone, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException { - List dstSecHosts = _ssvmMgr.listSecondaryStorageHostsInOneZone(dstZone.getId()); - long tmpltId = template.getId(); + List dstSecHosts = _ssvmMgr.listSecondaryStorageHostsInOneZone(dstZone.getId()); + long tmpltId = template.getId(); long dstZoneId = dstZone.getId(); - if (dstSecHosts == null || dstSecHosts.isEmpty() ) { - throw new StorageUnavailableException("Destination zone is not ready", DataCenter.class, dstZone.getId()); - } + if (dstSecHosts == null || dstSecHosts.isEmpty() ) { + throw new StorageUnavailableException("Destination zone is not ready", DataCenter.class, dstZone.getId()); + } AccountVO account = _accountDao.findById(template.getAccountId()); + VMTemplateHostVO srcTmpltHost = _tmpltHostDao.findByHostTemplate(srcSecHost.getId(), tmpltId); + _resourceLimitMgr.checkResourceLimit(account, ResourceType.template); - - // Event details + _resourceLimitMgr.checkResourceLimit(account, ResourceType.secondary_storage, new Long(srcTmpltHost.getSize())); + + // Event details String copyEventType; String createEventType; if (template.getFormat().equals(ImageFormat.ISO)){ @@ -912,12 +915,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, copyEventType = EventTypes.EVENT_TEMPLATE_COPY; createEventType = EventTypes.EVENT_TEMPLATE_CREATE; } - - + Transaction txn = Transaction.currentTxn(); txn.start(); - - VMTemplateHostVO srcTmpltHost = _tmpltHostDao.findByHostTemplate(srcSecHost.getId(), tmpltId); + for ( HostVO dstSecHost : dstSecHosts ) { VMTemplateHostVO dstTmpltHost = null; try { @@ -948,7 +949,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } finally { txn.commit(); } - + if(_downloadMonitor.copyTemplate(template, srcSecHost, dstSecHost) ) { _tmpltDao.addTemplateToZone(template, dstZoneId); @@ -1711,6 +1712,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, VMTemplateVO privateTemplate = null; Long accountId = null; SnapshotVO snapshot = null; + VolumeVO volume = null; try { TemplateInfo tmplInfo = this.tmplFactory.getTemplate(templateId); @@ -1725,6 +1727,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, SnapshotInfo snapInfo = this.snapshotFactory.getSnapshot(snapshotId); future = this.imageSvr.createTemplateFromSnapshotAsync(snapInfo, tmplInfo, store.get(0)); } else if (volumeId != null) { + volume = _volumeDao.findById(volumeId); VolumeInfo volInfo = this.volFactory.getVolume(volumeId); future = this.imageSvr.createTemplateFromVolumeAsync(volInfo, tmplInfo, store.get(0)); } else { @@ -1757,7 +1760,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, s_logger.debug("Failed to create template", e); throw new CloudRuntimeException("Failed to create template", e); } - + } finally { /*if (snapshot != null && snapshot.getSwiftId() != null && secondaryStorageURL != null && zoneId != null @@ -1773,8 +1776,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, // decrement resource count if (accountId != null) { - _resourceLimitMgr.decrementResourceCount(accountId, - ResourceType.template); + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template); + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, + new Long(volume != null ? volume.getSize() : snapshot.getSize())); } txn.commit(); } @@ -1854,6 +1858,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, HypervisorType hyperType; VolumeVO volume = null; + SnapshotVO snapshot = null; VMTemplateVO privateTemplate = null; if (volumeId != null) { // create template from volume volume = this._volumeDao.findById(volumeId); @@ -1879,7 +1884,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } hyperType = this._volumeDao.getHypervisorType(volumeId); } else { // create template from snapshot - SnapshotVO snapshot = _snapshotDao.findById(snapshotId); + snapshot = _snapshotDao.findById(snapshotId); if (snapshot == null) { throw new InvalidParameterValueException( "Failed to create private template record, unable to find snapshot " @@ -1911,8 +1916,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, hyperType = snapshot.getHypervisorType(); } - _resourceLimitMgr.checkResourceLimit(templateOwner, - ResourceType.template); + _resourceLimitMgr.checkResourceLimit(templateOwner, ResourceType.template); + _resourceLimitMgr.checkResourceLimit(templateOwner, ResourceType.secondary_storage, + new Long(volume != null ? volume.getSize() : snapshot.getSize())); if (!isAdmin || featured == null) { featured = Boolean.FALSE; @@ -1973,8 +1979,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, this._templateDetailsDao.persist(template.getId(), cmd.getDetails()); } - _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), - ResourceType.template); + _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.template); + _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.secondary_storage, + new Long(volume != null ? volume.getSize() : snapshot.getSize())); } if (template != null) { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/server/src/com/cloud/vm/UserVmManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 6e9cd43..24bce8b 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -3571,8 +3571,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Use DataCenterVO zone = _dcDao.findById(vm.getDataCenterId()); - // Get serviceOffering for Virtual Machine + // Get serviceOffering and Volumes for Virtual Machine ServiceOfferingVO offering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getServiceOfferingId()); + List volumes = _volsDao.findByInstance(cmd.getVmId()); //Remove vm from instance group removeInstanceFromInstanceGroup(cmd.getVmId()); @@ -3580,9 +3581,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Use // VV 2: check if account/domain is with in resource limits to create a new vm resourceLimitCheck(newAccount, new Long(offering.getCpu()), new Long(offering.getRamSize())); - // VV 3: check if volumes are with in resource limits + // VV 3: check if volumes and primary storage space are with in resource limits _resourceLimitMgr.checkResourceLimit(newAccount, ResourceType.volume, _volsDao.findByInstance(cmd.getVmId()).size()); + Long totalVolumesSize = (long) 0; + for (VolumeVO volume : volumes) { + totalVolumesSize += volume.getSize(); + } + _resourceLimitMgr.checkResourceLimit(newAccount, ResourceType.primary_storage, totalVolumesSize); // VV 4: Check if new owner can use the vm template VirtualMachineTemplate template = _templateDao.findById(vm @@ -3614,14 +3620,17 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Use _vmDao.persist(vm); // OS 2: update volume - List volumes = _volsDao.findByInstance(cmd.getVmId()); for (VolumeVO volume : volumes) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), Volume.class.getName(), volume.getUuid()); _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.volume); + _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.primary_storage, + new Long(volume.getSize())); volume.setAccountId(newAccount.getAccountId()); _volsDao.persist(volume); _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.volume); + _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.primary_storage, + new Long(volume.getSize())); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/server/test/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java ---------------------------------------------------------------------- diff --git a/server/test/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java b/server/test/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java index d311ad3..4d2fd7d 100644 --- a/server/test/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java +++ b/server/test/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java @@ -16,8 +16,6 @@ // under the License. package com.cloud.resourcelimit; -import javax.inject.Inject; - import junit.framework.TestCase; import org.apache.log4j.Logger; @@ -72,11 +70,23 @@ public class ResourceLimitManagerImplTest extends TestCase{ // update resource Limit for a domain for resource_type = 8 (CPU) resourceLimitServiceCall(null, (long) 1, 8, (long) 40); - // update resource Limit for an account for resource_type = 9 (Memory) + // update resource Limit for an account for resource_type = 9 (Memory (in MiB)) resourceLimitServiceCall((long) 1, (long) 1, 9, (long) 4096); - // update resource Limit for a domain for resource_type = 9 (Memory) + // update resource Limit for a domain for resource_type = 9 (Memory (in MiB)) resourceLimitServiceCall(null, (long) 1, 9, (long) 10240); + + // update resource Limit for an account for resource_type = 10 (Primary storage (in GiB)) + resourceLimitServiceCall((long) 1, (long) 1, 10, (long) 200); + + // update resource Limit for a domain for resource_type = 10 (Primary storage (in GiB)) + resourceLimitServiceCall(null, (long) 1, 10, (long) 200); + + // update resource Limit for an account for resource_type = 11 (Secondary storage (in GiB)) + resourceLimitServiceCall((long) 1, (long) 1, 10, (long) 400); + + // update resource Limit for a domain for resource_type = 11 (Secondary storage (in GiB)) + resourceLimitServiceCall(null, (long) 1, 10, (long) 400); } private void resourceLimitServiceCall(Long accountId, Long domainId, Integer resourceType, Long max) { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/server/test/com/cloud/vpc/MockResourceLimitManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/test/com/cloud/vpc/MockResourceLimitManagerImpl.java b/server/test/com/cloud/vpc/MockResourceLimitManagerImpl.java index b9fc861..367ca45 100644 --- a/server/test/com/cloud/vpc/MockResourceLimitManagerImpl.java +++ b/server/test/com/cloud/vpc/MockResourceLimitManagerImpl.java @@ -133,6 +133,14 @@ public class MockResourceLimitManagerImpl extends ManagerBase implements Resourc } /* (non-Javadoc) + * @see com.cloud.user.ResourceLimitService#calculateSecondaryStorageForAccount(long) + */ + public long calculateSecondaryStorageForAccount(long accountId) { + // TODO Auto-generated method stub + return 0; + } + + /* (non-Javadoc) * @see com.cloud.user.ResourceLimitService#getResourceCount(com.cloud.user.Account, com.cloud.configuration.Resource.ResourceType) */ @Override http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/setup/db/db/schema-40to410.sql ---------------------------------------------------------------------- diff --git a/setup/db/db/schema-40to410.sql b/setup/db/db/schema-40to410.sql index 60ad025..3dae1cc 100644 --- a/setup/db/db/schema-40to410.sql +++ b/setup/db/db/schema-40to410.sql @@ -263,14 +263,6 @@ CREATE TABLE `cloud`.`region` ( INSERT INTO `cloud`.`region` values ('1','Local','http://localhost:8080/client/'); -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Account Defaults', 'DEFAULT', 'management-server', 'max.account.cpus', '40', 'The default maximum number of cpu cores that can be used for an account'); - -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Account Defaults', 'DEFAULT', 'management-server', 'max.account.memory', '40960', 'The default maximum memory (in MB) that can be used for an account'); - -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Project Defaults', 'DEFAULT', 'management-server', 'max.project.cpus', '40', 'The default maximum number of cpu cores that can be used for a project'); - -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Project Defaults', 'DEFAULT', 'management-server', 'max.project.memory', '40960', 'The default maximum memory (in MB) that can be used for a project'); - ALTER TABLE `cloud_usage`.`account` ADD COLUMN `region_id` int unsigned NOT NULL DEFAULT '1'; CREATE TABLE `cloud`.`nicira_nvp_router_map` ( @@ -1293,10 +1285,6 @@ CREATE VIEW `cloud`.`account_view` AS projectcount.count projectTotal, networklimit.max networkLimit, networkcount.count networkTotal, - cpulimit.max cpuLimit, - cpucount.count cpuTotal, - memorylimit.max memoryLimit, - memorycount.count memoryTotal, async_job.id job_id, async_job.uuid job_uuid, async_job.job_status job_status, @@ -1365,18 +1353,6 @@ CREATE VIEW `cloud`.`account_view` AS `cloud`.`resource_count` networkcount ON account.id = networkcount.account_id and networkcount.type = 'network' left join - `cloud`.`resource_limit` cpulimit ON account.id = cpulimit.account_id - and cpulimit.type = 'cpu' - left join - `cloud`.`resource_count` cpucount ON account.id = cpucount.account_id - and cpucount.type = 'cpu' - left join - `cloud`.`resource_limit` memorylimit ON account.id = memorylimit.account_id - and memorylimit.type = 'memory' - left join - `cloud`.`resource_count` memorycount ON account.id = memorycount.account_id - and memorycount.type = 'memory' - left join `cloud`.`async_job` ON async_job.instance_id = account.id and async_job.instance_type = 'Account' and async_job.job_status = 0; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/setup/db/db/schema-410to420.sql ---------------------------------------------------------------------- diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index 9eed0a6..f1e24fa 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -202,7 +202,6 @@ CREATE VIEW `cloud`.`event_view` AS left join `cloud`.`event` eve ON event.start_id = eve.id; - ALTER TABLE `cloud`.`region` ADD COLUMN `gslb_service_enabled` tinyint(1) unsigned NOT NULL DEFAULT 1 COMMENT 'Is GSLB service enalbed in the Region'; ALTER TABLE `cloud`.`external_load_balancer_devices` ADD COLUMN `is_gslb_provider` int(1) unsigned NOT NULL DEFAULT 0 COMMENT '1 if load balancer appliance is acting as gslb service provider in the zone'; @@ -239,3 +238,155 @@ CREATE TABLE `cloud`.`global_load_balancer_lb_rule_map` ( CONSTRAINT `fk_gslb_rule_id` FOREIGN KEY(`gslb_rule_id`) REFERENCES `global_load_balancing_rules`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_lb_rule_id` FOREIGN KEY(`lb_rule_id`) REFERENCES `load_balancing_rules`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Account Defaults', 'DEFAULT', 'management-server', 'max.account.cpus', '40', 'The default maximum number of cpu cores that can be used for an account'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Account Defaults', 'DEFAULT', 'management-server', 'max.account.memory', '40960', 'The default maximum memory (in MiB) that can be used for an account'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Account Defaults', 'DEFAULT', 'management-server', 'max.account.primary.storage', '200', 'The default maximum primary storage space (in GiB) that can be used for an account'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Account Defaults', 'DEFAULT', 'management-server', 'max.account.secondary.storage', '400', 'The default maximum secondary storage space (in GiB) that can be used for an account'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Project Defaults', 'DEFAULT', 'management-server', 'max.project.cpus', '40', 'The default maximum number of cpu cores that can be used for a project'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Project Defaults', 'DEFAULT', 'management-server', 'max.project.memory', '40960', 'The default maximum memory (in MiB) that can be used for a project'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Project Defaults', 'DEFAULT', 'management-server', 'max.project.primary.storage', '200', 'The default maximum primary storage space (in GiB) that can be used for a project'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Project Defaults', 'DEFAULT', 'management-server', 'max.project.secondary.storage', '400', 'The default maximum secondary storage space (in GiB) that can be used for a project'); + +DROP VIEW IF EXISTS `cloud`.`account_view`; +CREATE VIEW `cloud`.`account_view` AS + select + account.id, + account.uuid, + account.account_name, + account.type, + account.state, + account.removed, + account.cleanup_needed, + account.network_domain, + domain.id domain_id, + domain.uuid domain_uuid, + domain.name domain_name, + domain.path domain_path, + data_center.id data_center_id, + data_center.uuid data_center_uuid, + data_center.name data_center_name, + account_netstats_view.bytesReceived, + account_netstats_view.bytesSent, + vmlimit.max vmLimit, + vmcount.count vmTotal, + runningvm.vmcount runningVms, + stoppedvm.vmcount stoppedVms, + iplimit.max ipLimit, + ipcount.count ipTotal, + free_ip_view.free_ip ipFree, + volumelimit.max volumeLimit, + volumecount.count volumeTotal, + snapshotlimit.max snapshotLimit, + snapshotcount.count snapshotTotal, + templatelimit.max templateLimit, + templatecount.count templateTotal, + vpclimit.max vpcLimit, + vpccount.count vpcTotal, + projectlimit.max projectLimit, + projectcount.count projectTotal, + networklimit.max networkLimit, + networkcount.count networkTotal, + cpulimit.max cpuLimit, + cpucount.count cpuTotal, + memorylimit.max memoryLimit, + memorycount.count memoryTotal, + primary_storage_limit.max primaryStorageLimit, + primary_storage_count.count primaryStorageTotal, + secondary_storage_limit.max secondaryStorageLimit, + secondary_storage_count.count secondaryStorageTotal, + async_job.id job_id, + async_job.uuid job_uuid, + async_job.job_status job_status, + async_job.account_id job_account_id + from + `cloud`.`free_ip_view`, + `cloud`.`account` + inner join + `cloud`.`domain` ON account.domain_id = domain.id + left join + `cloud`.`data_center` ON account.default_zone_id = data_center.id + left join + `cloud`.`account_netstats_view` ON account.id = account_netstats_view.account_id + left join + `cloud`.`resource_limit` vmlimit ON account.id = vmlimit.account_id + and vmlimit.type = 'user_vm' + left join + `cloud`.`resource_count` vmcount ON account.id = vmcount.account_id + and vmcount.type = 'user_vm' + left join + `cloud`.`account_vmstats_view` runningvm ON account.id = runningvm.account_id + and runningvm.state = 'Running' + left join + `cloud`.`account_vmstats_view` stoppedvm ON account.id = stoppedvm.account_id + and stoppedvm.state = 'Stopped' + left join + `cloud`.`resource_limit` iplimit ON account.id = iplimit.account_id + and iplimit.type = 'public_ip' + left join + `cloud`.`resource_count` ipcount ON account.id = ipcount.account_id + and ipcount.type = 'public_ip' + left join + `cloud`.`resource_limit` volumelimit ON account.id = volumelimit.account_id + and volumelimit.type = 'volume' + left join + `cloud`.`resource_count` volumecount ON account.id = volumecount.account_id + and volumecount.type = 'volume' + left join + `cloud`.`resource_limit` snapshotlimit ON account.id = snapshotlimit.account_id + and snapshotlimit.type = 'snapshot' + left join + `cloud`.`resource_count` snapshotcount ON account.id = snapshotcount.account_id + and snapshotcount.type = 'snapshot' + left join + `cloud`.`resource_limit` templatelimit ON account.id = templatelimit.account_id + and templatelimit.type = 'template' + left join + `cloud`.`resource_count` templatecount ON account.id = templatecount.account_id + and templatecount.type = 'template' + left join + `cloud`.`resource_limit` vpclimit ON account.id = vpclimit.account_id + and vpclimit.type = 'vpc' + left join + `cloud`.`resource_count` vpccount ON account.id = vpccount.account_id + and vpccount.type = 'vpc' + left join + `cloud`.`resource_limit` projectlimit ON account.id = projectlimit.account_id + and projectlimit.type = 'project' + left join + `cloud`.`resource_count` projectcount ON account.id = projectcount.account_id + and projectcount.type = 'project' + left join + `cloud`.`resource_limit` networklimit ON account.id = networklimit.account_id + and networklimit.type = 'network' + left join + `cloud`.`resource_count` networkcount ON account.id = networkcount.account_id + and networkcount.type = 'network' + left join + `cloud`.`resource_limit` cpulimit ON account.id = cpulimit.account_id + and cpulimit.type = 'cpu' + left join + `cloud`.`resource_count` cpucount ON account.id = cpucount.account_id + and cpucount.type = 'cpu' + left join + `cloud`.`resource_limit` memorylimit ON account.id = memorylimit.account_id + and memorylimit.type = 'memory' + left join + `cloud`.`resource_count` memorycount ON account.id = memorycount.account_id + and memorycount.type = 'memory' + left join + `cloud`.`resource_limit` primary_storage_limit ON account.id = primary_storage_limit.account_id + and primary_storage_limit.type = 'primary_storage' + left join + `cloud`.`resource_count` primary_storage_count ON account.id = primary_storage_count.account_id + and primary_storage_count.type = 'primary_storage' + left join + `cloud`.`resource_limit` secondary_storage_limit ON account.id = secondary_storage_limit.account_id + and secondary_storage_limit.type = 'secondary_storage' + left join + `cloud`.`resource_count` secondary_storage_count ON account.id = secondary_storage_count.account_id + and secondary_storage_count.type = 'secondary_storage' + left join + `cloud`.`async_job` ON async_job.instance_id = account.id + and async_job.instance_type = 'Account' + and async_job.job_status = 0; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/18695276/utils/src/com/cloud/utils/UriUtils.java ---------------------------------------------------------------------- diff --git a/utils/src/com/cloud/utils/UriUtils.java b/utils/src/com/cloud/utils/UriUtils.java index a8b5ccb..3bcee7a 100644 --- a/utils/src/com/cloud/utils/UriUtils.java +++ b/utils/src/com/cloud/utils/UriUtils.java @@ -17,10 +17,14 @@ package com.cloud.utils; import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; +import javax.net.ssl.HttpsURLConnection; + import com.cloud.utils.exception.CloudRuntimeException; public class UriUtils { @@ -78,4 +82,33 @@ public class UriUtils { // no need to do URL component encoding return url; } + + // Get the size of a file from URL response header. + public static Long getRemoteSize(String url) { + Long remoteSize = (long) 0; + HttpURLConnection httpConn = null; + HttpsURLConnection httpsConn = null; + try { + URI uri = new URI(url); + if(uri.getScheme().equalsIgnoreCase("http")) { + httpConn = (HttpURLConnection) uri.toURL().openConnection(); + remoteSize = Long.parseLong(httpConn.getHeaderField("content-length")); + } + else if(uri.getScheme().equalsIgnoreCase("https")) { + httpsConn = (HttpsURLConnection) uri.toURL().openConnection(); + remoteSize = Long.parseLong(httpsConn.getHeaderField("content-length")); + } + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid URL " + url); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to establish connection with URL " + url); + } finally { + if (httpConn != null) { + httpConn.disconnect(); + } else if (httpsConn != null) { + httpsConn.disconnect(); + } + } + return remoteSize; + } }