cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject git commit: updated refs/heads/4.2 to faeddc1
Date Sun, 28 Jul 2013 12:12:18 GMT
Updated Branches:
  refs/heads/4.2 f83187585 -> faeddc15f


CLOUDSTACK-3659
Implement the download url expiration functionality for volumes
Signed off by : nitin mehta<nitin.mehta@citrix.com>


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/faeddc15
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/faeddc15
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/faeddc15

Branch: refs/heads/4.2
Commit: faeddc15ff33246ee8d59ab188743d414701e816
Parents: f831875
Author: Nitin Mehta <nitin.mehta@citrix.com>
Authored: Sun Jul 28 17:35:10 2013 +0530
Committer: Nitin Mehta <nitin.mehta@citrix.com>
Committed: Sun Jul 28 17:40:26 2013 +0530

----------------------------------------------------------------------
 .../datastore/db/VolumeDataStoreDao.java        |  2 +
 .../image/datastore/ImageStoreEntity.java       |  2 +
 .../storage/image/store/ImageStoreImpl.java     |  4 +
 .../storage/image/BaseImageStoreDriverImpl.java |  6 ++
 .../storage/image/ImageStoreDriver.java         |  4 +
 .../image/db/VolumeDataStoreDaoImpl.java        | 14 ++++
 .../driver/CloudStackImageStoreDriverImpl.java  | 21 +++++
 .../com/cloud/storage/StorageManagerImpl.java   | 82 +++++++++++++++-----
 utils/src/com/cloud/utils/DateUtil.java         | 11 +++
 9 files changed, 127 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
b/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
index 698465f..cdc2999 100644
--- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
+++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
@@ -40,4 +40,6 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
     VolumeDataStoreVO findByStoreVolume(long storeId, long volumeId, boolean lock);
 
     List<VolumeDataStoreVO> listDestroyed(long storeId);
+
+    List<VolumeDataStoreVO> listVolumeDownloadUrls();
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
b/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
index 43a0f75..cdea293 100644
--- a/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
+++ b/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
@@ -43,4 +43,6 @@ public interface ImageStoreEntity extends DataStore, ImageStore {
     String getMountPoint(); // get the mount point on ssvm.
 
     String createEntityExtractUrl(String installPath, ImageFormat format, DataObject dataObject);
 // get the entity download URL
+
+    void deleteExtractUrl(String installPath, String url, ImageFormat format);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
index 855d8cb..00ac277 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
@@ -194,5 +194,9 @@ public class ImageStoreImpl implements ImageStoreEntity {
         return driver.createEntityExtractUrl(this, installPath, format, dataObject);
     }
 
+    @Override
+    public void deleteExtractUrl(String installPath, String url, ImageFormat format) {
+        driver.deleteEntityExtractUrl(this, installPath, url, format);
+    }
 
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
index e2fc8b7..17b10f1 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
@@ -24,6 +24,7 @@ import com.cloud.agent.api.storage.Proxy;
 import com.cloud.agent.api.to.DataObjectType;
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.configuration.dao.ConfigurationDao;
+import com.cloud.storage.Storage;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.VolumeVO;
@@ -246,4 +247,9 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver
{
     @Override
     public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> callback)
{
     }
+
+    @Override
+    public void deleteEntityExtractUrl(DataStore store, String installPath, String url, Storage.ImageFormat
format){
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
b/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
index fa7ea37..2ca248f 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
@@ -25,5 +25,9 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import com.cloud.storage.Storage.ImageFormat;
 
 public interface ImageStoreDriver extends DataStoreDriver {
+
     String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format,
DataObject dataObject);
+
+    void deleteEntityExtractUrl(DataStore store, String installPath, String url, ImageFormat
format);
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
index 04f8b70..9771909 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
@@ -45,6 +45,7 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO,
Lo
     private SearchBuilder<VolumeDataStoreVO> storeSearch;
     private SearchBuilder<VolumeDataStoreVO> cacheSearch;
     private SearchBuilder<VolumeDataStoreVO> storeVolumeSearch;
+    private SearchBuilder<VolumeDataStoreVO> downloadVolumeSearch;
 
     @Override
     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException
{
@@ -77,6 +78,12 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO,
Lo
         updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ);
         updateStateSearch.and("updatedCount", updateStateSearch.entity().getUpdatedCount(),
Op.EQ);
         updateStateSearch.done();
+
+        downloadVolumeSearch = createSearchBuilder();
+        downloadVolumeSearch.and("download_url", downloadVolumeSearch.entity().getExtractUrl(),
Op.NNULL);
+        downloadVolumeSearch.and("destroyed", downloadVolumeSearch.entity().getDestroyed(),
SearchCriteria.Op.EQ);
+        downloadVolumeSearch.done();
+
         return true;
     }
 
@@ -186,4 +193,11 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO,
Lo
         sc.setParameters("destroyed", true);
         return listIncludingRemovedBy(sc);
     }
+
+    @Override
+    public List<VolumeDataStoreVO> listVolumeDownloadUrls() {
+        SearchCriteria<VolumeDataStoreVO> sc = downloadVolumeSearch.create();
+        sc.setParameters("destroyed", false);
+        return listBy(sc);
+    }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
index 6001c54..15d294c 100644
--- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
+++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
@@ -23,6 +23,10 @@ import java.util.UUID;
 import javax.inject.Inject;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
+
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
+import com.cloud.storage.Storage;
+import com.cloud.storage.Upload;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
@@ -76,6 +80,23 @@ public class CloudStackImageStoreDriverImpl extends BaseImageStoreDriverImpl
{
         return generateCopyUrl(ep.getPublicAddr(), uuid);
     }
 
+    @Override
+    public void deleteEntityExtractUrl(DataStore store, String installPath, String downloadUrl,
ImageFormat format) {
+        // find an endpoint to send command
+        EndPoint ep = _epSelector.select(store);
+        // Create Symlink at ssvm
+        //CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity)
store).getMountPoint(), installPath, uuid);
+        DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(installPath,
Upload.Type.VOLUME, downloadUrl, ((ImageStoreEntity) store).getMountPoint());
+
+        Answer ans = ep.sendMessage(cmd);
+        if (ans == null || !ans.getResult()) {
+            String errorString = "Unable to delete the url " + downloadUrl + " for path "
+ installPath + " on ssvm, " + ans.getDetails();
+            s_logger.error(errorString);
+            throw new CloudRuntimeException(errorString);
+        }
+
+    }
+
     private String generateCopyUrl(String ipAddress, String uuid){
 
         String hostname = ipAddress;

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/server/src/com/cloud/storage/StorageManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java
index 235c694..11f6540 100755
--- a/server/src/com/cloud/storage/StorageManagerImpl.java
+++ b/server/src/com/cloud/storage/StorageManagerImpl.java
@@ -41,6 +41,8 @@ import javax.ejb.Local;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
+import com.cloud.utils.DateUtil;
 import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
 import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
@@ -49,26 +51,8 @@ import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingSto
 import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd;
 import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
-import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
-import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
-import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
-import org.apache.cloudstack.engine.subsystem.api.storage.ImageStoreProvider;
-import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
-import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
-import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
-import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
+import org.apache.cloudstack.engine.subsystem.api.storage.*;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
-import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
@@ -81,6 +65,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
+import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -257,6 +242,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager,
C
     DataStoreManager _dataStoreMgr;
     @Inject
     DataStoreProviderManager _dataStoreProviderMgr;
+
     @Inject
     private TemplateService _imageSrv;
     @Inject
@@ -291,6 +277,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager,
C
     boolean _storageCleanupEnabled;
     boolean _templateCleanupEnabled = true;
     int _storageCleanupInterval;
+    int _downloadUrlCleanupInterval;
+    int _downloadUrlExpirationInterval;
     private int _createVolumeFromSnapshotWait;
     private int _copyvolumewait;
     int _storagePoolAcquisitionWaitSeconds = 1800; // 30 minutes
@@ -510,6 +498,12 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager,
C
         int wrks = NumbersUtil.parseInt(workers, 10);
         _executor = Executors.newScheduledThreadPool(wrks, new NamedThreadFactory("StorageManager-Scavenger"));
 
+        String cleanupInterval = configs.get("extract.url.cleanup.interval");
+        _downloadUrlCleanupInterval = NumbersUtil.parseInt(cleanupInterval, 7200);
+
+        String urlExpirationInterval = configs.get("extract.url.expiration.interval");
+        _downloadUrlExpirationInterval = NumbersUtil.parseInt(urlExpirationInterval, 14400);
+
         _agentMgr.registerForHostEvents(ComponentContext.inject(LocalStoragePoolListener.class),
true, false, false);
 
         String maxVolumeSizeInGbString = _configDao.getValue("storage.max.volume.size");
@@ -571,6 +565,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager,
C
         } else {
             s_logger.debug("Storage cleanup is not enabled, so the storage cleanup thread
is not being scheduled.");
         }
+
+        _executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval,
_downloadUrlCleanupInterval, TimeUnit.SECONDS);
         return true;
     }
 
@@ -1097,6 +1093,35 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager,
C
         }
     }
 
+
+    public void cleanupDownloadUrls(){
+        // find volumesOnImageStoreList with download url
+        List<VolumeDataStoreVO> volumesOnImageStoreList = _volumeStoreDao.listVolumeDownloadUrls();
+
+        for(VolumeDataStoreVO volumeOnImageStore : volumesOnImageStoreList){
+
+          try {
+            long downloadUrlCurrentAgeInSecs = DateUtil.getTimeDifference(DateUtil.now(),
volumeOnImageStore.getUpdated());
+            if(downloadUrlCurrentAgeInSecs < _downloadUrlExpirationInterval){  // URL
hasnt expired yet
+                continue;
+            }
+
+            s_logger.debug("Removing download url " + volumeOnImageStore.getExtractUrl()
+ " for volume id " + volumeOnImageStore.getVolumeId());
+
+            // Remove it from image store
+            ImageStoreEntity secStore = (ImageStoreEntity) _dataStoreMgr.getDataStore(volumeOnImageStore.getDataStoreId(),
DataStoreRole.Image);
+            secStore.deleteExtractUrl(volumeOnImageStore.getInstallPath(), volumeOnImageStore.getExtractUrl(),
ImageFormat.VHD);
+
+             // Now remove it from DB.
+             volumeOnImageStore.setExtractUrl(null);
+            _volumeStoreDao.update(volumeOnImageStore.getId(), volumeOnImageStore);
+          }catch(Throwable th){
+              s_logger.warn("caught exception while deleting download url " +volumeOnImageStore.getExtractUrl(),
th);
+          }
+        }
+
+    }
+
     @Override
     @DB
     public void cleanupSecondaryStorage(boolean recurring) {
@@ -1262,6 +1287,25 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager,
C
         }
     }
 
+
+    protected class DownloadURLGarbageCollector implements Runnable {
+
+        public DownloadURLGarbageCollector() {
+        }
+
+        @Override
+        public void run() {
+            try {
+                s_logger.trace("Download URL Garbage Collection Thread is running.");
+
+                cleanupDownloadUrls();
+
+            } catch (Exception e) {
+                s_logger.error("Caught the following Exception", e);
+            }
+        }
+    }
+
     @Override
     public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long
selfNodeId) {
         // TODO Auto-generated method stub

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/utils/src/com/cloud/utils/DateUtil.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/DateUtil.java b/utils/src/com/cloud/utils/DateUtil.java
index 1854e15..af89046 100644
--- a/utils/src/com/cloud/utils/DateUtil.java
+++ b/utils/src/com/cloud/utils/DateUtil.java
@@ -228,6 +228,17 @@ public class DateUtil {
 
     	return scheduleTime.getTime();
     }
+
+    public static long getTimeDifference(Date date1, Date date2){
+
+        Calendar dateCalendar1 = Calendar.getInstance();
+        dateCalendar1.setTime(date1);
+        Calendar dateCalendar2 = Calendar.getInstance();
+        dateCalendar2.setTime(date2);
+
+        return (dateCalendar1.getTimeInMillis() - dateCalendar2.getTimeInMillis() )/1000;
+
+    }
     
 	// test only
 	public static void main(String[] args) {


Mime
View raw message