cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject git commit: updated refs/heads/4.2 to 1bfe9eb
Date Mon, 29 Jul 2013 14:18:16 GMT
Updated Branches:
  refs/heads/4.2 03176e285 -> 1bfe9eb0a


CLOUDSTACK-3892
Implement the download url expiration functionality for templates. Also persist the template
download urls after their creation
Signed off by : nitin mehta<nitin.mehta@citrix.com>


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/1bfe9eb0
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/1bfe9eb0
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/1bfe9eb0

Branch: refs/heads/4.2
Commit: 1bfe9eb0a4ed754a41293319c560c50349d0cb5f
Parents: 03176e2
Author: Nitin Mehta <nitin.mehta@citrix.com>
Authored: Mon Jul 29 19:47:45 2013 +0530
Committer: Nitin Mehta <nitin.mehta@citrix.com>
Committed: Mon Jul 29 19:47:45 2013 +0530

----------------------------------------------------------------------
 .../datastore/db/TemplateDataStoreDao.java      |  2 ++
 .../datastore/db/TemplateDataStoreVO.java       | 23 ++++++++++++++
 .../storage/datastore/db/VolumeDataStoreVO.java | 12 ++++++++
 .../image/datastore/ImageStoreEntity.java       |  2 +-
 .../storage/image/store/ImageStoreImpl.java     |  4 +--
 .../storage/image/BaseImageStoreDriverImpl.java |  2 +-
 .../storage/image/ImageStoreDriver.java         |  2 +-
 .../image/db/TemplateDataStoreDaoImpl.java      | 14 +++++++++
 .../driver/CloudStackImageStoreDriverImpl.java  |  2 +-
 .../com/cloud/storage/StorageManagerImpl.java   | 32 ++++++++++++++++++--
 .../com/cloud/storage/VolumeManagerImpl.java    |  8 ++---
 .../com/cloud/template/TemplateManagerImpl.java | 14 ++++++++-
 setup/db/db/schema-410to420.sql                 |  3 ++
 13 files changed, 107 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
b/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
index 9350751..048ce22 100644
--- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
+++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
@@ -62,4 +62,6 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
     TemplateDataStoreVO findByTemplateZone(long templateId, Long zoneId, DataStoreRole role);
 
     List<TemplateDataStoreVO> listByTemplate(long templateId);
+
+    List<TemplateDataStoreVO> listTemplateDownloadUrls();
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
b/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
index b6af559..0af3e00 100755
--- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
+++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
@@ -95,6 +95,13 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
     @Column(name = "url")
     private String downloadUrl;
 
+    @Column(name = "download_url")
+    private String extractUrl;
+
+    @Column(name = "download_url_created")
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date extractUrlCreated = null;
+
     @Column(name = "is_copy")
     private boolean isCopy = false;
 
@@ -369,4 +376,20 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
         this.refCnt--;
     }
 
+    public String getExtractUrl() {
+        return extractUrl;
+    }
+
+    public void setExtractUrl(String extractUrl) {
+        this.extractUrl = extractUrl;
+    }
+
+    public Date getExtractUrlCreated() {
+        return extractUrlCreated;
+    }
+
+    public void setExtractUrlCreated(Date extractUrlCreated) {
+        this.extractUrlCreated = extractUrlCreated;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
b/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
index 0d19d4e..a7689d2 100755
--- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
+++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
@@ -100,6 +100,10 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
     @Column(name = "download_url")
     private String extractUrl;
 
+    @Column(name = "download_url_created")
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date extractUrlCreated = null;
+
     @Column(name = "destroyed")
     boolean destroyed = false;
 
@@ -355,4 +359,12 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
     public void setExtractUrl(String extractUrl) {
         this.extractUrl = extractUrl;
     }
+
+    public Date getExtractUrlCreated() {
+        return extractUrlCreated;
+    }
+
+    public void setExtractUrlCreated(Date extractUrlCreated) {
+        this.extractUrlCreated = extractUrlCreated;
+    }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
b/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
index cdea293..7ebfd0d 100644
--- a/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
+++ b/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
@@ -44,5 +44,5 @@ public interface ImageStoreEntity extends DataStore, ImageStore {
 
     String createEntityExtractUrl(String installPath, ImageFormat format, DataObject dataObject);
 // get the entity download URL
 
-    void deleteExtractUrl(String installPath, String url, ImageFormat format);
+    void deleteExtractUrl(String installPath, String url);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
index 00ac277..7bbe324 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
@@ -195,8 +195,8 @@ public class ImageStoreImpl implements ImageStoreEntity {
     }
 
     @Override
-    public void deleteExtractUrl(String installPath, String url, ImageFormat format) {
-        driver.deleteEntityExtractUrl(this, installPath, url, format);
+    public void deleteExtractUrl(String installPath, String url) {
+        driver.deleteEntityExtractUrl(this, installPath, url);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
index 17b10f1..660b951 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
@@ -249,7 +249,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver
{
     }
 
     @Override
-    public void deleteEntityExtractUrl(DataStore store, String installPath, String url, Storage.ImageFormat
format){
+    public void deleteEntityExtractUrl(DataStore store, String installPath, String url){
     }
 
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
b/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
index 2ca248f..444a6c7 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
@@ -28,6 +28,6 @@ public interface ImageStoreDriver extends DataStoreDriver {
 
     String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format,
DataObject dataObject);
 
-    void deleteEntityExtractUrl(DataStore store, String installPath, String url, ImageFormat
format);
+    void deleteEntityExtractUrl(DataStore store, String installPath, String url);
 
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
index 7388ca9..f5140e0 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
@@ -32,6 +32,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreState
 import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
+import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -57,6 +58,7 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
     private SearchBuilder<TemplateDataStoreVO> storeTemplateSearch;
     private SearchBuilder<TemplateDataStoreVO> storeTemplateStateSearch;
     private SearchBuilder<TemplateDataStoreVO> storeTemplateDownloadStatusSearch;
+    private SearchBuilder<TemplateDataStoreVO> downloadTemplateSearch;
 
     @Inject
     private DataStoreManager _storeMgr;
@@ -126,6 +128,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         storeTemplateSearch.and("destroyed", storeTemplateSearch.entity().getDestroyed(),
SearchCriteria.Op.EQ);
         storeTemplateSearch.done();
 
+        downloadTemplateSearch = createSearchBuilder();
+        downloadTemplateSearch.and("download_url", downloadTemplateSearch.entity().getExtractUrl(),
Op.NNULL);
+        downloadTemplateSearch.and("destroyed", downloadTemplateSearch.entity().getDestroyed(),
SearchCriteria.Op.EQ);
+        downloadTemplateSearch.done();
+
         return true;
     }
 
@@ -341,4 +348,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         return null;
     }
 
+    @Override
+    public List<TemplateDataStoreVO> listTemplateDownloadUrls() {
+        SearchCriteria<TemplateDataStoreVO> sc = downloadTemplateSearch.create();
+        sc.setParameters("destroyed", false);
+        return listBy(sc);
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
index 15d294c..cfb9e2d 100644
--- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
+++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
@@ -81,7 +81,7 @@ public class CloudStackImageStoreDriverImpl extends BaseImageStoreDriverImpl
{
     }
 
     @Override
-    public void deleteEntityExtractUrl(DataStore store, String installPath, String downloadUrl,
ImageFormat format) {
+    public void deleteEntityExtractUrl(DataStore store, String installPath, String downloadUrl)
{
         // find an endpoint to send command
         EndPoint ep = _epSelector.select(store);
         // Create Symlink at ssvm

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/server/src/com/cloud/storage/StorageManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java
index db4b738..5cf2b54 100755
--- a/server/src/com/cloud/storage/StorageManagerImpl.java
+++ b/server/src/com/cloud/storage/StorageManagerImpl.java
@@ -1125,16 +1125,44 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager,
C
 
             // Remove it from image store
             ImageStoreEntity secStore = (ImageStoreEntity) _dataStoreMgr.getDataStore(volumeOnImageStore.getDataStoreId(),
DataStoreRole.Image);
-            secStore.deleteExtractUrl(volumeOnImageStore.getInstallPath(), volumeOnImageStore.getExtractUrl(),
ImageFormat.VHD);
+            secStore.deleteExtractUrl(volumeOnImageStore.getInstallPath(), volumeOnImageStore.getExtractUrl());
 
              // Now remove it from DB.
              volumeOnImageStore.setExtractUrl(null);
-            _volumeStoreDao.update(volumeOnImageStore.getId(), volumeOnImageStore);
+             volumeOnImageStore.setExtractUrlCreated(null);
+              _volumeStoreDao.update(volumeOnImageStore.getId(), volumeOnImageStore);
           }catch(Throwable th){
               s_logger.warn("caught exception while deleting download url " +volumeOnImageStore.getExtractUrl(),
th);
           }
         }
 
+        // Cleanup expired template URLs
+        List<TemplateDataStoreVO> templatesOnImageStoreList = _templateStoreDao.listTemplateDownloadUrls();
+
+        for(TemplateDataStoreVO templateOnImageStore : templatesOnImageStoreList){
+
+            try {
+                long downloadUrlCurrentAgeInSecs = DateUtil.getTimeDifference(DateUtil.now(),
templateOnImageStore.getExtractUrlCreated());
+                if(downloadUrlCurrentAgeInSecs < _downloadUrlExpirationInterval){  //
URL hasnt expired yet
+                    continue;
+                }
+
+                s_logger.debug("Removing download url " + templateOnImageStore.getExtractUrl()
+ " for template id " + templateOnImageStore.getTemplateId());
+
+                // Remove it from image store
+                ImageStoreEntity secStore = (ImageStoreEntity) _dataStoreMgr.getDataStore(templateOnImageStore.getDataStoreId(),
DataStoreRole.Image);
+                secStore.deleteExtractUrl(templateOnImageStore.getInstallPath(), templateOnImageStore.getExtractUrl());
+
+                // Now remove it from DB.
+                templateOnImageStore.setExtractUrl(null);
+                templateOnImageStore.setExtractUrlCreated(null);
+                _templateStoreDao.update(templateOnImageStore.getId(), templateOnImageStore);
+            }catch(Throwable th){
+                s_logger.warn("caught exception while deleting download url " +templateOnImageStore.getExtractUrl(),
th);
+            }
+        }
+
+
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/server/src/com/cloud/storage/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/VolumeManagerImpl.java b/server/src/com/cloud/storage/VolumeManagerImpl.java
index b2623b9..580578f 100644
--- a/server/src/com/cloud/storage/VolumeManagerImpl.java
+++ b/server/src/com/cloud/storage/VolumeManagerImpl.java
@@ -31,6 +31,9 @@ import java.util.concurrent.ExecutionException;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.utils.*;
+import org.apache.commons.lang.StringUtils;
+
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -156,10 +159,6 @@ import com.cloud.user.dao.AccountDao;
 import com.cloud.user.dao.UserDao;
 import com.cloud.user.dao.VmDiskStatisticsDao;
 import com.cloud.uservm.UserVm;
-import com.cloud.utils.EnumUtils;
-import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.Pair;
-import com.cloud.utils.UriUtils;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.Transaction;
@@ -2822,6 +2821,7 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager
{
         String extractUrl = secStore.createEntityExtractUrl(vol.getPath(), vol.getFormat(),
vol);
         volumeStoreRef = _volumeStoreDao.findByVolume(volumeId);
         volumeStoreRef.setExtractUrl(extractUrl);
+        volumeStoreRef.setExtractUrlCreated(DateUtil.now());
         _volumeStoreDao.update(volumeStoreRef.getId(), volumeStoreRef);
 
         return extractUrl;

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/server/src/com/cloud/template/TemplateManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java
index 12a8fb5..8ccc567 100755
--- a/server/src/com/cloud/template/TemplateManagerImpl.java
+++ b/server/src/com/cloud/template/TemplateManagerImpl.java
@@ -29,6 +29,7 @@ import javax.ejb.Local;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.utils.DateUtil;
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
 import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd;
 import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd;
@@ -474,7 +475,18 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
 
         DataObject templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);
 
-        return tmpltStore.createEntityExtractUrl(tmpltStoreRef.getInstallPath(), template.getFormat(),
templateObject);
+        /*
+        +        String extractUrl = secStore.createEntityExtractUrl(vol.getPath(), vol.getFormat());
++        volumeStoreRef = _volumeStoreDao.findByVolume(volumeId);
++        volumeStoreRef.setExtractUrl(extractUrl);
++        _volumeStoreDao.update(volumeStoreRef.getId(), volumeStoreRef);
+         */
+
+        String extractUrl = tmpltStore.createEntityExtractUrl(tmpltStoreRef.getInstallPath(),
template.getFormat(), templateObject);
+        tmpltStoreRef.setExtractUrl(extractUrl);
+        tmpltStoreRef.setExtractUrlCreated(DateUtil.now());
+        _tmplStoreDao.update(tmpltStoreRef.getId(), tmpltStoreRef);
+        return extractUrl;
     }
 
     public void prepareTemplateInAllStoragePools(final VMTemplateVO template, long zoneId)
{

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1bfe9eb0/setup/db/db/schema-410to420.sql
----------------------------------------------------------------------
diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql
index 7377cbd..15044a5 100644
--- a/setup/db/db/schema-410to420.sql
+++ b/setup/db/db/schema-410to420.sql
@@ -161,6 +161,8 @@ CREATE TABLE  `cloud`.`template_store_ref` (
   `local_path` varchar(255),
   `install_path` varchar(255),
   `url` varchar(255),
+  `download_url` varchar(255),
+  `download_url_created` datetime, 
   `state` varchar(255) NOT NULL,
   `destroyed` tinyint(1) COMMENT 'indicates whether the template_store entry was destroyed
by the user or not',
   `is_copy` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'indicates whether this was copied ',
@@ -227,6 +229,7 @@ CREATE TABLE  `cloud`.`volume_store_ref` (
   `install_path` varchar(255),
   `url` varchar(255),
   `download_url` varchar(255),
+  `download_url_created` datetime, 
   `state` varchar(255) NOT NULL,  
   `destroyed` tinyint(1) COMMENT 'indicates whether the volume_host entry was destroyed by
the user or not',
   `update_count` bigint unsigned,


Mime
View raw message