cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhais...@apache.org
Subject [31/50] [abbrv] git commit: updated refs/heads/master to 7072d8d
Date Thu, 16 Oct 2014 13:26:03 GMT
CLOUDSTACK-6826: Improving the download url expiration where the expiration work would be handled
by the ssvm that has the corresponding symlink created on it. In case it doesnt exist, then
would be any one of the ssvm in the zone.
Also when the ssvm is destroyed all the download urls are expired to be cleaned up in the
next run by the new ssvm.

(cherry picked from commit ce908373573757372a3a09e46f835c3e269f3d02)


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/50ee9810
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/50ee9810
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/50ee9810

Branch: refs/heads/master
Commit: 50ee9810a8ff7f147f3355f0f176bbaae16e7c75
Parents: 003269e
Author: Nitin Mehta <nitin.mehta@citrix.com>
Authored: Fri Oct 10 16:28:48 2014 -0700
Committer: David Nalley <david@gnsa.us>
Committed: Mon Oct 13 00:40:03 2014 -0400

----------------------------------------------------------------------
 .../subsystem/api/storage/EndPointSelector.java |  2 ++
 .../schema/src/com/cloud/host/dao/HostDao.java  |  2 ++
 .../src/com/cloud/host/dao/HostDaoImpl.java     | 13 +++++++++
 .../datastore/db/TemplateDataStoreDao.java      |  2 ++
 .../datastore/db/VolumeDataStoreDao.java        |  2 ++
 .../endpoint/DefaultEndPointSelector.java       | 29 ++++++++++++++++++++
 .../image/db/TemplateDataStoreDaoImpl.java      | 20 ++++++++++++++
 .../image/db/VolumeDataStoreDaoImpl.java        | 21 ++++++++++++++
 .../driver/CloudStackImageStoreDriverImpl.java  |  7 +++--
 .../SecondaryStorageManagerImpl.java            |  7 ++++-
 10 files changed, 102 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java
b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java
index 4657316..4d6465b 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java
@@ -36,4 +36,6 @@ public interface EndPointSelector {
     EndPoint select(Scope scope, Long storeId);
 
     EndPoint selectHypervisorHost(Scope scope);
+
+    EndPoint select(DataStore store, String downloadUrl);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/engine/schema/src/com/cloud/host/dao/HostDao.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/com/cloud/host/dao/HostDao.java b/engine/schema/src/com/cloud/host/dao/HostDao.java
index 4d57bde..bd48482 100755
--- a/engine/schema/src/com/cloud/host/dao/HostDao.java
+++ b/engine/schema/src/com/cloud/host/dao/HostDao.java
@@ -88,4 +88,6 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status,
Stat
     List<HostVO> listByDataCenterId(long id);
 
     List<Long> listAllHosts(long zoneId);
+
+    HostVO findByPublicIp(String publicIp);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java
index 58d472e..f1567ee 100755
--- a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java
+++ b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java
@@ -84,6 +84,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements
HostDao
     protected SearchBuilder<HostVO> MsStatusSearch;
     protected SearchBuilder<HostVO> DcPrivateIpAddressSearch;
     protected SearchBuilder<HostVO> DcStorageIpAddressSearch;
+    protected SearchBuilder<HostVO> PublicIpAddressSearch;
 
     protected SearchBuilder<HostVO> GuidSearch;
     protected SearchBuilder<HostVO> DcSearch;
@@ -207,6 +208,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements
HostDao
         DcStorageIpAddressSearch.and("dc", DcStorageIpAddressSearch.entity().getDataCenterId(),
SearchCriteria.Op.EQ);
         DcStorageIpAddressSearch.done();
 
+        PublicIpAddressSearch = createSearchBuilder();
+        PublicIpAddressSearch.and("publicIpAddress", PublicIpAddressSearch.entity().getPublicIpAddress(),
SearchCriteria.Op.EQ);
+        PublicIpAddressSearch.done();
+
         GuidSearch = createSearchBuilder();
         GuidSearch.and("guid", GuidSearch.entity().getGuid(), SearchCriteria.Op.EQ);
         GuidSearch.done();
@@ -1069,6 +1074,14 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long>
implements HostDao
     }
 
     @Override
+    public HostVO findByPublicIp(String publicIp) {
+        SearchCriteria<HostVO> sc = PublicIpAddressSearch.create();
+        sc.setParameters("publicIpAddress", publicIp);
+        return findOneBy(sc);
+    }
+
+
+    @Override
     public List<HostVO> findHypervisorHostInCluster(long clusterId) {
         SearchCriteria<HostVO> sc = TypeClusterStatusSearch.create();
         sc.setParameters("type", Host.Type.Routing);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
index 13c4d31..9f8bccd 100644
--- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
+++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
@@ -78,4 +78,6 @@ public interface TemplateDataStoreDao extends GenericDao<TemplateDataStoreVO,
Lo
     List<TemplateDataStoreVO> listTemplateDownloadUrls();
 
     void removeByTemplateStore(long templateId, long imageStoreId);
+
+    void expireDnldUrlsForZone(Long dcId);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
index 0016185..625cc29 100644
--- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
+++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
@@ -44,4 +44,6 @@ public interface VolumeDataStoreDao extends GenericDao<VolumeDataStoreVO,
Long>,
     void duplicateCacheRecordsOnRegionStore(long storeId);
 
     List<VolumeDataStoreVO> listVolumeDownloadUrls();
+
+    void expireDnldUrlsForZone(Long dcId);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
index c620c40..18c8991 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
@@ -18,6 +18,8 @@
  */
 package org.apache.cloudstack.storage.endpoint;
 
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -262,6 +264,33 @@ public class DefaultEndPointSelector implements EndPointSelector {
     }
 
     @Override
+    public EndPoint select(DataStore store, String downloadUrl){
+
+        HostVO host = null;
+        try {
+            URI uri = new URI(downloadUrl);
+            String scheme = uri.getScheme();
+            String publicIp = uri.getHost();
+            // If its https then public ip will be of the form xxx-xxx-xxx-xxx.mydomain.com
+            if(scheme.equalsIgnoreCase("https")){
+                publicIp = publicIp.split("\\.")[0]; // We want xxx-xxx-xxx-xxx
+                publicIp = publicIp.replace("-","."); // We not want the IP -  xxx.xxx.xxx.xxx
+            }
+            host = hostDao.findByPublicIp(publicIp);
+            if(host != null){
+                return RemoteHostEndPoint.getHypervisorHostEndPoint(host);
+            }
+
+        } catch (URISyntaxException e) {
+            s_logger.debug("Received URISyntaxException for url" +downloadUrl);
+        }
+
+        // If ssvm doesnt exist then find any ssvm in the zone.
+        s_logger.debug("Coudn't find ssvm for url" +downloadUrl);
+        return findEndpointForImageStorage(store);
+    }
+
+    @Override
     public EndPoint select(DataStore store) {
         if (store.getRole() == DataStoreRole.Primary) {
             return findEndpointForPrimaryStorage(store);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
index a5cf6f8..dad0b6d 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.storage.image.db;
 
+import java.sql.PreparedStatement;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -64,6 +65,7 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
     private SearchBuilder<TemplateDataStoreVO> storeTemplateStateSearch;
     private SearchBuilder<TemplateDataStoreVO> storeTemplateDownloadStatusSearch;
     private SearchBuilder<TemplateDataStoreVO> downloadTemplateSearch;
+    private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update template_store_ref
set download_url_created=? where store_id in (select id from image_store where data_center_id=?)";
 
     @Inject
     private DataStoreManager _storeMgr;
@@ -511,4 +513,22 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         return listBy(sc);
     }
 
+    @Override
+    public void expireDnldUrlsForZone(Long dcId){
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        PreparedStatement pstmt = null;
+        try {
+            txn.start();
+            pstmt = txn.prepareAutoCloseStatement(EXPIRE_DOWNLOAD_URLS_FOR_ZONE);
+            pstmt.setDate(1, new java.sql.Date(-1l));// Set the time before the epoch time.
+            pstmt.setLong(2, dcId);
+            pstmt.executeUpdate();
+            txn.commit();
+        } catch (Exception e) {
+            txn.rollback();
+            s_logger.warn("Failed expiring download urls for dcId: " + dcId, e);
+        }
+
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
index 9309f6e..2e29ad7 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.storage.image.db;
 
+import java.sql.PreparedStatement;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
@@ -51,6 +52,8 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO,
Lo
     private SearchBuilder<VolumeDataStoreVO> cacheSearch;
     private SearchBuilder<VolumeDataStoreVO> storeVolumeSearch;
     private SearchBuilder<VolumeDataStoreVO> downloadVolumeSearch;
+    private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update volume_store_ref
set download_url_created=? where store_id in (select id from image_store where data_center_id=?)";
+
 
     @Inject
     DataStoreManager storeMgr;
@@ -267,4 +270,22 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO,
Lo
         sc.setParameters("destroyed", false);
         return listBy(sc);
     }
+
+    @Override
+    public void expireDnldUrlsForZone(Long dcId){
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        PreparedStatement pstmt = null;
+        try {
+            txn.start();
+            pstmt = txn.prepareAutoCloseStatement(EXPIRE_DOWNLOAD_URLS_FOR_ZONE);
+            pstmt.setDate(1, new java.sql.Date(-1l));// Set the time before the epoch time.
+            pstmt.setLong(2, dcId);
+            pstmt.executeUpdate();
+            txn.commit();
+        } catch (Exception e) {
+            txn.rollback();
+            s_logger.warn("Failed expiring download urls for dcId: " + dcId, e);
+        }
+
+    }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
index c25e99f..05222c8 100644
--- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
+++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
@@ -23,6 +23,7 @@ import java.util.UUID;
 import javax.inject.Inject;
 
 import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
+import com.cloud.host.dao.HostDao;
 import com.cloud.storage.Upload;
 import org.apache.log4j.Logger;
 
@@ -49,6 +50,8 @@ public class CloudStackImageStoreDriverImpl extends BaseImageStoreDriverImpl
{
     @Inject
     ConfigurationDao _configDao;
     @Inject
+    HostDao _hostDao;
+    @Inject
     EndPointSelector _epSelector;
 
     @Override
@@ -113,8 +116,8 @@ public class CloudStackImageStoreDriverImpl extends BaseImageStoreDriverImpl
{
 
     @Override
     public void deleteEntityExtractUrl(DataStore store, String installPath, String downloadUrl,
Upload.Type entityType) {
-        // find an endpoint to send command
-        EndPoint ep = _epSelector.select(store);
+        // find an endpoint to send command based on the ssvm on which the url was created.
+        EndPoint ep = _epSelector.select(store, downloadUrl);
 
         // Delete Symlink at ssvm. In case of volume also delete the volume.
         DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(installPath,
entityType, downloadUrl, ((ImageStoreEntity) store).getMountPoint());

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/50ee9810/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
index 9288be4..045c622 100755
--- a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
+++ b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
@@ -43,6 +43,7 @@ import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.log4j.Logger;
 
@@ -236,6 +237,8 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements
Secondar
     ImageStoreDao _imageStoreDao;
     @Inject
     TemplateDataStoreDao _tmplStoreDao;
+    @Inject
+    VolumeDataStoreDao _volumeStoreDao;
     private long _capacityScanInterval = DEFAULT_CAPACITY_SCAN_INTERVAL;
     private int _secStorageVmMtuSize;
 
@@ -987,7 +990,9 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements
Secondar
                 s_logger.debug("Removing host entry for ssvm id=" + vmId);
                 _hostDao.remove(host.getId());
             }
-
+            //Expire the download urls in the entire zone for templates and volumes.
+            _tmplStoreDao.expireDnldUrlsForZone(host.getDataCenterId());
+            _volumeStoreDao.expireDnldUrlsForZone(host.getDataCenterId());
             return true;
         } catch (ResourceUnavailableException e) {
             s_logger.warn("Unable to expunge " + ssvm, e);


Mime
View raw message