cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mc...@apache.org
Subject [3/8] git commit: updated refs/heads/object_store_migration to 9da56fe
Date Sat, 26 Oct 2013 04:18:46 GMT
Trigger syncing templates to S3 in case of copyTemplate and
extractTemplate.

Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/184e5394
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/184e5394
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/184e5394

Branch: refs/heads/object_store_migration
Commit: 184e53940ae247a744c4a6c5222c20d331af7d2e
Parents: 1b07c33
Author: Min Chen <min.chen@citrix.com>
Authored: Mon Oct 21 17:23:58 2013 -0700
Committer: Min Chen <min.chen@citrix.com>
Committed: Mon Oct 21 17:23:58 2013 -0700

----------------------------------------------------------------------
 .../api/storage/TemplateDataFactory.java        |  2 +
 .../subsystem/api/storage/TemplateService.java  |  2 +
 .../datastore/db/TemplateDataStoreDao.java      |  2 +
 .../storage/image/TemplateDataFactoryImpl.java  | 22 +++++--
 .../storage/image/TemplateServiceImpl.java      | 68 ++++++++++++++++++++
 .../image/db/TemplateDataStoreDaoImpl.java      | 12 ++++
 .../com/cloud/template/TemplateManagerImpl.java | 20 ++++--
 7 files changed, 117 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/184e5394/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
index 0b78da0..980ed31 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
@@ -28,4 +28,6 @@ public interface TemplateDataFactory {
     TemplateInfo getTemplate(long templateId, DataStoreRole storeRole);
 
     TemplateInfo getTemplate(long templateId, DataStoreRole storeRole, Long zoneId);
+
+    TemplateInfo getReadyTemplateOnCache(long templateId);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/184e5394/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
index 0d8b3c1..185d254 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
@@ -54,6 +54,8 @@ public interface TemplateService {
 
     AsyncCallFuture<TemplateApiResult> prepareTemplateOnPrimary(TemplateInfo srcTemplate,
StoragePool pool);
 
+    void syncTemplateToRegionStore(long templateId, DataStore store);
+
     void handleSysTemplateDownload(HypervisorType hostHyper, Long dcId);
 
     void handleTemplateSync(DataStore store);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/184e5394/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
index ba569c1..fc61be5 100644
--- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
+++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
@@ -64,4 +64,6 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
     List<TemplateDataStoreVO> listByTemplate(long templateId);
 
     void duplicateCacheRecordsOnRegionStore(long storeId);
+
+    TemplateDataStoreVO findReadyOnCache(long templateId);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/184e5394/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
index a67f08f..fe1c62c 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
@@ -20,6 +20,9 @@ package org.apache.cloudstack.storage.image;
 
 import javax.inject.Inject;
 
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -28,8 +31,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.image.store.TemplateObject;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
 
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.VMTemplateStoragePoolVO;
@@ -87,7 +88,7 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
         TemplateDataStoreVO tmplStore = templateStoreDao.findByTemplate(templateId, storeRole);
         DataStore store = null;
         if (tmplStore != null) {
-            store = this.storeMgr.getDataStore(tmplStore.getDataStoreId(), storeRole);
+            store = storeMgr.getDataStore(tmplStore.getDataStoreId(), storeRole);
         }
         return this.getTemplate(templateId, store);
     }
@@ -97,7 +98,7 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
         TemplateDataStoreVO tmplStore = templateStoreDao.findByTemplateZone(templateId, zoneId,
storeRole);
         DataStore store = null;
         if (tmplStore != null) {
-            store = this.storeMgr.getDataStore(tmplStore.getDataStoreId(), storeRole);
+            store = storeMgr.getDataStore(tmplStore.getDataStoreId(), storeRole);
         }
         return this.getTemplate(templateId, store);
     }
@@ -113,4 +114,17 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
         tmpObj.setUrl(origTmpl.getUrl());
         return tmpObj;
     }
+
+    @Override
+    public TemplateInfo getReadyTemplateOnCache(long templateId) {
+        TemplateDataStoreVO tmplStore = templateStoreDao.findReadyOnCache(templateId);
+        if (tmplStore != null) {
+            DataStore store = storeMgr.getDataStore(tmplStore.getDataStoreId(), DataStoreRole.ImageCache);
+            return getTemplate(templateId, store);
+        } else {
+            return null;
+        }
+
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/184e5394/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index 06cb652..163c125 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -73,6 +73,7 @@ import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage.TemplateType;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
@@ -609,6 +610,73 @@ public class TemplateServiceImpl implements TemplateService {
         return copyAsync(volume, template, store);
     }
 
+    private AsyncCallFuture<TemplateApiResult> syncToRegionStoreAsync(TemplateInfo
template, DataStore store) {
+        AsyncCallFuture<TemplateApiResult> future = new AsyncCallFuture<TemplateApiResult>();
+        // no need to create entry on template_store_ref here, since entries are already
created when prepareSecondaryStorageForMigration is invoked.
+        TemplateInfo templateOnStore = _templateFactory.getTemplate(template, store);
+        TemplateOpContext<TemplateApiResult> context = new TemplateOpContext<TemplateApiResult>(null,
+                (TemplateObject)templateOnStore, future);
+        AsyncCallbackDispatcher<TemplateServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this);
+        caller.setCallback(caller.getTarget().syncTemplateCallBack(null, null)).setContext(context);
+        _motionSrv.copyAsync(template, templateOnStore, caller);
+        return future;
+    }
+
+    protected Void syncTemplateCallBack(AsyncCallbackDispatcher<TemplateServiceImpl, CopyCommandResult>
callback,
+            TemplateOpContext<TemplateApiResult> context) {
+        TemplateInfo destTemplate = context.getTemplate();
+        CopyCommandResult result = callback.getResult();
+        AsyncCallFuture<TemplateApiResult> future = context.getFuture();
+        TemplateApiResult res = new TemplateApiResult(destTemplate);
+        try {
+            if (result.isFailed()) {
+                res.setResult(result.getResult());
+                // no change to existing template_store_ref, will try to re-sync later if
other call triggers this sync operation, like copy template
+            } else {
+                // this will update install path properly, next time it will not sync anymore.
+                destTemplate.processEvent(Event.OperationSuccessed, result.getAnswer());
+            }
+            future.complete(res);
+        } catch (Exception e) {
+            s_logger.debug("Failed to process sync template callback", e);
+            res.setResult(e.toString());
+            future.complete(res);
+        }
+
+        return null;
+    }
+
+    // This routine is used to push templates currently on cache store, but not in region
store to region store.
+    // used in migrating existing NFS secondary storage to S3.
+    @Override
+    public void syncTemplateToRegionStore(long templateId, DataStore store) {
+        if (store.getScope().getScopeType() == ScopeType.REGION) {
+            // if template is on region wide object store, check if it is really downloaded
there (by checking install_path). Sync template to region
+            // wide store if it is not there physically.
+            TemplateInfo tmplOnStore = _templateFactory.getTemplate(templateId, store);
+            if (tmplOnStore == null) {
+                throw new CloudRuntimeException("Cannot find an entry in template_store_ref
for template " + templateId + " on region store: " + store.getName());
+            }
+            if (tmplOnStore.getInstallPath() == null || tmplOnStore.getInstallPath().length()
== 0) {
+                // template is not on region store yet, sync to region store
+                TemplateInfo srcTemplate = _templateFactory.getReadyTemplateOnCache(templateId);
+                if (srcTemplate == null) {
+                    throw new CloudRuntimeException("Cannot find template " + templateId
+ "  on cache store");
+                }
+                AsyncCallFuture<TemplateApiResult> future = syncToRegionStoreAsync(srcTemplate,
store);
+                try {
+                    TemplateApiResult result = future.get();
+                    if (result.isFailed()) {
+                        throw new CloudRuntimeException("sync template from cache to region
wide store failed for image store " + store.getName() + ":"
+                                + result.getResult());
+                    }
+                } catch (Exception ex) {
+                    throw new CloudRuntimeException("sync template from cache to region wide
store failed for image store " + store.getName());
+                }
+            }
+        }
+    }
+
     @Override
     public AsyncCallFuture<TemplateApiResult> copyTemplate(TemplateInfo srcTemplate,
DataStore destStore) {
         // generate a URL from source template ssvm to download to destination data store

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/184e5394/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
index 008180f..25f3486 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
@@ -31,6 +31,7 @@ import org.springframework.stereotype.Component;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
 import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
 import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
@@ -94,6 +95,7 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         templateRoleSearch.and("template_id", templateRoleSearch.entity().getTemplateId(),
SearchCriteria.Op.EQ);
         templateRoleSearch.and("store_role", templateRoleSearch.entity().getDataStoreRole(),
SearchCriteria.Op.EQ);
         templateRoleSearch.and("destroyed", templateRoleSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
+        templateRoleSearch.and("state", templateRoleSearch.entity().getState(), SearchCriteria.Op.EQ);
         templateRoleSearch.done();
 
         updateStateSearch = this.createSearchBuilder();
@@ -323,6 +325,16 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
     }
 
     @Override
+    public TemplateDataStoreVO findReadyOnCache(long templateId) {
+        SearchCriteria<TemplateDataStoreVO> sc = templateRoleSearch.create();
+        sc.setParameters("template_id", templateId);
+        sc.setParameters("store_role", DataStoreRole.ImageCache);
+        sc.setParameters("destroyed", false);
+        sc.setParameters("state", ObjectInDataStoreStateMachine.State.Ready);
+        return findOneIncludingRemovedBy(sc);
+    }
+
+    @Override
     public List<TemplateDataStoreVO> listByTemplate(long templateId) {
         SearchCriteria<TemplateDataStoreVO> sc = templateSearch.create();
         sc.setParameters("template_id", templateId);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/184e5394/server/src/com/cloud/template/TemplateManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java
index c1ce892..452e079 100755
--- a/server/src/com/cloud/template/TemplateManagerImpl.java
+++ b/server/src/com/cloud/template/TemplateManagerImpl.java
@@ -479,6 +479,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
             throw new InvalidParameterValueException("The " + desc + " has not been downloaded
");
         }
 
+        // Handle NFS to S3 object store migration case, we trigger template sync from NFS
to S3 during extract template or copy template
+        _tmpltSvr.syncTemplateToRegionStore(templateId, tmpltStore);
+
         DataObject templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);
 
         return tmpltStore.createEntityExtractUrl(tmpltStoreRef.getInstallPath(), template.getFormat(),
templateObject);
@@ -698,7 +701,15 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
             throw new InvalidParameterValueException("Unable to find template with id");
         }
 
+        DataStore srcSecStore = getImageStore(sourceZoneId, templateId);
+        if (srcSecStore == null) {
+            throw new InvalidParameterValueException("There is no template " + templateId
+ " in zone " + sourceZoneId);
+        }
+
         if (template.isCrossZones()){
+            //TODO: we may need UI still enable CopyTemplate in case of cross zone template
to trigger sync to region store.
+            // sync template from cache store to region store if it is not there, for cases
where we are going to migrate existing NFS to S3.
+            _tmpltSvr.syncTemplateToRegionStore(templateId, srcSecStore);
             s_logger.debug("Template " + templateId + " is cross-zone, don't need to copy");
             return template;
         }
@@ -710,11 +721,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
             return template;
         }
 
-        DataStore srcSecStore = getImageStore(sourceZoneId, templateId);
-        if (srcSecStore == null) {
-            throw new InvalidParameterValueException("There is no template " + templateId
+ " in zone " + sourceZoneId);
-        }
-
         _accountMgr.checkAccess(caller, AccessType.ModifyEntry, true, template);
 
         boolean success = copy(userId, template, srcSecStore, dstZone);
@@ -781,7 +787,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         }
 
         try {
-            StoragePool pool = (StoragePool) this._dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId());
+            StoragePool pool = (StoragePool) _dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId());
             VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId());
 
             if (s_logger.isDebugEnabled()) {
@@ -1830,6 +1836,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
 
     @Inject
     public void setTemplateAdapters(List<TemplateAdapter> adapters) {
-        this._adapters = adapters;
+        _adapters = adapters;
     }
 }


Mime
View raw message