cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mc...@apache.org
Subject git commit: updated refs/heads/object_store_migration to afcf09e
Date Tue, 29 Oct 2013 00:24:55 GMT
Updated Branches:
  refs/heads/object_store_migration 216d0343e -> afcf09eda


Various bug fixes for object store migration.

Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/afcf09ed
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/afcf09ed
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/afcf09ed

Branch: refs/heads/object_store_migration
Commit: afcf09edaa77904ba089828e507c2a4716b4a3e7
Parents: 216d034
Author: Min Chen <min.chen@citrix.com>
Authored: Mon Oct 28 17:23:59 2013 -0700
Committer: Min Chen <min.chen@citrix.com>
Committed: Mon Oct 28 17:23:59 2013 -0700

----------------------------------------------------------------------
 .../motion/AncientDataMotionStrategy.java       |  7 ++---
 .../storage/image/TemplateServiceImpl.java      | 14 ++++++++-
 .../storage/image/store/TemplateObject.java     |  8 +++++
 .../image/db/SnapshotDataStoreDaoImpl.java      | 13 ++++++--
 .../image/db/TemplateDataStoreDaoImpl.java      | 31 +++++++++++++-------
 .../image/db/VolumeDataStoreDaoImpl.java        | 13 ++++++--
 .../com/cloud/template/TemplateManagerImpl.java |  9 +++---
 7 files changed, 71 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/afcf09ed/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
----------------------------------------------------------------------
diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
index 47d087b..6149aa0 100644
--- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
+++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
@@ -51,7 +51,6 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
 import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
-import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.MigrateVolumeAnswer;
@@ -66,7 +65,6 @@ import com.cloud.host.Host;
 import com.cloud.host.dao.HostDao;
 import com.cloud.server.ManagementService;
 import com.cloud.storage.DataStoreRole;
-import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VolumeVO;
@@ -139,8 +137,9 @@ AncientDataMotionStrategy implements DataMotionStrategy {
         DataTO destTO = destData.getTO();
         DataStoreTO srcStoreTO = srcTO.getDataStore();
         DataStoreTO destStoreTO = destTO.getDataStore();
-        if (srcStoreTO instanceof NfsTO || srcStoreTO.getRole() == DataStoreRole.ImageCache
||
-                (srcStoreTO instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO)srcStoreTO).getPoolType()
== StoragePoolType.NetworkFilesystem)) {
+        if (srcStoreTO instanceof NfsTO || srcStoreTO.getRole() == DataStoreRole.ImageCache)
{
+            //||
+            //    (srcStoreTO instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO)srcStoreTO).getPoolType()
== StoragePoolType.NetworkFilesystem)) {
             return false;
         }
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/afcf09ed/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index 4121e85..ce6198d 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -616,7 +616,12 @@ public class TemplateServiceImpl implements TemplateService {
     private AsyncCallFuture<TemplateApiResult> syncToRegionStoreAsync(TemplateInfo
template, DataStore store) {
         AsyncCallFuture<TemplateApiResult> future = new AsyncCallFuture<TemplateApiResult>();
         // no need to create entry on template_store_ref here, since entries are already
created when prepareSecondaryStorageForMigration is invoked.
+        // But we need to set default install path so that sync can be done in the right
s3 path
         TemplateInfo templateOnStore = _templateFactory.getTemplate(template, store);
+        String installPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + "/"
+                + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR
+                + template.getAccountId() + "/" + template.getId() + "/" + template.getUniqueName();
+        ((TemplateObject)templateOnStore).setInstallPath(installPath);
         TemplateOpContext<TemplateApiResult> context = new TemplateOpContext<TemplateApiResult>(null,
                 (TemplateObject)templateOnStore, future);
         AsyncCallbackDispatcher<TemplateServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this);
@@ -649,11 +654,18 @@ public class TemplateServiceImpl implements TemplateService {
         return null;
     }
 
+    private boolean isRegionStore(DataStore store) {
+        if (store.getScope().getScopeType() == ScopeType.ZONE && store.getScope().getScopeId()
== null)
+            return true;
+        else
+            return false;
+    }
+
     // This routine is used to push templates currently on cache store, but not in region
store to region store.
     // used in migrating existing NFS secondary storage to S3.
     @Override
     public void syncTemplateToRegionStore(long templateId, DataStore store) {
-        if (store.getScope().getScopeType() == ScopeType.REGION) {
+        if (isRegionStore(store)) {
             // if template is on region wide object store, check if it is really downloaded
there (by checking install_path). Sync template to region
             // wide store if it is not there physically.
             TemplateInfo tmplOnStore = _templateFactory.getTemplate(templateId, store);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/afcf09ed/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
index f0675f3..0a5b608 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
@@ -58,6 +58,7 @@ public class TemplateObject implements TemplateInfo {
     private VMTemplateVO imageVO;
     private DataStore dataStore;
     private String url;
+    private String installPath; // temporarily set installPath before passing to resource
for entries with empty installPath for object store migration case
     @Inject
     VMTemplateDao imageDao;
     @Inject
@@ -293,6 +294,9 @@ public class TemplateObject implements TemplateInfo {
 
     @Override
     public String getInstallPath() {
+        if (installPath != null)
+            return installPath;
+
         if (dataStore == null) {
             return null;
         }
@@ -300,6 +304,10 @@ public class TemplateObject implements TemplateInfo {
         return obj.getInstallPath();
     }
 
+    public void setInstallPath(String installPath) {
+        this.installPath = installPath;
+    }
+
     @Override
     public long getAccountId() {
         return imageVO.getAccountId();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/afcf09ed/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
index 053c806..ee00dd5 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
@@ -258,8 +258,14 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
         if (snapshots != null) {
             s_logger.info("Duplicate " + snapshots.size() + " snapshot cache store records
to region store");
             for (SnapshotDataStoreVO snap : snapshots) {
+                SnapshotDataStoreVO snapStore = findByStoreSnapshot(DataStoreRole.Image,
storeId, snap.getSnapshotId());
+                if (snapStore != null) {
+                    s_logger.info("There is already entry for snapshot " + snap.getSnapshotId()
+ " on region store " + storeId);
+                    continue;
+                }
+                s_logger.info("Persisting an entry for snapshot " + snap.getSnapshotId()
+ " on region store " + storeId);
                 SnapshotDataStoreVO ss = new SnapshotDataStoreVO();
-                ss.setSnapshotId(snap.getId());
+                ss.setSnapshotId(snap.getSnapshotId());
                 ss.setDataStoreId(storeId);
                 ss.setRole(DataStoreRole.Image);
                 ss.setVolumeId(snap.getVolumeId());
@@ -267,8 +273,11 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
                 ss.setState(snap.getState());
                 ss.setSize(snap.getSize());
                 ss.setPhysicalSize(snap.getPhysicalSize());
-                ss.setRefCnt(snap.getRefCnt() + 1); // increase ref_cnt so that this will
not be recycled before the content is pushed to region-wide store
+                ss.setRefCnt(snap.getRefCnt());
                 persist(ss);
+                // increase ref_cnt so that this will not be recycled before the content
is pushed to region-wide store
+                snap.incrRefCnt();
+                update(snap.getId(), snap);
             }
         }
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/afcf09ed/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
index bc43e6c..ee7c4fc 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
@@ -40,6 +40,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 
 import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage.TemplateType;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.dao.VMTemplateDao;
@@ -380,10 +381,25 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         sc.setParameters("store_role", DataStoreRole.ImageCache);
         sc.setParameters("destroyed", false);
         List<TemplateDataStoreVO> tmpls = listBy(sc);
-        // create an entry for each record, but with empty install path since the content
is not yet on region-wide store yet
+        // create an entry for each template record, but with empty install path since the
content is not yet on region-wide store yet
         if (tmpls != null) {
             s_logger.info("Duplicate " + tmpls.size() + " template cache store records to
region store");
             for (TemplateDataStoreVO tmpl : tmpls) {
+                long templateId = tmpl.getTemplateId();
+                VMTemplateVO template = _tmpltDao.findById(templateId);
+                if (template == null) {
+                    throw new CloudRuntimeException("No template is found for template id:
" + templateId);
+                }
+                if (template.getTemplateType() == TemplateType.SYSTEM) {
+                    s_logger.info("No need to duplicate system template since it will be
automatically downloaded while adding region store");
+                    continue;
+                }
+                TemplateDataStoreVO tmpStore = findByStoreTemplate(storeId, tmpl.getTemplateId());
+                if (tmpStore != null) {
+                    s_logger.info("There is already entry for template " + tmpl.getTemplateId()
+ " on region store " + storeId);
+                    continue;
+                }
+                s_logger.info("Persisting an entry for template " + tmpl.getTemplateId()
+ " on region store " + storeId);
                 TemplateDataStoreVO ts = new TemplateDataStoreVO();
                 ts.setTemplateId(tmpl.getTemplateId());
                 ts.setDataStoreId(storeId);
@@ -395,17 +411,12 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
                 ts.setPhysicalSize(tmpl.getPhysicalSize());
                 ts.setErrorString(tmpl.getErrorString());
                 ts.setDownloadUrl(tmpl.getDownloadUrl());
-                ts.setRefCnt(tmpl.getRefCnt() + 1); // increase ref_cnt so that this will
not be recycled before the content is pushed to region-wide store
+                ts.setRefCnt(tmpl.getRefCnt());
                 persist(ts);
-            }
+                // increase ref_cnt of cache store entry so that this will not be recycled
before the content is pushed to region-wide store
+                tmpl.incrRefCnt();
+                this.update(tmpl.getId(), tmpl);
 
-            // mark template as cross-zones and add template_zone association
-            for (TemplateDataStoreVO tmpl : tmpls) {
-                long templateId = tmpl.getTemplateId();
-                VMTemplateVO template = _tmpltDao.findById(templateId);
-                if (template == null) {
-                    throw new CloudRuntimeException("No template is found for template id:
" + templateId);
-                }
                 // mark the template as cross-zones
                 template.setCrossZones(true);
                 _tmpltDao.update(templateId, template);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/afcf09ed/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
index 9324cd7..a3ff566 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
@@ -211,8 +211,14 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO,
Lo
         if (vols != null) {
             s_logger.info("Duplicate " + vols.size() + " volume cache store records to region
store");
             for (VolumeDataStoreVO vol : vols) {
+                VolumeDataStoreVO volStore = findByStoreVolume(storeId, vol.getVolumeId());
+                if (volStore != null) {
+                    s_logger.info("There is already entry for volume " + vol.getVolumeId()
+ " on region store " + storeId);
+                    continue;
+                }
+                s_logger.info("Persisting an entry for volume " + vol.getVolumeId() + " on
region store " + storeId);
                 VolumeDataStoreVO vs = new VolumeDataStoreVO();
-                vs.setVolumeId(vol.getId());
+                vs.setVolumeId(vol.getVolumeId());
                 vs.setDataStoreId(storeId);
                 vs.setState(vol.getState());
                 vs.setDownloadPercent(vol.getDownloadPercent());
@@ -220,8 +226,11 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO,
Lo
                 vs.setSize(vol.getSize());
                 vs.setPhysicalSize(vol.getPhysicalSize());
                 vs.setErrorString(vol.getErrorString());
-                vs.setRefCnt(vol.getRefCnt() + 1); // increase ref_cnt so that this will
not be recycled before the content is pushed to region-wide store
+                vs.setRefCnt(vol.getRefCnt());
                 persist(vs);
+                // increase ref_cnt so that this will not be recycled before the content
is pushed to region-wide store
+                vol.incrRefCnt();
+                this.update(vol.getId(), vol);
             }
         }
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/afcf09ed/server/src/com/cloud/template/TemplateManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java
index aa4473d..cc593fe 100755
--- a/server/src/com/cloud/template/TemplateManagerImpl.java
+++ b/server/src/com/cloud/template/TemplateManagerImpl.java
@@ -31,6 +31,7 @@ import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
 import org.apache.log4j.Logger;
+
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
 import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd;
 import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd;
@@ -51,7 +52,6 @@ import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd;
 import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -176,7 +176,6 @@ import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.Transaction;
-import com.cloud.utils.db.TransactionCallback;
 import com.cloud.utils.db.TransactionCallbackNoReturn;
 import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -484,9 +483,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         // Handle NFS to S3 object store migration case, we trigger template sync from NFS
to S3 during extract template or copy template
         _tmpltSvr.syncTemplateToRegionStore(templateId, tmpltStore);
 
-        DataObject templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);
+        TemplateInfo templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);
 
-        return tmpltStore.createEntityExtractUrl(tmpltStoreRef.getInstallPath(), template.getFormat(),
templateObject);
+        return tmpltStore.createEntityExtractUrl(templateObject.getInstallPath(), template.getFormat(),
templateObject);
     }
 
     public void prepareTemplateInAllStoragePools(final VMTemplateVO template, long zoneId)
{
@@ -1292,7 +1291,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         Account owner = _accountMgr.getAccount(ownerId);
         final Domain domain = _domainDao.findById(owner.getDomainId());
         if ("add".equalsIgnoreCase(operation)) {
-            final List<String> accountNamesFinal = accountNames; 
+            final List<String> accountNamesFinal = accountNames;
             Transaction.execute(new TransactionCallbackNoReturn() {
                 @Override
                 public void doInTransactionWithoutResult(TransactionStatus status) {


Mime
View raw message