cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aprat...@apache.org
Subject [21/50] git commit: updated refs/heads/ldapplugin to 1f64354
Date Thu, 25 Jul 2013 11:56:59 GMT
fix create template from snapshot if it's swift


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/be3883b6
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/be3883b6
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/be3883b6

Branch: refs/heads/ldapplugin
Commit: be3883b67820126328ff5dd9098b267c2d656354
Parents: b7a4836
Author: Edison Su <sudison@gmail.com>
Authored: Fri Jul 19 18:17:37 2013 -0700
Committer: Min Chen <min.chen@citrix.com>
Committed: Wed Jul 24 09:48:07 2013 -0700

----------------------------------------------------------------------
 .../cloud/agent/api/GetStorageStatsCommand.java |  10 ++
 .../src/com/cloud/storage/JavaStorageLayer.java |  16 ++
 core/src/com/cloud/storage/StorageLayer.java    |   2 +
 .../datastore/db/SnapshotDataStoreVO.java       |   2 +-
 .../datastore/db/TemplateDataStoreVO.java       |   2 +-
 .../storage/datastore/db/VolumeDataStoreVO.java |   2 +-
 .../storage/volume/VolumeServiceImpl.java       |   1 +
 .../xen/resource/XenServerStorageProcessor.java |  25 +--
 scripts/vm/hypervisor/xenserver/vmopsSnapshot   |  28 ++--
 server/src/com/cloud/server/StatsCollector.java |   3 +-
 .../resource/NfsSecondaryStorageResource.java   | 159 ++++++++++++++++---
 .../LocalNfsSecondaryStorageResourceTest.java   |  18 ++-
 setup/db/db/schema-410to420.sql                 |   2 +-
 utils/src/com/cloud/utils/SwiftUtil.java        | 104 +++++++++++-
 14 files changed, 315 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/core/src/com/cloud/agent/api/GetStorageStatsCommand.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/agent/api/GetStorageStatsCommand.java b/core/src/com/cloud/agent/api/GetStorageStatsCommand.java
index f7ebd51..8f51e35 100755
--- a/core/src/com/cloud/agent/api/GetStorageStatsCommand.java
+++ b/core/src/com/cloud/agent/api/GetStorageStatsCommand.java
@@ -17,6 +17,7 @@
 package com.cloud.agent.api;
 
 import com.cloud.agent.api.LogLevel.Log4jLevel;
+import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.storage.Storage.StoragePoolType;
 
 @LogLevel(Log4jLevel.Trace)
@@ -25,6 +26,7 @@ public class GetStorageStatsCommand extends Command {
     private String localPath;
     private StoragePoolType pooltype;
     private String secUrl;
+    private DataStoreTO store;
 
 
     public String getSecUrl() {
@@ -46,6 +48,10 @@ public class GetStorageStatsCommand extends Command {
         this.pooltype = pooltype;
     }
 
+    public GetStorageStatsCommand(DataStoreTO store) {
+        this.store = store;
+    }
+
     public GetStorageStatsCommand(String secUrl) {
         this.secUrl = secUrl;
     }
@@ -69,6 +75,10 @@ public class GetStorageStatsCommand extends Command {
         return this.localPath;
     }
 
+    public DataStoreTO getStore() {
+        return this.store;
+    }
+
     @Override
     public boolean executeInSequence() {
         return false;

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/core/src/com/cloud/storage/JavaStorageLayer.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/JavaStorageLayer.java b/core/src/com/cloud/storage/JavaStorageLayer.java
index 525d429..bfaa767 100644
--- a/core/src/com/cloud/storage/JavaStorageLayer.java
+++ b/core/src/com/cloud/storage/JavaStorageLayer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
 import javax.ejb.Local;
 import javax.naming.ConfigurationException;
@@ -164,6 +165,21 @@ public class JavaStorageLayer implements StorageLayer {
     }
 
     @Override
+    public File createUniqDir() {
+        String dirName = System.getProperty("java.io.tmpdir");
+        if (dirName != null) {
+            File dir = new File(dirName);
+            if (dir.exists()) {
+                String uniqDirName = dir.getAbsolutePath() + File.separator + UUID.randomUUID().toString();
+                if (this.mkdir(uniqDirName)) {
+                    return new File(uniqDirName);
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
     public boolean mkdirs(String path) {
         synchronized(path.intern()) {
             File dir = new File(path);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/core/src/com/cloud/storage/StorageLayer.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/StorageLayer.java b/core/src/com/cloud/storage/StorageLayer.java
index b640191..7d8583d 100644
--- a/core/src/com/cloud/storage/StorageLayer.java
+++ b/core/src/com/cloud/storage/StorageLayer.java
@@ -39,6 +39,8 @@ public interface StorageLayer extends Manager {
      */
     long getSize(String path);
 
+    File createUniqDir();
+
     /**
      * Is this path a directory?
      * @param path path to check.

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java
b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java
index 2ae3e8c..929b2c8 100644
--- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java
+++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java
@@ -93,7 +93,7 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
     ObjectInDataStoreStateMachine.State state;
 
     @Column(name = "ref_cnt")
-    Long refCnt;
+    Long refCnt = 0L;
 
     public String getInstallPath() {
         return installPath;

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
b/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
index c6b434d..b6af559 100755
--- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
+++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
@@ -113,7 +113,7 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
     ObjectInDataStoreStateMachine.State state;
 
     @Column(name = "ref_cnt")
-    Long refCnt;
+    Long refCnt = 0L;
 
     public TemplateDataStoreVO(Long hostId, long templateId) {
         super();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
b/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
index 222447f..a5d0830 100755
--- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
+++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
@@ -112,7 +112,7 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
     ObjectInDataStoreStateMachine.State state;
 
     @Column(name = "ref_cnt")
-    Long refCnt;
+    Long refCnt = 0L;
 
     public String getInstallPath() {
         return installPath;

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
index cc73b47..559bd37 100644
--- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
+++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
@@ -395,6 +395,7 @@ public class VolumeServiceImpl implements VolumeService {
         int storagePoolMaxWaitSeconds = NumbersUtil.parseInt(
                 configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600);
         templatePoolRef = _tmpltPoolDao.acquireInLockTable(templatePoolRefId, storagePoolMaxWaitSeconds);
+
         if (templatePoolRef == null) {
             templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId());
             if (templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready )
{

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java
b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java
index f13225d..17a4856 100644
--- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java
+++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java
@@ -90,6 +90,7 @@ import java.util.UUID;
 public class XenServerStorageProcessor implements StorageProcessor {
     private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
     protected CitrixResourceBase hypervisorResource;
+    private String BaseMountPointOnHost = "/var/run/cloud_mount";
 
     public XenServerStorageProcessor(CitrixResourceBase resource) {
         this.hypervisorResource = resource;
@@ -1043,10 +1044,10 @@ public class XenServerStorageProcessor implements StorageProcessor
{
         return false;
     }
 
-    protected String deleteSnapshotBackup(Connection conn, String path, String secondaryStorageMountPath,
String backupUUID) {
+    protected String deleteSnapshotBackup(Connection conn, String localMountPoint, String
path, String secondaryStorageMountPath, String backupUUID) {
 
         // If anybody modifies the formatting below again, I'll skin them
-        String result = hypervisorResource.callHostPlugin(conn, "vmopsSnapshot", "deleteSnapshotBackup",
"backupUUID", backupUUID, "path", path, "secondaryStorageMountPath", secondaryStorageMountPath);
+        String result = hypervisorResource.callHostPlugin(conn, "vmopsSnapshot", "deleteSnapshotBackup",
"backupUUID", backupUUID, "path", path, "secondaryStorageMountPath", secondaryStorageMountPath,
"localMountPoint", localMountPoint);
 
         return result;
     }
@@ -1147,7 +1148,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
 
     }
 
-    protected String backupSnapshot(Connection conn, String primaryStorageSRUuid, String
path, String secondaryStorageMountPath, String snapshotUuid, String prevBackupUuid, Boolean
isISCSI, int wait) {
+    protected String backupSnapshot(Connection conn, String primaryStorageSRUuid, String
localMountPoint, String path, String secondaryStorageMountPath, String snapshotUuid, String
prevBackupUuid, Boolean isISCSI, int wait) {
         String backupSnapshotUuid = null;
 
         if (prevBackupUuid == null) {
@@ -1159,7 +1160,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
         String backupUuid = UUID.randomUUID().toString();
         String results = hypervisorResource.callHostPluginAsync(conn, "vmopsSnapshot", "backupSnapshot",
wait,
                 "primaryStorageSRUuid", primaryStorageSRUuid, "path", path, "secondaryStorageMountPath",
secondaryStorageMountPath,
-                "snapshotUuid", snapshotUuid, "prevBackupUuid", prevBackupUuid, "backupUuid",
backupUuid, "isISCSI", isISCSI.toString());
+                "snapshotUuid", snapshotUuid, "prevBackupUuid", prevBackupUuid, "backupUuid",
backupUuid, "isISCSI", isISCSI.toString(), "localMountPath", localMountPoint);
         String errMsg = null;
         if (results == null || results.isEmpty()) {
             errMsg = "Could not copy backupUuid: " + backupSnapshotUuid
@@ -1282,6 +1283,8 @@ public class XenServerStorageProcessor implements StorageProcessor {
             DataStoreTO destStore = destData.getDataStore();
             String folder = destPath;
             String finalPath = null;
+
+            String localMountPoint =  BaseMountPointOnHost + File.separator + UUID.nameUUIDFromBytes(secondaryStorageUrl.getBytes()).toString();
             if (fullbackup) {
                 // the first snapshot is always a full snapshot
 
@@ -1300,11 +1303,11 @@ public class XenServerStorageProcessor implements StorageProcessor
{
                     if( destStore instanceof SwiftTO) {
                         try {
                             String container = "S-" + snapshotTO.getVolume().getVolumeId().toString();
-                            snapshotBackupUuid = swiftBackupSnapshot(conn, (SwiftTO)destStore,
snapshotSr.getUuid(conn), snapshotBackupUuid, container, false, wait);
-                            String swiftPath = container + File.separator + snapshotBackupUuid;
-                            finalPath = container + File.separator + swiftPath;
+                            String destSnapshotName = swiftBackupSnapshot(conn, (SwiftTO)destStore,
snapshotSr.getUuid(conn), snapshotBackupUuid, container, false, wait);
+                            String swiftPath = container + File.separator + destSnapshotName;
+                            finalPath = swiftPath;
                         } finally {
-                            deleteSnapshotBackup(conn, folder, secondaryStorageMountPath,
snapshotBackupUuid);
+                            deleteSnapshotBackup(conn, localMountPoint, folder, secondaryStorageMountPath,
snapshotBackupUuid);
                         }
 
                     } else if (destStore instanceof S3TO) {
@@ -1312,7 +1315,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
                             backupSnapshotToS3(conn, (S3TO)destStore, snapshotSr.getUuid(conn),
snapshotBackupUuid, isISCSI, wait);
                             snapshotBackupUuid = snapshotBackupUuid + ".vhd";
                         } finally {
-                            deleteSnapshotBackup(conn, folder, secondaryStorageMountPath,
snapshotBackupUuid);
+                            deleteSnapshotBackup(conn, localMountPoint, folder, secondaryStorageMountPath,
snapshotBackupUuid);
                         }
                         finalPath = folder + File.separator + snapshotBackupUuid;
                     } else {
@@ -1334,8 +1337,8 @@ public class XenServerStorageProcessor implements StorageProcessor {
                     backupSnapshotToS3(conn, (S3TO)destStore, primaryStorageSRUuid, snapshotPaUuid,
isISCSI, wait);
                     finalPath = folder + File.separator + snapshotPaUuid;
                 } else {
-                    snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, folder
+ File.separator + UUID.nameUUIDFromBytes(secondaryStorageMountPath.getBytes())
-                             , secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI,
wait);
+                    snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, localMountPoint,
folder,
+                             secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI,
wait);
                     finalPath = folder + File.separator + snapshotBackupUuid;
                 }
             }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/scripts/vm/hypervisor/xenserver/vmopsSnapshot
----------------------------------------------------------------------
diff --git a/scripts/vm/hypervisor/xenserver/vmopsSnapshot b/scripts/vm/hypervisor/xenserver/vmopsSnapshot
index 87a5083..9673986 100755
--- a/scripts/vm/hypervisor/xenserver/vmopsSnapshot
+++ b/scripts/vm/hypervisor/xenserver/vmopsSnapshot
@@ -321,19 +321,18 @@ def umount(localDir):
     util.SMlog("Successfully unmounted " + localDir)
     return
 
-def mountSnapshotsDir(secondaryStorageMountPath, relativeDir, dcId, accountId, instanceId,
secHostId):
+def mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path):
     # The aim is to mount secondaryStorageMountPath on 
     # And create <accountId>/<instanceId> dir on it, if it doesn't exist already.
     # Assuming that secondaryStorageMountPath  exists remotely
 
     # Just mount secondaryStorageMountPath/<relativeDir>/SecondaryStorageHost/ everytime
     # Never unmount.
+    # path is like "snapshots/account/volumeId", we mount secondary_storage:/snapshots
+    relativeDir = path.split("/")[0]
+    restDir = "/".join(path.split("/")[1:])
     snapshotsDir = os.path.join(secondaryStorageMountPath, relativeDir)
 
-    # Mkdir local mount point dir, if it doesn't exist.
-    localMountPointPath = os.path.join(CLOUD_DIR, dcId)
-    localMountPointPath = os.path.join(localMountPointPath, relativeDir, secHostId)
-
     makedirs(localMountPointPath)
     # if something is not mounted already on localMountPointPath,
     # mount secondaryStorageMountPath on localMountPath
@@ -346,8 +345,7 @@ def mountSnapshotsDir(secondaryStorageMountPath, relativeDir, dcId, accountId,
i
         mount(snapshotsDir, localMountPointPath)
 
     # Create accountId/instanceId dir on localMountPointPath, if it doesn't exist
-    backupsDir = os.path.join(localMountPointPath, accountId)
-    backupsDir = os.path.join(backupsDir, instanceId)
+    backupsDir = os.path.join(localMountPointPath, restDir)
     makedirs(backupsDir)
     return backupsDir
 
@@ -485,16 +483,13 @@ def getVhdParent(session, args):
 def backupSnapshot(session, args):
     util.SMlog("Called backupSnapshot with " + str(args))
     primaryStorageSRUuid      = args['primaryStorageSRUuid']
-    dcId                      = args['dcId']
-    accountId                 = args['accountId']
-    volumeId                  = args['volumeId']
     secondaryStorageMountPath = args['secondaryStorageMountPath']
     snapshotUuid              = args['snapshotUuid']
     prevBackupUuid            = args['prevBackupUuid']
     backupUuid                = args['backupUuid']
     isISCSI                   = getIsTrueString(args['isISCSI'])
-    secHostId                 = args['secHostId']
-
+    path = args['path']
+    localMountPoint = args['localMountPoint']
     primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI)
     util.SMlog("primarySRPath: " + primarySRPath)
 
@@ -507,7 +502,7 @@ def backupSnapshot(session, args):
     # Mount secondary storage mount path on XenServer along the path
     # /var/run/sr-mount/<dcId>/snapshots/ and create <accountId>/<volumeId>
dir
     # on it.
-    backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId,
volumeId, secHostId)
+    backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path)
     util.SMlog("Backups dir " + backupsDir)
 
     # Check existence of snapshot on primary storage
@@ -538,13 +533,12 @@ def backupSnapshot(session, args):
 @echo
 def deleteSnapshotBackup(session, args):
     util.SMlog("Calling deleteSnapshotBackup with " + str(args))
-    dcId                      = args['dcId']
-    accountId                 = args['accountId']
-    volumeId                  = args['volumeId']
     secondaryStorageMountPath = args['secondaryStorageMountPath']
     backupUUID                = args['backupUUID']
+    path = args['path']
+    localMountPoint = args['localMountPoint']
 
-    backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId,
volumeId)
+    backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path)
     # chdir to the backupsDir for convenience
     chdir(backupsDir)
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/server/src/com/cloud/server/StatsCollector.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/server/StatsCollector.java b/server/src/com/cloud/server/StatsCollector.java
index 1a6e424..3be7461 100755
--- a/server/src/com/cloud/server/StatsCollector.java
+++ b/server/src/com/cloud/server/StatsCollector.java
@@ -534,7 +534,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
                     if ( store.getUri() == null ) {
                         continue;
                     }
-                    GetStorageStatsCommand command = new GetStorageStatsCommand(store.getUri());
+
+                    GetStorageStatsCommand command = new GetStorageStatsCommand(store.getTO());
                     EndPoint ssAhost = _epSelector.select(store);
                     if (ssAhost == null) {
                         s_logger.debug("There is no secondary storage VM for secondary storage
host " + store.getName());

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index 003c87a..a82714b 100755
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -164,6 +164,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements
S
     public void disconnected() {
     }
 
+    public void setInSystemVM(boolean inSystemVM) {
+        this._inSystemVM = inSystemVM;
+    }
+
     @Override
     public Answer executeRequest(Command cmd) {
         if (cmd instanceof DownloadProgressCommand) {
@@ -215,6 +219,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements
S
 
 
     protected CopyCmdAnswer postProcessing(File destFile, String downloadPath, String destPath,
DataTO srcData, DataTO destData) throws ConfigurationException {
+        if (destData.getObjectType() == DataObjectType.SNAPSHOT) {
+            SnapshotObjectTO snapshot = new SnapshotObjectTO();
+            snapshot.setPath(destPath + File.separator + destFile.getName());
+
+            CopyCmdAnswer answer = new CopyCmdAnswer(snapshot);
+            return answer;
+        }
         // do post processing to unzip the file if it is compressed
         String scriptsDir = "scripts/storage/secondary";
         String createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh");
@@ -238,7 +249,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements
S
         String extension = null;
         if (srcData.getObjectType() == DataObjectType.TEMPLATE) {
             extension = ((TemplateObjectTO) srcData).getFormat().getFileExtension();
-        } else {
+        } else if (srcData.getObjectType() == DataObjectType.VOLUME) {
             extension = ((VolumeObjectTO) srcData).getFormat().getFileExtension();
         }
 
@@ -403,11 +414,51 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
 
         if (srcData.getHypervisorType() == HypervisorType.XenServer) {
             return copySnapshotToTemplateFromNfsToNfsXenserver(cmd, srcData, srcDataStore,
destData, destDataStore);
+        } else if (srcData.getHypervisorType() == HypervisorType.KVM) {
+            File srcFile = getFile(srcData.getPath(), srcDataStore.getUrl());
+            File destFile = getFile(destData.getPath(), destDataStore.getUrl());
+            s_logger.debug("copy snapshot to template");
+            Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFile.getAbsolutePath());
+            QCOW2Processor processor = new QCOW2Processor();
+            Map<String, Object> params = new HashMap<String, Object>();
+            params.put(StorageLayer.InstanceConfigKey, _storage);
+            try {
+                processor.configure("qcow2 processor", params);
+                String destPath = destFile.getAbsolutePath();
+                String templateName = srcFile.getName();
+                FormatInfo info = processor.process(destPath, null, templateName);
+                TemplateLocation loc = new TemplateLocation(_storage, destPath);
+                loc.create(1, true, srcFile.getName());
+                loc.addFormat(info);
+                loc.save();
+                TemplateProp prop = loc.getTemplateInfo();
+                TemplateObjectTO newTemplate = new TemplateObjectTO();
+                newTemplate.setPath(destData.getPath() + File.separator + templateName);
+                newTemplate.setFormat(ImageFormat.VHD);
+                newTemplate.setSize(prop.getSize());
+                return new CopyCmdAnswer(newTemplate);
+            } catch (ConfigurationException e) {
+                s_logger.debug("Failed to create template:" + e.toString());
+                return new CopyCmdAnswer(e.toString());
+            } catch (IOException e) {
+                s_logger.debug("Failed to create template:" + e.toString());
+                return new CopyCmdAnswer(e.toString());
+            }
         }
 
         return new CopyCmdAnswer("");
     }
 
+    protected File getFile(String path, String nfsPath) {
+        String filePath = getRootDir(nfsPath) + File.separator + path;
+        File f = new File(filePath);
+        if (!f.exists()) {
+            _storage.mkdirs(filePath);
+            f = new File(filePath);
+        }
+        return f;
+    }
+
     protected Answer createTemplateFromSnapshot(CopyCommand cmd) {
         DataTO srcData = cmd.getSrcTO();
         DataTO destData = cmd.getDestTO();
@@ -422,8 +473,32 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements
S
             if (destDataStore instanceof NfsTO) {
                 return copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO) srcData,
(NfsTO) srcDataStore,
                         (TemplateObjectTO) destData, (NfsTO) destDataStore);
-            }
+            } else if (destDataStore instanceof SwiftTO) {
+                //create template on the same data store
+                CopyCmdAnswer answer = (CopyCmdAnswer)copySnapshotToTemplateFromNfsToNfs(cmd,
(SnapshotObjectTO) srcData, (NfsTO) srcDataStore,
+                        (TemplateObjectTO) destData, (NfsTO) srcDataStore);
+                if (!answer.getResult()) {
+                    return answer;
+                }
+                s_logger.debug("starting copy template to swift");
+                DataTO newTemplate = (DataTO)answer.getNewData();
+                File templateFile = getFile(newTemplate.getPath(), ((NfsTO) srcDataStore).getUrl());
+                SwiftTO swift = (SwiftTO)destDataStore;
+                String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(),
destData.getId());
+                String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName,
templateFile.getName());
+                //upload template.properties
+                File properties = new File(templateFile.getParent() + File.separator + _tmpltpp);
+                if (properties.exists()) {
+                    SwiftUtil.putObject(swift, properties, containterName, _tmpltpp);
+                }
 
+                TemplateObjectTO template = new TemplateObjectTO();
+                template.setPath(swiftPath);
+                template.setSize(templateFile.length());
+                SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData;
+                template.setFormat(snapshot.getVolume().getFormat());
+                return new CopyCmdAnswer(template);
+            }
         }
         return new CopyCmdAnswer("");
     }
@@ -581,9 +656,29 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements
S
         File file = null;
         try {
             NfsTO nfsCacheStore = (NfsTO)cacheStore;
-            String fileName = UUID.randomUUID().toString() + "." + cmd.getFormat().getFileExtension();
+            String fileName = cmd.getName() + "." + cmd.getFormat().getFileExtension();
             file = downloadFromUrlToNfs(cmd.getUrl(), nfsCacheStore, path, fileName);
-            String swiftPath = SwiftUtil.putObject(swiftTO, file, "T-" + cmd.getId());
+            String container = "T-" + cmd.getId();
+            String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null);
+
+            //put metda file
+            File uniqDir = _storage.createUniqDir();
+            String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties";
+            _storage.create(uniqDir.getAbsolutePath(), "template.properties");
+            File metaFile = new File(metaFileName);
+            FileWriter writer = new FileWriter(metaFile);
+            BufferedWriter bufferWriter = new BufferedWriter(writer);
+            bufferWriter.write("uniquename=" + cmd.getName());
+            bufferWriter.write("\n");
+            bufferWriter.write("filename=" + fileName);
+            bufferWriter.write("\n");
+            bufferWriter.write("size=" + file.length());
+            bufferWriter.close();
+            writer.close();
+
+            SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties");
+            metaFile.delete();
+            uniqDir.delete();
             String md5sum = null;
             try {
                 md5sum = DigestUtils.md5Hex(new FileInputStream(file));
@@ -1361,32 +1456,52 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
     }
 
     Map<String, TemplateProp> swiftListTemplate(SwiftTO swift) {
-        String[] containers = swiftList(swift, "", "");
+        String[] containers = SwiftUtil.list(swift, "", null);
         if (containers == null) {
             return null;
         }
         Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>();
         for (String container : containers) {
             if (container.startsWith("T-")) {
-                String ldir = _tmpltDir + "/" + UUID.randomUUID().toString();
-                createLocalDir(ldir);
-                String lFullPath = ldir + "/" + _tmpltpp;
-                swiftDownload(swift, container, _tmpltpp, lFullPath);
-                TemplateLocation loc = new TemplateLocation(_storage, ldir);
+                String[] files = SwiftUtil.list(swift, container, "template.properties");
+                if (files.length != 1) {
+                    continue;
+                }
                 try {
-                    if (!loc.load()) {
-                        s_logger.warn("Can not parse template.properties file for template
" + container);
+                    File tempFile = File.createTempFile("template", ".tmp");
+                    File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator
+ "template.properties");
+                    if (tmpFile == null) {
                         continue;
                     }
+                    FileReader fr = new FileReader(tmpFile);
+                    BufferedReader brf = new BufferedReader(fr);
+                    String line = null;
+                    String uniqName = null;
+                    Long size = null;
+                    String name = null;
+                    while ((line = brf.readLine()) != null) {
+                        if (line.startsWith("uniquename=")) {
+                            uniqName = line.split("=")[1];
+                        } else if (line.startsWith("size=")) {
+                            size = Long.parseLong(line.split("=")[1]);
+                        } else if (line.startsWith("filename=")) {
+                            name = line.split("=")[1];
+                        }
+                    }
+                    brf.close();
+                    tempFile.delete();
+                    if (uniqName != null) {
+                        TemplateProp prop = new TemplateProp(uniqName, container + File.separator
+ name, size, size, true, false);
+                        tmpltInfos.put(uniqName, prop);
+                    }
+
                 } catch (IOException e) {
-                    s_logger.warn("Unable to load template location " + ldir + " due to "
+ e.toString(), e);
+                    s_logger.debug("Failed to create templ file:" + e.toString());
+                    continue;
+                } catch (Exception e) {
+                    s_logger.debug("Failed to get properties: " + e.toString());
                     continue;
                 }
-                TemplateProp tInfo = loc.getTemplateInfo();
-                tInfo.setInstallPath(container);
-                tmpltInfos.put(tInfo.getTemplateName(), tInfo);
-                loc.purge();
-                deleteLocalDir(ldir);
             }
         }
         return tmpltInfos;
@@ -1612,7 +1727,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
     }
 
     protected GetStorageStatsAnswer execute(final GetStorageStatsCommand cmd) {
-        String rootDir = getRootDir(cmd.getSecUrl());
+        DataStoreTO store = cmd.getStore();
+        if (store instanceof S3TO || store instanceof SwiftTO) {
+            long infinity = Integer.MAX_VALUE;
+            return new GetStorageStatsAnswer(cmd, infinity, 0L);
+        }
+
+        String rootDir = getRootDir(((NfsTO) store).getUrl());
         final long usedSize = getUsedSize(rootDir);
         final long totalSize = getTotalSize(rootDir);
         if (usedSize == -1 || totalSize == -1) {

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
index 7723321..0c355ec 100644
--- a/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
+++ b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
@@ -20,6 +20,8 @@ package org.apache.cloudstack.storage.resource;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.storage.ListTemplateAnswer;
+import com.cloud.agent.api.storage.ListTemplateCommand;
 import com.cloud.agent.api.to.DataObjectType;
 import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.SwiftTO;
@@ -28,9 +30,11 @@ import com.cloud.storage.Storage;
 import com.cloud.utils.SwiftUtil;
 import junit.framework.Assert;
 import junit.framework.TestCase;
+import org.apache.cloudstack.api.command.user.tag.ListTagsCmd;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 import org.apache.cloudstack.storage.command.CopyCommand;
 import org.apache.cloudstack.storage.command.DownloadCommand;
+import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.junit.Before;
 import org.junit.Test;
@@ -39,6 +43,7 @@ import org.mockito.Mockito;
 
 import javax.naming.ConfigurationException;
 import java.util.HashMap;
+import java.util.UUID;
 
 public class LocalNfsSecondaryStorageResourceTest extends TestCase {
     LocalNfsSecondaryStorageResource resource;
@@ -46,6 +51,8 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase {
     @Override
     public void setUp() throws ConfigurationException {
         resource = new LocalNfsSecondaryStorageResource();
+        resource.setInSystemVM(true);
+
         resource.setParentPath("/mnt");
         System.setProperty("paths.script", "/Users/edison/develop/asf-master/script");
         //resource.configure("test", new HashMap<String, Object>());
@@ -59,14 +66,15 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase {
         Mockito.when(swift.getEndPoint()).thenReturn("https://objects.dreamhost.com/auth");
         Mockito.when(swift.getAccount()).thenReturn("cloudstack");
         Mockito.when(swift.getUserName()).thenReturn("images");
-        //Mockito.when(swift.getKey()).thenReturn("something");
+        Mockito.when(swift.getKey()).thenReturn("oxvELQaOD1U5_VyosGfA-wpZ7uBWEff-CUBGCM0u");
 
         Mockito.when(template.getDataStore()).thenReturn(swift);
         Mockito.when(template.getPath()).thenReturn("template/1/1/");
         Mockito.when(template.isRequiresHvm()).thenReturn(true);
         Mockito.when(template.getId()).thenReturn(1L);
         Mockito.when(template.getFormat()).thenReturn(Storage.ImageFormat.VHD);
-        Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/ttylinux_pv.vhd");
+        Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/test.bz2");
+        Mockito.when(template.getName()).thenReturn(UUID.randomUUID().toString());
         Mockito.when(template.getObjectType()).thenReturn(DataObjectType.TEMPLATE);
 
         DownloadCommand cmd = new DownloadCommand(template, 100000L);
@@ -86,5 +94,11 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase {
         CopyCmdAnswer copyCmdAnswer = (CopyCmdAnswer)resource.executeRequest(cpyCmd);
         Assert.assertTrue(copyCmdAnswer.getResult());
 
+        //list template
+        ListTemplateCommand listCmd = new ListTemplateCommand(swift);
+        ListTemplateAnswer listAnswer = (ListTemplateAnswer)resource.executeRequest(listCmd);
+
+        Assert.assertTrue(listAnswer.getTemplateInfo().size() > 0);
     }
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/setup/db/db/schema-410to420.sql
----------------------------------------------------------------------
diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql
index 7fc5ea2..e109f9b 100644
--- a/setup/db/db/schema-410to420.sql
+++ b/setup/db/db/schema-410to420.sql
@@ -161,7 +161,7 @@ CREATE TABLE  `cloud`.`template_store_ref` (
   `destroyed` tinyint(1) COMMENT 'indicates whether the template_store entry was destroyed
by the user or not',
   `is_copy` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'indicates whether this was copied ',
   `update_count` bigint unsigned,
-  `ref_cnt` bigint unsigned,
+  `ref_cnt` bigint unsigned DEFAULT 0,
   `updated` datetime, 
   PRIMARY KEY  (`id`),
 --  CONSTRAINT `fk_template_store_ref__store_id` FOREIGN KEY `fk_template_store_ref__store_id`
(`store_id`) REFERENCES `image_store` (`id`) ON DELETE CASCADE,

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/be3883b6/utils/src/com/cloud/utils/SwiftUtil.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/SwiftUtil.java b/utils/src/com/cloud/utils/SwiftUtil.java
index c01de86..85da505 100644
--- a/utils/src/com/cloud/utils/SwiftUtil.java
+++ b/utils/src/com/cloud/utils/SwiftUtil.java
@@ -24,6 +24,8 @@ import com.cloud.utils.script.Script;
 import org.apache.log4j.Logger;
 
 import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
 
 
 public class SwiftUtil {
@@ -45,8 +47,34 @@ public class SwiftUtil {
         return swiftCLI;
     }
 
-    public static String putObject(SwiftClientCfg cfg, File srcFile, String container) {
+    public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map<String,
String> metas) {
         String swiftCli = getSwiftCLIPath();
+        StringBuilder cms = new StringBuilder();
+        for(Map.Entry<String, String> entry : metas.entrySet()) {
+            cms.append(" -m ");
+            cms.append(entry.getKey());
+            cms.append(":");
+            cms.append(entry.getValue());
+            cms.append(" ");
+        }
+        Script command = new Script("/bin/bash", logger);
+        command.add("-c");
+        command.add("/usr/bin/python " + swiftCli + " -A "
+                + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName()
+ " -K "
+                + cfg.getKey() + " post " + container + " " + object + " " + cms.toString());
+        OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
+        String result = command.execute(parser);
+        if (result != null) {
+            throw new CloudRuntimeException("Failed to post meta" + result);
+        }
+        return true;
+    }
+
+    public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String
fileName) {
+        String swiftCli = getSwiftCLIPath();
+        if (fileName == null) {
+            fileName = srcFile.getName();
+        }
         String srcDirectory = srcFile.getParent();
         Script command = new Script("/bin/bash", logger);
         long size = srcFile.length();
@@ -55,12 +83,12 @@ public class SwiftUtil {
             command.add("cd " + srcDirectory
                     + ";/usr/bin/python " + swiftCli + " -A "
                     + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName()
+ " -K "
-                    + cfg.getKey() + " upload " + container + " " + srcFile.getName());
+                    + cfg.getKey() + " upload " + container + " " + fileName);
         } else {
             command.add("cd " + srcDirectory
                     + ";/usr/bin/python " + swiftCli + " -A "
                     + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName()
+ " -K "
-                    + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + "
" + srcFile.getName());
+                    + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + "
" + fileName);
         }
         OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
         String result = command.execute(parser);
@@ -71,19 +99,74 @@ public class SwiftUtil {
         if (parser.getLines() != null) {
             String[] lines = parser.getLines().split("\\n");
             for (String line : lines) {
-                if (line.contains("Errno") || line.contains("failed")) {
+                if (line.contains("Errno") || line.contains("failed") || line.contains("not
found")) {
                     throw new CloudRuntimeException("Failed to upload file: " + lines.toString());
                 }
             }
         }
+
         return container + File.separator + srcFile.getName();
     }
 
+    private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) {
+        String swiftCli = getSwiftCLIPath();
+        StringBuilder sb = new StringBuilder();
+        sb.append(" /usr/bin/python ");
+        sb.append(swiftCli);
+        sb.append(" -A ");
+        sb.append(swift.getEndPoint());
+        sb.append(" -U ");
+        sb.append(swift.getAccount());
+        sb.append(":");
+        sb.append(swift.getUserName());
+        sb.append(" -K ");
+        sb.append(swift.getKey());
+        sb.append(" ");
+        return sb;
+    }
+
+    public static String[] list(SwiftClientCfg swift, String container, String rFilename)
{
+        String swiftCli = getSwiftCLIPath();
+        Script command = new Script("/bin/bash", logger);
+        command.add("-c");
+
+        StringBuilder swiftCmdBuilder = buildSwiftCmd(swift);
+        swiftCmdBuilder.append(" list ");
+        swiftCmdBuilder.append(container);
+
+        if (rFilename != null) {
+            swiftCmdBuilder.append(" -p ");
+            swiftCmdBuilder.append(rFilename);
+        }
+
+        command.add(swiftCmdBuilder.toString());
+        OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
+        String result = command.execute(parser);
+        if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase(""))
{
+            String[] lines = parser.getLines().split("\\n");
+            return lines;
+        } else {
+            if (result != null) {
+                String errMsg = "swiftList failed , err=" + result;
+                logger.debug("Failed to list " + errMsg);
+            } else {
+                String errMsg = "swiftList failed, no lines returns";
+                logger.debug("Failed to list " + errMsg);
+            }
+        }
+        return new String[0];
+    }
+
     public static File getObject(SwiftClientCfg cfg, File destDirectory, String swiftPath)
{
         int firstIndexOfSeparator = swiftPath.indexOf(File.separator);
         String container = swiftPath.substring(0, firstIndexOfSeparator);
         String srcPath = swiftPath.substring(firstIndexOfSeparator + 1);
-        String destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
+        String destFilePath = null;
+        if (destDirectory.isDirectory()) {
+            destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
+        } else {
+            destFilePath = destDirectory.getAbsolutePath();
+        }
         String swiftCli = getSwiftCLIPath();
         Script command = new Script("/bin/bash", logger);
         command.add("-c");
@@ -109,4 +192,15 @@ public class SwiftUtil {
         }
         return new File(destFilePath);
     }
+
+    public static String getContainerName(String type, Long id) {
+        if (type.startsWith("T")) {
+            return "T-" + id;
+        } else if (type.startsWith("S")) {
+            return "S-" + id;
+        } else if (type.startsWith("V")) {
+            return "V-" + id;
+        }
+        return null;
+    }
 }


Mime
View raw message