incubator-cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edi...@apache.org
Subject [2/3] Backs NFS-based secondary storage with an S3-compatible object store. Periodically, a reaper thread synchronizes templates and ISOs stored on a NFS secondary storage mount with a configured S3 object store. It also pushes snapshots to the object st
Date Fri, 14 Dec 2012 07:18:48 GMT
http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
index 155210d..d8fdc3a 100755
--- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
+++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
@@ -16,10 +16,20 @@
 // under the License.
 package com.cloud.storage.resource;
 
+import static com.cloud.utils.S3Utils.deleteDirectory;
+import static com.cloud.utils.S3Utils.getDirectory;
+import static com.cloud.utils.S3Utils.putDirectory;
+import static com.cloud.utils.StringUtils.join;
+import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock;
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+import static org.apache.commons.lang.StringUtils.substringAfterLast;
+
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileWriter;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigInteger;
@@ -32,6 +42,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 
 import javax.naming.ConfigurationException;
 
@@ -46,6 +57,9 @@ import com.cloud.agent.api.ComputeChecksumCommand;
 import com.cloud.agent.api.DeleteObjectFromSwiftCommand;
 import com.cloud.agent.api.DeleteSnapshotBackupCommand;
 import com.cloud.agent.api.DeleteSnapshotsDirCommand;
+import com.cloud.agent.api.DeleteTemplateFromS3Command;
+import com.cloud.agent.api.DownloadSnapshotFromS3Command;
+import com.cloud.agent.api.DownloadTemplateFromS3ToSecondaryStorageCommand;
 import com.cloud.agent.api.GetStorageStatsAnswer;
 import com.cloud.agent.api.GetStorageStatsCommand;
 import com.cloud.agent.api.PingCommand;
@@ -60,6 +74,8 @@ import com.cloud.agent.api.SecStorageSetupCommand.Certificates;
 import com.cloud.agent.api.StartupSecondaryStorageCommand;
 import com.cloud.agent.api.SecStorageVMSetupCommand;
 import com.cloud.agent.api.StartupCommand;
+import com.cloud.agent.api.StartupSecondaryStorageCommand;
+import com.cloud.agent.api.UploadTemplateToS3FromSecondaryStorageCommand;
 import com.cloud.agent.api.downloadSnapshotFromSwiftCommand;
 import com.cloud.agent.api.downloadTemplateFromSwiftToSecondaryStorageCommand;
 import com.cloud.agent.api.uploadTemplateToSwiftFromSecondaryStorageCommand;
@@ -75,6 +91,7 @@ import com.cloud.agent.api.storage.ListVolumeAnswer;
 import com.cloud.agent.api.storage.ListVolumeCommand;
 import com.cloud.agent.api.storage.UploadCommand;
 import com.cloud.agent.api.storage.ssCommand;
+import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.SwiftTO;
 import com.cloud.api.commands.DeleteVolumeCmd;
 import com.cloud.exception.InternalErrorException;
@@ -90,6 +107,9 @@ import com.cloud.storage.template.TemplateLocation;
 import com.cloud.storage.template.UploadManager;
 import com.cloud.storage.template.UploadManagerImpl;
 import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.S3Utils;
+import com.cloud.utils.S3Utils.FileNamingStrategy;
+import com.cloud.utils.S3Utils.ObjectNamingStrategy;
 import com.cloud.utils.component.ComponentLocator;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.NetUtils;
@@ -97,8 +117,15 @@ import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
 import com.cloud.vm.SecondaryStorageVm;
 
-public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
-    private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
+public class NfsSecondaryStorageResource extends ServerResourceBase implements
+        SecondaryStorageResource {
+
+    private static final Logger s_logger = Logger
+            .getLogger(NfsSecondaryStorageResource.class);
+
+    private static final String TEMPLATE_ROOT_DIR = "template/tmpl";
+    private static final String SNAPSHOT_ROOT_DIR = "snapshots";
+
     int _timeout;
     
     String _instance;  
@@ -168,16 +195,24 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
             return execute((ListVolumeCommand)cmd);
         }else if (cmd instanceof downloadSnapshotFromSwiftCommand){
             return execute((downloadSnapshotFromSwiftCommand)cmd);
+        } else if (cmd instanceof DownloadSnapshotFromS3Command) {
+            return execute((DownloadSnapshotFromS3Command) cmd);
         } else if (cmd instanceof DeleteSnapshotBackupCommand){
             return execute((DeleteSnapshotBackupCommand)cmd);
         } else if (cmd instanceof DeleteSnapshotsDirCommand){
             return execute((DeleteSnapshotsDirCommand)cmd);
         } else if (cmd instanceof downloadTemplateFromSwiftToSecondaryStorageCommand) {
             return execute((downloadTemplateFromSwiftToSecondaryStorageCommand) cmd);
+        } else if (cmd instanceof DownloadTemplateFromS3ToSecondaryStorageCommand) {
+            return execute((DownloadTemplateFromS3ToSecondaryStorageCommand) cmd);
         } else if (cmd instanceof uploadTemplateToSwiftFromSecondaryStorageCommand) {
             return execute((uploadTemplateToSwiftFromSecondaryStorageCommand) cmd);
+        } else if (cmd instanceof UploadTemplateToS3FromSecondaryStorageCommand) {
+            return execute((UploadTemplateToS3FromSecondaryStorageCommand) cmd);
         } else if (cmd instanceof DeleteObjectFromSwiftCommand) {
             return execute((DeleteObjectFromSwiftCommand) cmd);
+        } else if (cmd instanceof DeleteTemplateFromS3Command) {
+            return execute((DeleteTemplateFromS3Command) cmd);
         } else if (cmd instanceof CleanupSnapshotBackupCommand){
             return execute((CleanupSnapshotBackupCommand)cmd);
         } else {
@@ -185,6 +220,69 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
     
+    @SuppressWarnings("unchecked")
+    private String determineS3TemplateDirectory(final Long accountId,
+            final Long templateId) {
+        return join(asList(TEMPLATE_ROOT_DIR, accountId, templateId),
+                S3Utils.SEPARATOR);
+    }
+
+    @SuppressWarnings("unchecked")
+    private String determineStorageTemplatePath(final String storagePath,
+            final Long accountId, final Long templateId) {
+        return join(
+                asList(getRootDir(storagePath), TEMPLATE_ROOT_DIR, accountId,
+                        templateId), File.separator);
+    }
+
+    private Answer execute(
+            final DownloadTemplateFromS3ToSecondaryStorageCommand cmd) {
+
+        final S3TO s3 = cmd.getS3();
+        final String storagePath = cmd.getStoragePath();
+        final Long accountId = cmd.getAccountId();
+        final Long templateId = cmd.getTemplateId();
+
+        try {
+
+            final File downloadDirectory = _storage
+                    .getFile(determineStorageTemplatePath(storagePath,
+                            accountId, templateId));
+            downloadDirectory.mkdirs();
+
+            if (!downloadDirectory.exists()) {
+                final String errMsg = format(
+                        "Unable to create directory "
+                                + "download directory %1$s for download of template id "
+                                + "%2$s from S3.", downloadDirectory.getName(),
+                        templateId);
+                s_logger.error(errMsg);
+                return new Answer(cmd, false, errMsg);
+            }
+
+            getDirectory(s3, s3.getBucketName(),
+                    determineS3TemplateDirectory(accountId, templateId),
+                    downloadDirectory, new FileNamingStrategy() {
+                        @Override
+                        public String determineFileName(final String key) {
+                            return substringAfterLast(key, S3Utils.SEPARATOR);
+                        }
+                    });
+
+            return new Answer(cmd, true, format("Successfully downloaded "
+                    + "template id %1$s from S3 to directory %2$s", templateId,
+                    downloadDirectory.getName()));
+
+        } catch (Exception e) {
+
+            final String errMsg = format("Failed to upload template id %1$s "
+                    + "due to $2%s", templateId, e.getMessage());
+            s_logger.error(errMsg, e);
+            return new Answer(cmd, false, errMsg);
+
+        }
+
+    }
 
     private Answer execute(downloadTemplateFromSwiftToSecondaryStorageCommand cmd) {
         SwiftTO swift = cmd.getSwift();
@@ -256,6 +354,83 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
 
+    private Answer execute(UploadTemplateToS3FromSecondaryStorageCommand cmd) {
+
+        final S3TO s3 = cmd.getS3();
+        final Long accountId = cmd.getAccountId();
+        final Long templateId = cmd.getTemplateId();
+
+        try {
+
+            final String templatePath = determineStorageTemplatePath(
+                    cmd.getStoragePath(), accountId, templateId);
+
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Found template id " + templateId
+                        + " account id " + accountId + " from directory "
+                        + templatePath + " to upload to S3.");
+            }
+
+            if (!_storage.isDirectory(templatePath)) {
+                final String errMsg = format("S3 Sync Failure: Directory %1$s"
+                        + "for template id %2$s does not exist.", templatePath,
+                        templateId);
+                s_logger.error(errMsg);
+                return new Answer(cmd, false, errMsg);
+            }
+
+            if (!_storage.isFile(templatePath + "/template.properties")) {
+                final String errMsg = format("S3 Sync Failure: Template id "
+                        + "%1$s does not exist on the file system.",
+                        templatePath);
+                s_logger.error(errMsg);
+                return new Answer(cmd, false, errMsg);
+            }
+
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug(format(
+                        "Pushing template id %1$s from %2$s to S3...",
+                        templateId, templatePath));
+            }
+
+            final String bucket = s3.getBucketName();
+            putDirectory(s3, bucket, _storage.getFile(templatePath),
+                    new FilenameFilter() {
+                        @Override
+                        public boolean accept(final File directory,
+                                final String fileName) {
+                            return !fileName.startsWith(".");
+                        }
+                    }, new ObjectNamingStrategy() {
+                        @Override
+                        public String determineKey(final File file) {
+                            s_logger.debug(String
+                                    .format("Determining key using account id %1$s and template id %2$s",
+                                            accountId, templateId));
+                            return join(
+                                    asList(determineS3TemplateDirectory(
+                                            accountId, templateId), file
+                                            .getName()), S3Utils.SEPARATOR);
+                        }
+                    });
+
+            return new Answer(
+                    cmd,
+                    true,
+                    format("Uploaded the contents of directory %1$s for template id %2$s to S3 bucket %3$s",
+                            templatePath, templateId, bucket));
+
+        } catch (Exception e) {
+
+            final String errMsg = format("Failed to upload template id %1$s",
+                    templateId);
+            s_logger.error(errMsg, e);
+            return new Answer(cmd, false, errMsg);
+
+        }
+
+    }
+
     private Answer execute(DeleteObjectFromSwiftCommand cmd) {
         SwiftTO swift = cmd.getSwift();
         String container = cmd.getContainer();
@@ -279,6 +454,47 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
 
     }
 
+    private Answer execute(final DeleteTemplateFromS3Command cmd) {
+
+        final S3TO s3 = cmd.getS3();
+        final Long accountId = cmd.getAccountId();
+        final Long templateId = cmd.getTemplateId();
+
+        if (accountId == null || (accountId != null && accountId <= 0)) {
+            final String errorMessage = "No account id specified for S3 template deletion.";
+            s_logger.error(errorMessage);
+            return new Answer(cmd, false, errorMessage);
+        }
+
+        if (templateId == null || (templateId != null && templateId <= 0)) {
+            final String errorMessage = "No template id specified for S3 template deletion.";
+            s_logger.error(errorMessage);
+            return new Answer(cmd, false, errorMessage);
+        }
+
+        if (s3 == null) {
+            final String errorMessge = "No S3 client options provided";
+            s_logger.error(errorMessge);
+            return new Answer(cmd, false, errorMessge);
+        }
+
+        final String bucket = s3.getBucketName();
+        try {
+            deleteDirectory(s3, bucket,
+                    determineS3TemplateDirectory(templateId, accountId));
+            return new Answer(cmd, true, String.format(
+                    "Deleted template %1%s from bucket %2$s.", templateId,
+                    bucket));
+        } catch (Exception e) {
+            final String errorMessage = String
+                    .format("Failed to delete templaet id %1$s from bucket %2$s due to the following error: %3$s",
+                            templateId, bucket, e.getMessage());
+            s_logger.error(errorMessage, e);
+            return new Answer(cmd, false, errorMessage);
+        }
+
+    }
+
     String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) {
         Script command = new Script("/bin/bash", s_logger);
         command.add("-c");
@@ -451,6 +667,110 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
 
+    public Answer execute(final DownloadSnapshotFromS3Command cmd) {
+
+        final S3TO s3 = cmd.getS3();
+        final String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
+        final Long accountId = cmd.getAccountId();
+        final Long volumeId = cmd.getVolumeId();
+
+        try {
+
+            executeWithNoWaitLock(determineSnapshotLockId(accountId, volumeId),
+                    new Callable<Void>() {
+
+                        @Override
+                        public Void call() throws Exception {
+
+                            final String directoryName = determineSnapshotLocalDirectory(
+                                    secondaryStorageUrl, accountId, volumeId);
+
+                            String result = createLocalDir(directoryName);
+                            if (result != null) {
+                                throw new InternalErrorException(
+                                        format("Failed to create directory %1$s during S3 snapshot download.",
+                                                directoryName));
+                            }
+
+                            final String snapshotFileName = determineSnapshotBackupFilename(cmd
+                                    .getSnapshotUuid());
+                            final String key = determineSnapshotS3Key(
+                                    accountId, volumeId, snapshotFileName);
+                            final File targetFile = S3Utils.getFile(s3,
+                                    s3.getBucketName(), key,
+                                    _storage.getFile(directoryName),
+                                    new FileNamingStrategy() {
+
+                                        @Override
+                                        public String determineFileName(
+                                                String key) {
+                                            return snapshotFileName;
+                                        }
+
+                                    });
+
+                            if (cmd.getParent() != null) {
+
+                                final String parentPath = join(
+                                        File.pathSeparator, directoryName,
+                                        determineSnapshotBackupFilename(cmd
+                                                .getParent()));
+                                result = setVhdParent(
+                                        targetFile.getAbsolutePath(),
+                                        parentPath);
+                                if (result != null) {
+                                    throw new InternalErrorException(
+                                            format("Failed to set the parent for backup %1$s to %2$s due to %3$s.",
+                                                    targetFile
+                                                            .getAbsolutePath(),
+                                                    parentPath, result));
+                                }
+
+                            }
+
+                            return null;
+
+                        }
+
+                    });
+
+            return new Answer(
+                    cmd,
+                    true,
+                    format("Succesfully retrieved volume id %1$s for account id %2$s to %3$s from S3.",
+                            volumeId, accountId, secondaryStorageUrl));
+
+        } catch (Exception e) {
+            final String errMsg = format(
+                    "Failed to retrieve volume id %1$s for account id %2$s to %3$s from S3 due to exception %4$s",
+                    volumeId, accountId, secondaryStorageUrl, e.getMessage());
+            s_logger.error(errMsg);
+            return new Answer(cmd, false, errMsg);
+        }
+
+    }
+
+    private String determineSnapshotS3Directory(final Long accountId,
+            final Long volumeId) {
+        return join(S3Utils.SEPARATOR, SNAPSHOT_ROOT_DIR, accountId, volumeId);
+    }
+
+    private String determineSnapshotS3Key(final Long accountId,
+            final Long volumeId, final String snapshotFileName) {
+
+        final String directoryName = determineSnapshotS3Directory(accountId,
+                volumeId);
+        return join(S3Utils.SEPARATOR, directoryName, snapshotFileName);
+
+    }
+
+    private String determineSnapshotLocalDirectory(
+            final String secondaryStorageUrl, final Long accountId,
+            final Long volumeId) {
+        return join(File.pathSeparator, getRootDir(secondaryStorageUrl),
+                SNAPSHOT_ROOT_DIR, accountId, volumeId);
+    }
+
     public Answer execute(downloadSnapshotFromSwiftCommand cmd){
         SwiftTO swift = cmd.getSwift();
         String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
@@ -622,6 +942,92 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
     
+    private String deleteSnapshotBackupFromLocalFileSystem(
+            final String secondaryStorageUrl, final Long accountId,
+            final Long volumeId, final String name, final Boolean deleteAllFlag) {
+
+        final String lPath = determineSnapshotLocalDirectory(
+                secondaryStorageUrl, accountId, volumeId)
+                + File.pathSeparator
+                + (deleteAllFlag ? "*" : "*" + name + "*");
+
+        final String result = deleteLocalFile(lPath);
+
+        if (result != null) {
+            return "failed to delete snapshot " + lPath + " , err=" + result;
+        }
+
+        return null;
+
+    }
+
+    private String deleteSnapshotBackupfromS3(final S3TO s3,
+            final String secondaryStorageUrl, final Long accountId,
+            final Long volumeId, final String name, final Boolean deleteAllFlag) {
+
+        try {
+
+            final String bucket = s3.getBucketName();
+
+            final String result = executeWithNoWaitLock(
+                    determineSnapshotLockId(accountId, volumeId),
+                    new Callable<String>() {
+
+                        @Override
+                        public String call() throws Exception {
+
+                            final String innerResult = deleteSnapshotBackupFromLocalFileSystem(
+                                    secondaryStorageUrl, accountId, volumeId,
+                                    name, deleteAllFlag);
+                            if (innerResult != null) {
+                                return innerResult;
+                            }
+
+                            if (deleteAllFlag) {
+                                S3Utils.deleteDirectory(
+                                        s3,
+                                        bucket,
+                                        determineSnapshotS3Directory(accountId,
+                                                volumeId));
+                            } else {
+                                S3Utils.deleteObject(
+                                        s3,
+                                        bucket,
+                                        determineSnapshotS3Key(
+                                                accountId,
+                                                volumeId,
+                                                determineSnapshotBackupFilename(name)));
+                            }
+
+                            return null;
+
+                        }
+
+                    });
+
+            return result;
+
+        } catch (Exception e) {
+
+            s_logger.error(
+                    String.format(
+                            "Failed to delete snapshot backup for account id %1$s volume id %2$sfrom S3.",
+                            accountId, volumeId), e);
+            return e.getMessage();
+
+        }
+
+    }
+
+    private String determineSnapshotBackupFilename(final String snapshotUuid) {
+        return snapshotUuid + ".vhd";
+    }
+
+    private String determineSnapshotLockId(final Long accountId,
+            final Long volumeId) {
+        return join("_", "SNAPSHOT", accountId, volumeId);
+    }
+
     protected Answer execute(final DeleteSnapshotBackupCommand cmd) {
         String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
         Long accountId = cmd.getAccountId();
@@ -629,21 +1035,22 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         String name = cmd.getSnapshotUuid();
         try {
             SwiftTO swift = cmd.getSwift();
+            S3TO s3 = cmd.getS3();
             if (swift == null) {
-                String parent = getRootDir(secondaryStorageUrl);
-                String filename;
-                if (cmd.isAll()) {
-                    filename = "*";
-
-                } else {
-                    filename = "*" + name + "*";
+                final String result = deleteSnapshotBackupFromLocalFileSystem(
+                        secondaryStorageUrl, accountId, volumeId, name,
+                        cmd.isAll());
+                if (result != null) {
+                    s_logger.warn(result);
+                    return new Answer(cmd, false, result);
                 }
-                String lPath = parent + "/snapshots/" + String.valueOf(accountId) + "/" + String.valueOf(volumeId) + "/" + filename;
-                String result = deleteLocalFile(lPath);
+            } else if (s3 != null) {
+                final String result = deleteSnapshotBackupfromS3(s3,
+                        secondaryStorageUrl, accountId, volumeId, name,
+                        cmd.isAll());
                 if (result != null) {
-                    String errMsg = "failed to delete snapshot " + lPath + " , err=" + result;
-                    s_logger.warn(errMsg);
-                    return new Answer(cmd, false, errMsg);
+                    s_logger.warn(result);
+                    return new Answer(cmd, false, result);
                 }
             } else {
                 String filename;

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/patches/systemvm/debian/config/etc/sysctl.conf
----------------------------------------------------------------------
diff --git a/patches/systemvm/debian/config/etc/sysctl.conf b/patches/systemvm/debian/config/etc/sysctl.conf
index 7f945b0..961d471 100644
--- a/patches/systemvm/debian/config/etc/sysctl.conf
+++ b/patches/systemvm/debian/config/etc/sysctl.conf
@@ -8,7 +8,7 @@
 net.ipv4.ip_forward = 1
 
 # Controls source route verification
-net.ipv4.conf.default.rp_filter = 1
+net.ipv4.conf.default.rp_filter = 0
 
 # Do not accept source routing
 net.ipv4.conf.default.accept_source_route = 0

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java
index 967a299..313703c 100644
--- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java
+++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java
@@ -17,6 +17,10 @@
 package com.cloud.hypervisor.xen.resource;
 
 
+import java.beans.BeanInfo;
+import java.beans.IntrospectionException;
+import java.beans.Introspector;
+import java.beans.PropertyDescriptor;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
@@ -24,11 +28,13 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.StringReader;
+import java.lang.reflect.InvocationTargetException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -179,6 +185,7 @@ import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand;
 import com.cloud.agent.api.to.IpAddressTO;
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.PortForwardingRuleTO;
+import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.StaticNatRuleTO;
 import com.cloud.agent.api.to.StorageFilerTO;
 import com.cloud.agent.api.to.SwiftTO;
@@ -217,6 +224,8 @@ import com.cloud.storage.template.TemplateInfo;
 import com.cloud.template.VirtualMachineTemplate.BootloaderType;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
+import com.cloud.utils.S3Utils;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.NetUtils;
@@ -6507,7 +6516,14 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                         } finally {
                             deleteSnapshotBackup(conn, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotBackupUuid);
                         }
-                    }                    
+                    } else if (cmd.getS3() != null) {
+                        try {
+                            backupSnapshotToS3(conn, cmd.getS3(), snapshotSr.getUuid(conn), snapshotBackupUuid, isISCSI, wait);
+                            snapshotBackupUuid = snapshotBackupUuid + ".vhd";
+                        } finally {
+                            deleteSnapshotBackup(conn, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotBackupUuid);
+                        }
+                    }
                     success = true;
                 } finally {
                     if( snapshotSr != null) {
@@ -6524,6 +6540,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                         snapshotBackupUuid = snapshotPaUuid + ".vhd";
                     }
                     success = true;
+                } else if (cmd.getS3() != null) {
+                    backupSnapshotToS3(conn, cmd.getS3(), primaryStorageSRUuid, snapshotPaUuid, isISCSI, wait);
                 } else {
                     snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI, wait);
                     success = (snapshotBackupUuid != null);
@@ -6546,6 +6564,88 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         return new BackupSnapshotAnswer(cmd, success, details, snapshotBackupUuid, fullbackup);
     }
 
+    private static List<String> serializeProperties(final Object object,
+            final Class<?> propertySet) {
+
+        assert object != null;
+        assert propertySet != null;
+        assert propertySet.isAssignableFrom(object.getClass());
+
+        try {
+
+            final BeanInfo beanInfo = Introspector.getBeanInfo(propertySet);
+            final PropertyDescriptor[] descriptors = beanInfo
+                    .getPropertyDescriptors();
+
+            final List<String> serializedProperties = new ArrayList<String>();
+            for (final PropertyDescriptor descriptor : descriptors) {
+
+                serializedProperties.add(descriptor.getName());
+                final Object value = descriptor.getReadMethod().invoke(object);
+                serializedProperties.add(value != null ? value.toString()
+                        : "null");
+
+            }
+
+            return Collections.unmodifiableList(serializedProperties);
+
+        } catch (IntrospectionException e) {
+            s_logger.warn(
+                    "Ignored IntrospectionException when serializing class "
+                            + object.getClass().getCanonicalName(), e);
+        } catch (IllegalArgumentException e) {
+            s_logger.warn(
+                    "Ignored IllegalArgumentException when serializing class "
+                            + object.getClass().getCanonicalName(), e);
+        } catch (IllegalAccessException e) {
+            s_logger.warn(
+                    "Ignored IllegalAccessException when serializing class "
+                            + object.getClass().getCanonicalName(), e);
+        } catch (InvocationTargetException e) {
+            s_logger.warn(
+                    "Ignored InvocationTargetException when serializing class "
+                            + object.getClass().getCanonicalName(), e);
+        }
+
+        return Collections.emptyList();
+
+    }
+
+    private boolean backupSnapshotToS3(final Connection connection,
+            final S3TO s3, final String srUuid, final String snapshotUuid,
+            final Boolean iSCSIFlag, final int wait) {
+
+        final String filename = iSCSIFlag ? "VHD-" + snapshotUuid
+                : snapshotUuid + ".vhd";
+        final String dir = (iSCSIFlag ? "/dev/VG_XenStorage-"
+                : "/var/run/sr-mount/") + srUuid;
+        final String key = StringUtils.join("/", "snapshots", snapshotUuid);
+
+        try {
+
+            final List<String> parameters = new ArrayList<String>(
+                    serializeProperties(s3, S3Utils.ClientOptions.class));
+            parameters.addAll(Arrays.asList("operation", "put", "directory",
+                    dir, "filename", filename, "iSCSIFlag",
+                    iSCSIFlag.toString(), "key", key));
+            final String result = callHostPluginAsync(connection, "s3xen",
+                    "s3", wait,
+                    parameters.toArray(new String[parameters.size()]));
+
+            if (result != null && result.equals("true")) {
+                return true;
+            }
+
+        } catch (Exception e) {
+            s_logger.error(String.format(
+                    "S3 upload failed of snapshot %1$s due to %2$s.",
+                    snapshotUuid, e.toString()), e);
+        }
+
+        return false;
+
+    }
+
     protected CreateVolumeFromSnapshotAnswer execute(final CreateVolumeFromSnapshotCommand cmd) {
         Connection conn = getConnection();
         String primaryStorageNameLabel = cmd.getPrimaryStoragePoolNameLabel();

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2c2600b..e5aa024 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,7 +67,7 @@
     <cs.gson.version>1.7.1</cs.gson.version>
     <cs.xapi.version>5.6.100-1-SNAPSHOT</cs.xapi.version>
     <cs.httpclient.version>3.1</cs.httpclient.version>
-    <cs.httpcore.version>4.0</cs.httpcore.version>
+    <cs.httpcore.version>4.1</cs.httpcore.version>
     <cs.mysql.version>5.1.21</cs.mysql.version>
     <cs.xstream.version>1.3.1</cs.xstream.version>
     <cs.xmlrpc.version>3.1.3</cs.xmlrpc.version>
@@ -82,7 +82,9 @@
     <cs.selenium.server.version>1.0-20081010.060147</cs.selenium.server.version>
     <cs.vmware.api.version>4.1</cs.vmware.api.version>
     <cs.mockito.version>1.9.5</cs.mockito.version>
-
+    <cs.aws.sdk.version>1.3.21.1</cs.aws.sdk.version>
+    <cs.lang.version>2.6</cs.lang.version>
+    <cs.commons-io.version>1.4</cs.commons-io.version>
   </properties>
 
   <distributionManagement>

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/scripts/vm/hypervisor/xenserver/s3xen
----------------------------------------------------------------------
diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen
new file mode 100644
index 0000000..4d9c12d
--- /dev/null
+++ b/scripts/vm/hypervisor/xenserver/s3xen
@@ -0,0 +1,297 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Version @VERSION@
+#
+# A plugin for executing script needed by cloud  stack
+from __future__ import with_statement
+
+from copy import copy
+from datetime import datetime
+from httplib import *
+from string import join
+
+import os
+import sys
+import time
+import hashlib
+import base64
+import hmac
+import traceback
+import urllib2
+
+import XenAPIPlugin
+sys.path.extend(["/opt/xensource/sm/"])
+import util
+
+NULL = 'null'
+
+# Value conversion utility functions ...
+
+
+def to_none(value):
+    return value if value is not None and value.strip() != NULL else None
+
+
+def to_bool(value):
+    return True if to_none(value) in ['true', 'True', None] else False
+
+
+def to_integer(value, default):
+    return int(value) if to_none(value) is not None else default
+
+
+def optional_str_value(value, default):
+    return value if is_not_blank(value) else default
+
+
+def is_not_blank(value):
+    return True if to_none(value) is not None and value.strip != '' else False
+
+
+def get_optional_key(map, key, default=''):
+    return map[key] if key in map else default
+
+
+def log(message):
+    util.SMlog('#### VMOPS %s ####' % message)
+
+
+def echo(fn):
+    def wrapped(*v, **k):
+        name = fn.__name__
+        log("enter %s ####" % name)
+        res = fn(*v, **k)
+        log("exit %s with result %s" % name, res)
+        return res
+    return wrapped
+
+
+def require_str_value(value, error_message):
+
+    if is_not_blank(value):
+        return value
+
+    raise ValueError(error_message)
+
+
+def retry(max_attempts, fn):
+
+    attempts = 1
+    while attempts <= max_attempts:
+        log("Attempting execution {0}/{1} of {2}".
+            format(attempts, max_attempts, fn.__name__))
+        try:
+            return fn()
+        except:
+            if (attempts >= max_attempts):
+                raise
+            attempts = attempts + 1
+
+
+def compute_md5(filename, buffer_size=8192):
+
+    hasher = hashlib.md5()
+
+    with open(filename, 'rb') as file:
+        data = file.read(buffer_size)
+        while data != "":
+            hasher.update(data)
+            data = file.read(buffer_size)
+
+    return base64.encodestring(hasher.digest())[:-1]
+
+
+class S3Client(object):
+
+    DEFAULT_END_POINT = 's3.amazonaws.com'
+    DEFAULT_CONNECTION_TIMEOUT = 50000
+    DEFAULT_SOCKET_TIMEOUT = 50000
+    DEFAULT_MAX_ERROR_RETRY = 3
+
+    HEADER_CONTENT_MD5 = 'Content-MD5'
+    HEADER_CONTENT_TYPE = 'Content-Type'
+    HEADER_CONTENT_LENGTH = 'Content-Length'
+
+    def __init__(self, access_key, secret_key, end_point=None,
+                 https_flag=None, connection_timeout=None, socket_timeout=None,
+                 max_error_retry=None):
+
+        self.access_key = require_str_value(
+            access_key, 'An access key must be specified.')
+        self.secret_key = require_str_value(
+            secret_key, 'A secret key must be specified.')
+        self.end_point = optional_str_value(end_point, self.DEFAULT_END_POINT)
+        self.https_flag = to_bool(https_flag)
+        self.connection_timeout = to_integer(
+            connection_timeout, self.DEFAULT_CONNECTION_TIMEOUT)
+        self.socket_timeout = to_integer(
+            socket_timeout, self.DEFAULT_SOCKET_TIMEOUT)
+        self.max_error_retry = to_integer(
+            max_error_retry, self.DEFAULT_MAX_ERROR_RETRY)
+
+    def build_canocialized_resource(self, bucket, key):
+
+        return '/{bucket}/{key}'.format(bucket=bucket, key=key)
+
+    def noop_send_body():
+        pass
+
+    def noop_read(response):
+        return response.read()
+
+    def do_operation(
+        self, method, bucket, key, input_headers={},
+            fn_send_body=noop_send_body, fn_read=noop_read):
+
+        headers = copy(input_headers)
+        headers['Expect'] = '100-continue'
+
+        uri = self.build_canocialized_resource(bucket, key)
+        signature, request_date = self.sign_request(method, uri, headers)
+        headers['Authorization'] = "AWS {0}:{1}".format(
+            self.access_key, signature)
+        headers['Date'] = request_date
+
+        connection = HTTPSConnection(self.end_point) \
+            if self.https_flag else HTTPConnection(self.end_point)
+        connection.timeout = self.socket_timeout
+
+        def perform_request():
+
+            connection.request(method, uri, fn_send_body(), headers)
+            response = connection.getresponse()
+            log("Sent {0} request to {1} {2} with headers {3}.  \
+                Got response status {4}: {5}".
+                format(method, self.end_point, uri, headers,
+                response.status, response.reason))
+            return fn_read(response)
+
+        try:
+            return retry(self.max_error_retry, perform_request)
+        finally:
+            connection.close()
+
+    '''
+    See http://bit.ly/MMC5de for more information regarding the creation of
+    AWS authorization tokens and header signing
+    '''
+    def sign_request(self, operation, canocialized_resource, headers):
+
+        request_date = datetime.utcnow(
+        ).strftime('%a, %d %b %Y %H:%M:%S +0000')
+
+        content_hash = get_optional_key(headers, self.HEADER_CONTENT_MD5)
+        content_type = get_optional_key(headers, self.HEADER_CONTENT_TYPE)
+
+        string_to_sign = join(
+            [operation, content_hash, content_type, request_date,
+                canocialized_resource], '\n')
+
+        signature = base64.encodestring(
+            hmac.new(self.secret_key, string_to_sign.encode('utf8'),
+                     hashlib.sha1).digest())[:-1]
+
+        return signature, request_date
+
+    def put(self, bucket, key, src_filename):
+
+        headers = {
+            self.HEADER_CONTENT_MD5: compute_md5(src_filename),
+            self.HEADER_CONTENT_TYPE: 'application/octet-stream',
+            self.HEADER_CONTENT_LENGTH: os.stat(src_filename).st_size,
+        }
+
+        def send_body():
+            return open(src_filename, 'rb')
+
+        self.do_operation('PUT', bucket, key, headers, send_body)
+
+    def get(self, bucket, key, target_filename):
+
+        def read(response):
+
+            with open(target_filename, 'wb') as file:
+                while True:
+                    block = response.read(8192)
+                    if not block:
+                        break
+                        file.write(block)
+
+        return self.do_operation('GET', bucket, key, fn_read=read)
+
+    def delete(self, bucket, key):
+
+        return self.do_operation('DELETE', bucket, key)
+
+
+def parseArguments(args):
+
+    # The keys in the args map will correspond to the properties defined on
+    # the com.cloud.utils.S3Utils#ClientOptions interface
+    client = S3Client(
+        args['accessKey'], args['secretKey'], args['endPoint'],
+        args['isHttps'], args['connectionTimeout'], args['socketTimeout'])
+
+    operation = args['operation']
+    bucket = args['bucket']
+    key = args['key']
+    filename = args['filename']
+
+    if is_blank(operation):
+        raise ValueError('An operation must be specified.')
+
+    if is_blank(bucket):
+        raise ValueError('A bucket must be specified.')
+
+    if is_blank(key):
+        raise ValueError('A value must be specified.')
+
+    if is_blank(filename):
+        raise ValueError('A filename must be specified.')
+
+    return client, operation, bucket, key, filename
+
+
+@echo
+def s3(session, args):
+
+    client, operation, bucket, key, filename = parseArguments(args)
+
+    try:
+
+        if operation == 'put':
+            client.put(bucket, key, filename)
+        elif operation == 'get':
+            client.get(bucket, key, filename)
+        elif operation == 'delete':
+            client.delete(bucket, key, filename)
+        else:
+            raise RuntimeError(
+                "S3 plugin does not support operation {0}.".format(operation))
+
+        return 'true'
+
+    except:
+        log("Operation {0} on file {1} from/in bucket {2} key {3}.".format(
+            operation, filename, bucket, key))
+        log(traceback.format_exc())
+        return 'false'
+
+if __name__ == "__main__":
+    XenAPIPlugin.dispatch({"s3": s3})

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/scripts/vm/hypervisor/xenserver/xenserver56/patch
----------------------------------------------------------------------
diff --git a/scripts/vm/hypervisor/xenserver/xenserver56/patch b/scripts/vm/hypervisor/xenserver/xenserver56/patch
index d485414..36dba3d 100644
--- a/scripts/vm/hypervisor/xenserver/xenserver56/patch
+++ b/scripts/vm/hypervisor/xenserver/xenserver56/patch
@@ -62,3 +62,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin
 bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin
 swift=..,0755,/opt/xensource/bin
 swiftxen=..,0755,/etc/xapi.d/plugins
+s3xen=..,0755,/etc/xapi.d/plugins
+

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch
----------------------------------------------------------------------
diff --git a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch
index 9fe9740..d20e60f 100644
--- a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch
+++ b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch
@@ -61,3 +61,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin
 bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin
 swift=..,0755,/opt/xensource/bin
 swiftxen=..,0755,/etc/xapi.d/plugins
+s3xen=..,0755,/etc/xapi.d/plugins
+

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/scripts/vm/hypervisor/xenserver/xenserver60/patch
----------------------------------------------------------------------
diff --git a/scripts/vm/hypervisor/xenserver/xenserver60/patch b/scripts/vm/hypervisor/xenserver/xenserver60/patch
index f049109..c9125f4 100644
--- a/scripts/vm/hypervisor/xenserver/xenserver60/patch
+++ b/scripts/vm/hypervisor/xenserver/xenserver60/patch
@@ -66,3 +66,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin
 bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin
 swift=..,0755,/opt/xensource/bin
 swiftxen=..,0755,/etc/xapi.d/plugins
+s3xen=..,0755,/etc/xapi.d/plugins
+

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index e3308d8..5ae926f 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -63,13 +63,17 @@
       <version>${cs.jstl.version}</version>
     </dependency>
     <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>${cs.codec.version}</version>
+    </dependency>
+    <dependency>
       <groupId>org.apache.cloudstack</groupId>
       <artifactId>cloud-utils</artifactId>
       <version>${project.version}</version>
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
-    
   </dependencies>
   <build>
     <defaultGoal>install</defaultGoal>

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/api/ApiDBUtils.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/api/ApiDBUtils.java b/server/src/com/cloud/api/ApiDBUtils.java
index 3b5f634..cdd5339 100755
--- a/server/src/com/cloud/api/ApiDBUtils.java
+++ b/server/src/com/cloud/api/ApiDBUtils.java
@@ -115,6 +115,7 @@ import com.cloud.storage.StoragePoolVO;
 import com.cloud.storage.StorageStats;
 import com.cloud.storage.UploadVO;
 import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateS3VO;
 import com.cloud.storage.VMTemplateSwiftVO;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.Volume.Type;
@@ -129,6 +130,7 @@ import com.cloud.storage.dao.UploadDao;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VMTemplateDetailsDao;
 import com.cloud.storage.dao.VMTemplateHostDao;
+import com.cloud.storage.dao.VMTemplateS3Dao;
 import com.cloud.storage.dao.VMTemplateSwiftDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.dao.VolumeHostDao;
@@ -196,6 +198,7 @@ public class ApiDBUtils {
     private static VMTemplateDetailsDao _templateDetailsDao;
     private static VMTemplateHostDao _templateHostDao;
     private static VMTemplateSwiftDao _templateSwiftDao;
+    private static VMTemplateS3Dao _templateS3Dao;
     private static UploadDao _uploadDao;
     private static UserDao _userDao;
     private static UserStatisticsDao _userStatsDao;
@@ -260,6 +263,7 @@ public class ApiDBUtils {
         _templateDetailsDao = locator.getDao(VMTemplateDetailsDao.class);
         _templateHostDao = locator.getDao(VMTemplateHostDao.class);
         _templateSwiftDao = locator.getDao(VMTemplateSwiftDao.class);
+        _templateS3Dao = locator.getDao(VMTemplateS3Dao.class);
         _uploadDao = locator.getDao(UploadDao.class);
         _userDao = locator.getDao(UserDao.class);
         _userStatsDao = locator.getDao(UserStatisticsDao.class);
@@ -575,6 +579,10 @@ public class ApiDBUtils {
         return _templateSwiftDao.findOneByTemplateId(templateId);
     }
 
+    public static VMTemplateS3VO findTemplateS3Ref(long templateId) {
+        return _templateS3Dao.findOneByTemplateId(templateId);
+    }
+
     public static UploadVO findUploadById(Long id) {
         return _uploadDao.findById(id);
     }

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/api/ApiDispatcher.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/api/ApiDispatcher.java b/server/src/com/cloud/api/ApiDispatcher.java
index dfe4a1f..9fd0b2e 100755
--- a/server/src/com/cloud/api/ApiDispatcher.java
+++ b/server/src/com/cloud/api/ApiDispatcher.java
@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.api;
 
+import static org.apache.commons.lang.StringUtils.isNotBlank;
+
 import java.lang.reflect.Field;
 import java.text.DateFormat;
 import java.text.ParseException;
@@ -41,7 +43,6 @@ import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.utils.IdentityProxy;
 import com.cloud.server.ManagementServer;
 import com.cloud.user.Account;
 import com.cloud.user.UserContext;
@@ -417,10 +418,20 @@ public class ApiDispatcher {
                 }
                 break;
             case FLOAT:
-                field.set(cmdObj, Float.valueOf(paramObj.toString()));
+                // Assuming that the parameters have been checked for required before now,
+                // we ignore blank or null values and defer to the command to set a default
+                // value for optional parameters ...
+                if (paramObj != null && isNotBlank(paramObj.toString())) {
+                    field.set(cmdObj, Float.valueOf(paramObj.toString()));
+                }
                 break;
             case INTEGER:
-                field.set(cmdObj, Integer.valueOf(paramObj.toString()));
+                // Assuming that the parameters have been checked for required before now,
+                // we ignore blank or null values and defer to the command to set a default
+                // value for optional parameters ...
+                if (paramObj != null && isNotBlank(paramObj.toString())) {
+                    field.set(cmdObj, Integer.valueOf(paramObj.toString()));
+                }
                 break;
             case LIST:
                 List listParam = new ArrayList();

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/api/ApiResponseHelper.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java
index ebe8415..a574710 100755
--- a/server/src/com/cloud/api/ApiResponseHelper.java
+++ b/server/src/com/cloud/api/ApiResponseHelper.java
@@ -16,6 +16,9 @@
 // under the License.
 package com.cloud.api;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
 import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.Date;
@@ -82,6 +85,7 @@ import com.cloud.api.response.RemoteAccessVpnResponse;
 import com.cloud.api.response.ResourceCountResponse;
 import com.cloud.api.response.ResourceLimitResponse;
 import com.cloud.api.response.ResourceTagResponse;
+import com.cloud.api.response.S3Response;
 import com.cloud.api.response.SecurityGroupResponse;
 import com.cloud.api.response.SecurityGroupResultObject;
 import com.cloud.api.response.SecurityGroupRuleResponse;
@@ -131,7 +135,6 @@ import com.cloud.dc.Vlan.VlanType;
 import com.cloud.dc.VlanVO;
 import com.cloud.domain.Domain;
 import com.cloud.event.Event;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.host.Host;
 import com.cloud.host.HostStats;
 import com.cloud.host.HostVO;
@@ -189,6 +192,7 @@ import com.cloud.server.ResourceTag.TaggedResourceType;
 import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.GuestOS;
 import com.cloud.storage.GuestOSCategoryVO;
+import com.cloud.storage.S3;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.ImageFormat;
@@ -200,6 +204,7 @@ import com.cloud.storage.StorageStats;
 import com.cloud.storage.Swift;
 import com.cloud.storage.UploadVO;
 import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateS3VO;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.storage.VMTemplateSwiftVO;
 import com.cloud.storage.VMTemplateVO;
@@ -744,6 +749,25 @@ public class ApiResponseHelper implements ResponseGenerator {
     }
 
     @Override
+    public S3Response createS3Response(final S3 result) {
+
+        final S3Response response = new S3Response();
+
+        response.setAccessKey(result.getAccessKey());
+        response.setConnectionTimeout(result.getConnectionTimeout());
+        response.setEndPoint(result.getEndPoint());
+        response.setHttpsFlag(result.getHttpsFlag());
+        response.setMaxErrorRetry(result.getMaxErrorRetry());
+        response.setObjectId(result.getId());
+        response.setSecretKey(result.getSecretKey());
+        response.setSocketTimeout(result.getSocketTimeout());
+        response.setTemplateBucketName(result.getBucketName());
+
+        return response;
+
+    }
+
+    @Override
     public VlanIpRangeResponse createVlanIpRangeResponse(Vlan vlan) {
         Long podId = ApiDBUtils.getPodIdForVlan(vlan.getId());
 
@@ -2153,7 +2177,7 @@ public class ApiResponseHelper implements ResponseGenerator {
     @Override
     public List<TemplateResponse> createIsoResponses(long isoId, Long zoneId, boolean readyOnly) {
 
-        List<TemplateResponse> isoResponses = new ArrayList<TemplateResponse>();
+        final List<TemplateResponse> isoResponses = new ArrayList<TemplateResponse>();
         VirtualMachineTemplate iso = findTemplateById(isoId);
         if (iso.getTemplateType() == TemplateType.PERHOST) {
             TemplateResponse isoResponse = new TemplateResponse();
@@ -2191,11 +2215,17 @@ public class ApiResponseHelper implements ResponseGenerator {
             return isoResponses;
         } else {
             if (zoneId == null || zoneId == -1) {
-                isoResponses = createSwiftIsoResponses(iso);
+                isoResponses.addAll(createSwiftIsoResponses(iso));
+                if (!isoResponses.isEmpty()) {
+                    return isoResponses;
+                }
+
+                isoResponses.addAll(createS3IsoResponses(iso));
                 if (!isoResponses.isEmpty()) {
                     return isoResponses;
                 }
-                List<DataCenterVO> dcs = new ArrayList<DataCenterVO>();
+
+                final List<DataCenterVO> dcs = new ArrayList<DataCenterVO>();
                 dcs.addAll(ApiDBUtils.listZones());
                 for (DataCenterVO dc : dcs) {
                     isoResponses.addAll(createIsoResponses(iso, dc.getId(), readyOnly));
@@ -2207,6 +2237,65 @@ public class ApiResponseHelper implements ResponseGenerator {
         }
     }
 
+    private List<? extends TemplateResponse> createS3IsoResponses(final VirtualMachineTemplate iso) {
+
+        final VMTemplateS3VO s3Iso = ApiDBUtils.findTemplateS3Ref(iso.getId());
+
+        if (s3Iso == null) {
+            return emptyList();
+        }
+
+        final TemplateResponse templateResponse = new TemplateResponse();
+
+        templateResponse.setId(iso.getId());
+        templateResponse.setName(iso.getName());
+        templateResponse.setDisplayText(iso.getDisplayText());
+        templateResponse.setPublic(iso.isPublicTemplate());
+        templateResponse.setExtractable(iso.isExtractable());
+        templateResponse.setCreated(s3Iso.getCreated());
+        templateResponse.setReady(true);
+        templateResponse.setBootable(iso.isBootable());
+        templateResponse.setFeatured(iso.isFeatured());
+        templateResponse.setCrossZones(iso.isCrossZones());
+        templateResponse.setChecksum(iso.getChecksum());
+        templateResponse.setDetails(iso.getDetails());
+
+        final GuestOS os = ApiDBUtils.findGuestOSById(iso.getGuestOSId());
+
+        if (os != null) {
+            templateResponse.setOsTypeId(os.getId());
+            templateResponse.setOsTypeName(os.getDisplayName());
+        } else {
+            templateResponse.setOsTypeId(-1L);
+            templateResponse.setOsTypeName("");
+        }
+
+        final Account account = ApiDBUtils.findAccountByIdIncludingRemoved(iso.getAccountId());
+        populateAccount(templateResponse, account.getId());
+        populateDomain(templateResponse, account.getDomainId());
+
+        boolean isAdmin = false;
+        if ((account == null) || BaseCmd.isAdmin(account.getType())) {
+            isAdmin = true;
+        }
+
+        // If the user is an admin, add the template download status
+        if (isAdmin || account.getId() == iso.getAccountId()) {
+            // add download status
+            templateResponse.setStatus("Successfully Installed");
+        }
+
+        final Long isoSize = s3Iso.getSize();
+        if (isoSize > 0) {
+            templateResponse.setSize(isoSize);
+        }
+
+        templateResponse.setObjectName("iso");
+
+        return singletonList(templateResponse);
+
+    }
+
     private List<TemplateResponse> createSwiftIsoResponses(VirtualMachineTemplate iso) {
         long isoId = iso.getId();
         List<TemplateResponse> isoResponses = new ArrayList<TemplateResponse>();

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/api/doc/ApiXmlDocWriter.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/api/doc/ApiXmlDocWriter.java b/server/src/com/cloud/api/doc/ApiXmlDocWriter.java
index d31ef5a..ffdd5be 100644
--- a/server/src/com/cloud/api/doc/ApiXmlDocWriter.java
+++ b/server/src/com/cloud/api/doc/ApiXmlDocWriter.java
@@ -314,6 +314,11 @@ public class ApiXmlDocWriter {
             impl = clas.getSuperclass().getAnnotation(Implementation.class);
         }
 
+        if (impl == null) {
+            throw new IllegalStateException(String.format("An %1$s annotation is required for class %2$s.", 
+                    Implementation.class.getCanonicalName(), clas.getCanonicalName()));
+        }
+
         if (impl.includeInApiDoc()) {
             String commandDescription = impl.description();
             if (commandDescription != null && !commandDescription.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/configuration/Config.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java
index 66ac276..9b82fcc 100755
--- a/server/src/com/cloud/configuration/Config.java
+++ b/server/src/com/cloud/configuration/Config.java
@@ -140,8 +140,8 @@ public enum Config {
     JobExpireMinutes("Advanced", ManagementServer.class, String.class, "job.expire.minutes", "1440", "Time (in minutes) for async-jobs to be kept in system", null),
     JobCancelThresholdMinutes("Advanced", ManagementServer.class, String.class, "job.cancel.threshold.minutes", "60", "Time (in minutes) for async-jobs to be forcely cancelled if it has been in process for long", null),
     SwiftEnable("Advanced", ManagementServer.class, Boolean.class, "swift.enable", "false", "enable swift ", null),
-	
-	EventPurgeInterval("Advanced", ManagementServer.class, Integer.class, "event.purge.interval", "86400", "The interval (in seconds) to wait before running the event purge thread", null),
+    S3Enable("Advanced", ManagementServer.class, Boolean.class, "s3.enable", "false", "enable s3 ", null),
+    EventPurgeInterval("Advanced", ManagementServer.class, Integer.class, "event.purge.interval", "86400", "The interval (in seconds) to wait before running the event purge thread", null),
 	AccountCleanupInterval("Advanced", ManagementServer.class, Integer.class, "account.cleanup.interval", "86400", "The interval (in seconds) between cleanup for removed accounts", null),
 	AllowPublicUserTemplates("Advanced", ManagementServer.class, Integer.class, "allow.public.user.templates", "true", "If false, users will not be able to create public templates.", null),
 	InstanceName("Advanced", AgentManager.class, String.class, "instance.name", "VM", "Name of the deployment instance.", "instanceName"),

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java
index 33baaf1..2d7dfe2 100755
--- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java
+++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java
@@ -63,7 +63,9 @@ import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.SwiftVO;
 import com.cloud.storage.dao.DiskOfferingDao;
+import com.cloud.storage.dao.S3Dao;
 import com.cloud.storage.dao.SwiftDao;
+import com.cloud.storage.s3.S3Manager;
 import com.cloud.storage.secondary.SecondaryStorageVmManager;
 import com.cloud.storage.swift.SwiftManager;
 import com.cloud.test.IPRangeConfig;
@@ -118,6 +120,8 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
     @Inject
     SwiftDao _swiftDao;
     @Inject
+    S3Dao _s3Dao;
+    @Inject
     ServiceOfferingDao _serviceOfferingDao;
     @Inject
     DiskOfferingDao _diskOfferingDao;
@@ -158,6 +162,8 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
     @Inject
     SwiftManager _swiftMgr;
     @Inject
+    S3Manager _s3Mgr;
+    @Inject
     PhysicalNetworkTrafficTypeDao _trafficTypeDao;
     @Inject
     NicDao _nicDao;
@@ -419,6 +425,14 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
                 if (swift != null) {
                     return " can not change " + Config.SwiftEnable.key() + " after you have added Swift";
                 }
+                if (this._s3Mgr.isS3Enabled()) {
+                    return String.format("Swift is not supported when S3 is enabled.");
+                }
+            }
+            if (Config.S3Enable.key().equals(name)) {
+                if (this._swiftMgr.isSwiftEnabled()) {
+                    return String.format("S3-backed Secondary Storage is not supported when Swift is enabled.");
+                }
             }
             return null;
         }
@@ -1520,6 +1534,7 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
             createDefaultSystemNetworks(zone.getId());
 
             _swiftMgr.propagateSwiftTmplteOnZone(zone.getId());
+            _s3Mgr.propagateTemplatesToZone(zone);
             txn.commit();
             return zone;
         } catch (Exception ex) {

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/configuration/DefaultComponentLibrary.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/configuration/DefaultComponentLibrary.java b/server/src/com/cloud/configuration/DefaultComponentLibrary.java
index ef61044..226ca49 100755
--- a/server/src/com/cloud/configuration/DefaultComponentLibrary.java
+++ b/server/src/com/cloud/configuration/DefaultComponentLibrary.java
@@ -147,6 +147,7 @@ import com.cloud.storage.dao.DiskOfferingDaoImpl;
 import com.cloud.storage.dao.GuestOSCategoryDaoImpl;
 import com.cloud.storage.dao.GuestOSDaoImpl;
 import com.cloud.storage.dao.LaunchPermissionDaoImpl;
+import com.cloud.storage.dao.S3DaoImpl;
 import com.cloud.storage.dao.SnapshotDaoImpl;
 import com.cloud.storage.dao.SnapshotPolicyDaoImpl;
 import com.cloud.storage.dao.SnapshotScheduleDaoImpl;
@@ -159,11 +160,13 @@ import com.cloud.storage.dao.VMTemplateDaoImpl;
 import com.cloud.storage.dao.VMTemplateDetailsDaoImpl;
 import com.cloud.storage.dao.VMTemplateHostDaoImpl;
 import com.cloud.storage.dao.VMTemplatePoolDaoImpl;
+import com.cloud.storage.dao.VMTemplateS3DaoImpl;
 import com.cloud.storage.dao.VMTemplateSwiftDaoImpl;
 import com.cloud.storage.dao.VMTemplateZoneDaoImpl;
 import com.cloud.storage.dao.VolumeDaoImpl;
 import com.cloud.storage.dao.VolumeHostDaoImpl;
 import com.cloud.storage.download.DownloadMonitorImpl;
+import com.cloud.storage.s3.S3ManagerImpl;
 import com.cloud.storage.secondary.SecondaryStorageManagerImpl;
 import com.cloud.storage.snapshot.SnapshotManagerImpl;
 import com.cloud.storage.snapshot.SnapshotSchedulerImpl;
@@ -270,6 +273,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com
         addDao("VMTemplateHostDao", VMTemplateHostDaoImpl.class);
         addDao("VolumeHostDao", VolumeHostDaoImpl.class);
         addDao("VMTemplateSwiftDao", VMTemplateSwiftDaoImpl.class);
+        addDao("VMTemplateS3Dao", VMTemplateS3DaoImpl.class);
         addDao("UploadDao", UploadDaoImpl.class);
         addDao("VMTemplatePoolDao", VMTemplatePoolDaoImpl.class);
         addDao("LaunchPermissionDao", LaunchPermissionDaoImpl.class);
@@ -318,6 +322,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com
         addDao("KeystoreDao", KeystoreDaoImpl.class);
         addDao("DcDetailsDao", DcDetailsDaoImpl.class);
         addDao("SwiftDao", SwiftDaoImpl.class);
+        addDao("S3Dao", S3DaoImpl.class);
         addDao("AgentTransferMapDao", HostTransferMapDaoImpl.class);
         addDao("ProjectDao", ProjectDaoImpl.class);
         addDao("InlineLoadBalancerNicMapDao", InlineLoadBalancerNicMapDaoImpl.class);
@@ -403,6 +408,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com
         info.addParameter("consoleproxy.sslEnabled", "true");
         addManager("ProjectManager", ProjectManagerImpl.class);
         addManager("SwiftManager", SwiftManagerImpl.class);
+        addManager("S3Manager", S3ManagerImpl.class);
         addManager("StorageNetworkManager", StorageNetworkManagerImpl.class);
         addManager("ExternalLoadBalancerUsageManager", ExternalLoadBalancerUsageManagerImpl.class);
         addManager("HA Manager", HighAvailabilityManagerImpl.class);

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/resource/ResourceManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java
index ced601b..8d50886 100755
--- a/server/src/com/cloud/resource/ResourceManagerImpl.java
+++ b/server/src/com/cloud/resource/ResourceManagerImpl.java
@@ -30,6 +30,11 @@ import java.util.Set;
 import javax.ejb.Local;
 import javax.naming.ConfigurationException;
 
+import com.cloud.api.commands.AddS3Cmd;
+import com.cloud.api.commands.ListS3sCmd;
+import com.cloud.storage.S3;
+import com.cloud.storage.S3VO;
+import com.cloud.storage.s3.S3Manager;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
@@ -181,6 +186,8 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma
     @Inject
     protected SwiftManager _swiftMgr;
     @Inject
+    protected S3Manager                      _s3Mgr;
+    @Inject
     protected HostDetailsDao                 _hostDetailsDao;
     @Inject
     protected ConfigurationDao _configDao;
@@ -561,6 +568,16 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma
         return _swiftMgr.listSwifts(cmd);
     }
 
+    @Override
+    public S3 discoverS3(final AddS3Cmd cmd) throws DiscoveryException {
+        return this._s3Mgr.addS3(cmd);
+    }
+
+    @Override
+    public List<S3VO> listS3s(final ListS3sCmd cmd) {
+        return this._s3Mgr.listS3s(cmd);
+    }
+
     private List<HostVO> discoverHostsFull(Long dcId, Long podId, Long clusterId, String clusterName, String url, String username, String password, String hypervisorType, List<String> hostTags,
             Map<String, String> params) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException {
         URI uri = null;

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/storage/StorageManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java
index a768610..7ecc3ec 100755
--- a/server/src/com/cloud/storage/StorageManagerImpl.java
+++ b/server/src/com/cloud/storage/StorageManagerImpl.java
@@ -151,11 +151,13 @@ import com.cloud.storage.dao.StoragePoolWorkDao;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VMTemplateHostDao;
 import com.cloud.storage.dao.VMTemplatePoolDao;
+import com.cloud.storage.dao.VMTemplateS3Dao;
 import com.cloud.storage.dao.VMTemplateSwiftDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.dao.VolumeHostDao;
 import com.cloud.storage.download.DownloadMonitor;
 import com.cloud.storage.listener.StoragePoolMonitor;
+import com.cloud.storage.s3.S3Manager;
 import com.cloud.storage.secondary.SecondaryStorageVmManager;
 import com.cloud.storage.snapshot.SnapshotManager;
 import com.cloud.storage.snapshot.SnapshotScheduler;
@@ -262,6 +264,10 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
     @Inject
     protected VMTemplateSwiftDao _vmTemplateSwiftDao = null;
     @Inject
+    protected VMTemplateS3Dao _vmTemplateS3Dao;
+    @Inject
+    protected S3Manager _s3Mgr;
+    @Inject
     protected VMTemplateDao _vmTemplateDao = null;
     @Inject
     protected StoragePoolHostDao _poolHostDao = null;
@@ -707,6 +713,8 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
         try {
             if (snapshot.getSwiftId() != null && snapshot.getSwiftId() != 0) {
                 _snapshotMgr.downloadSnapshotsFromSwift(snapshot);
+            } else if (snapshot.getS3Id() != null && snapshot.getS3Id() != 0) {
+                _snapshotMgr.downloadSnapshotsFromS3(snapshot);
             }
             CreateVolumeFromSnapshotCommand createVolumeFromSnapshotCommand = new CreateVolumeFromSnapshotCommand(primaryStoragePoolNameLabel, secondaryStoragePoolUrl, dcId, accountId, volumeId,
                     backedUpSnapshotUuid, snapshot.getName(), _createVolumeFromSnapshotWait);
@@ -2983,6 +2991,14 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
         if (tsvs != null && tsvs.size() > 0) {
             size = tsvs.get(0).getSize();
         }
+
+        if (size == null && _s3Mgr.isS3Enabled()) {
+            VMTemplateS3VO vmTemplateS3VO = _vmTemplateS3Dao.findOneByTemplateId(template.getId());
+            if (vmTemplateS3VO != null) {
+                size = vmTemplateS3VO.getSize();
+            }
+        }
+
         if (size == null) {
             List<VMTemplateHostVO> sss = _vmTemplateHostDao.search(sc, null);
             if (sss == null || sss.size() == 0) {

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/storage/dao/S3Dao.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/dao/S3Dao.java b/server/src/com/cloud/storage/dao/S3Dao.java
new file mode 100644
index 0000000..ebea353
--- /dev/null
+++ b/server/src/com/cloud/storage/dao/S3Dao.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.dao;
+
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.storage.S3VO;
+import com.cloud.utils.db.GenericDao;
+
+public interface S3Dao extends GenericDao<S3VO, Long> {
+
+    S3TO getS3TO(final Long id);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/storage/dao/S3DaoImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/dao/S3DaoImpl.java b/server/src/com/cloud/storage/dao/S3DaoImpl.java
new file mode 100644
index 0000000..6162e6e
--- /dev/null
+++ b/server/src/com/cloud/storage/dao/S3DaoImpl.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.dao;
+
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.storage.S3VO;
+import com.cloud.utils.db.GenericDaoBase;
+
+import javax.ejb.Local;
+
+@Local(S3Dao.class)
+public class S3DaoImpl extends GenericDaoBase<S3VO, Long> implements S3Dao {
+
+    @Override
+    public S3TO getS3TO(final Long id) {
+
+        if (id != null) {
+
+            final S3VO s3VO = findById(id);
+            if (s3VO != null) {
+                return s3VO.toS3TO();
+            }
+
+        }
+
+        // NOTE: Excluded listAll / shuffle operation implemented in SwiftDaoImpl ...
+
+        return null;
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/storage/dao/VMTemplateDao.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/dao/VMTemplateDao.java b/server/src/com/cloud/storage/dao/VMTemplateDao.java
index f5b6913..1284ba1 100755
--- a/server/src/com/cloud/storage/dao/VMTemplateDao.java
+++ b/server/src/com/cloud/storage/dao/VMTemplateDao.java
@@ -71,4 +71,6 @@ public interface VMTemplateDao extends GenericDao<VMTemplateVO, Long> {
     List<Long> listPrivateTemplatesByHost(Long hostId);
     public Long countTemplatesForAccount(long accountId);
 	
+    List<VMTemplateVO> findTemplatesToSyncToS3();
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java b/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java
index 2a0dfc8..5c71f1b 100755
--- a/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java
+++ b/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java
@@ -92,6 +92,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
     
     private final String SELECT_TEMPLATE_SWIFT_REF = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, "
             + "t.checksum, t.display_text, t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type FROM vm_template t";
+
+    private static final String SELECT_S3_CANDIDATE_TEMPLATES = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, " +
+        "t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, t.checksum, t.display_text, " +
+        "t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type " +
+        "FROM vm_template t JOIN template_host_ref r ON t.id=r.template_id JOIN host h ON h.id=r.host_id " +
+        "WHERE t.hypervisor_type IN (SELECT hypervisor_type FROM host) AND r.download_state = 'DOWNLOADED' AND " +
+        "r.template_id NOT IN (SELECT template_id FROM template_s3_ref) AND r.destroyed = 0 AND t.type <> 'PERHOST'";
+
     protected SearchBuilder<VMTemplateVO> TemplateNameSearch;
     protected SearchBuilder<VMTemplateVO> UniqueNameSearch;
     protected SearchBuilder<VMTemplateVO> tmpltTypeSearch;
@@ -917,5 +925,10 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
 	            (accountType == Account.ACCOUNT_TYPE_DOMAIN_ADMIN) ||
 	            (accountType == Account.ACCOUNT_TYPE_READ_ONLY_ADMIN));
 	}
-    
+
+    @Override
+    public List<VMTemplateVO> findTemplatesToSyncToS3() {
+        return executeList(SELECT_S3_CANDIDATE_TEMPLATES, new Object[] {});
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java b/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java
new file mode 100644
index 0000000..d36fb3a
--- /dev/null
+++ b/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.dao;
+
+import com.cloud.storage.VMTemplateS3VO;
+import com.cloud.utils.db.GenericDao;
+
+import java.util.List;
+
+public interface VMTemplateS3Dao extends GenericDao<VMTemplateS3VO, Long> {
+
+    List<VMTemplateS3VO> listByS3Id(long id);
+
+    VMTemplateS3VO findOneByTemplateId(long id);
+
+    VMTemplateS3VO findOneByS3Template(long s3Id, long templateId);
+
+    void expungeAllByTemplateId(long templateId);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java b/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java
new file mode 100644
index 0000000..f23b803
--- /dev/null
+++ b/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.dao;
+
+import static com.cloud.utils.db.SearchCriteria.Op.*;
+import static com.cloud.storage.VMTemplateS3VO.*;
+
+import com.cloud.storage.VMTemplateS3VO;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+
+import javax.ejb.Local;
+import java.util.List;
+
+@Local(VMTemplateS3Dao.class)
+public class VMTemplateS3DaoImpl extends GenericDaoBase<VMTemplateS3VO, Long>
+        implements VMTemplateS3Dao {
+
+    private final SearchBuilder<VMTemplateS3VO> searchBuilder;
+
+    public VMTemplateS3DaoImpl() {
+
+        super();
+
+        this.searchBuilder = createSearchBuilder();
+        this.searchBuilder
+                .and(S3_ID_COLUMN_NAME, this.searchBuilder.entity().getS3Id(),
+                        EQ)
+                .and(TEMPLATE_ID_COLUMN_NAME,
+                        this.searchBuilder.entity().getTemplateId(), EQ).done();
+
+    }
+
+    @Override
+    public List<VMTemplateS3VO> listByS3Id(final long s3id) {
+
+        final SearchCriteria<VMTemplateS3VO> criteria = this.searchBuilder
+                .create();
+
+        criteria.setParameters(S3_ID_COLUMN_NAME, s3id);
+
+        return this.listBy(criteria);
+
+    }
+
+    @Override
+    public VMTemplateS3VO findOneByTemplateId(final long templateId) {
+
+        final SearchCriteria<VMTemplateS3VO> criteria = this.searchBuilder
+                .create();
+
+        criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId);
+
+        return this.findOneBy(criteria);
+
+    }
+
+    @Override
+    public VMTemplateS3VO findOneByS3Template(final long s3Id,
+            final long templateId) {
+
+        final SearchCriteria<VMTemplateS3VO> criteria = this.searchBuilder
+                .create();
+
+        criteria.setParameters(S3_ID_COLUMN_NAME, s3Id);
+        criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId);
+
+        return this.findOneBy(criteria);
+
+    }
+
+    @Override
+    public void expungeAllByTemplateId(long templateId) {
+
+        final SearchCriteria<VMTemplateS3VO> criteria = this.searchBuilder
+                .create();
+
+        criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId);
+
+        this.expunge(criteria);
+
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b70c1a5a/server/src/com/cloud/storage/s3/S3Manager.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/s3/S3Manager.java b/server/src/com/cloud/storage/s3/S3Manager.java
new file mode 100644
index 0000000..357f2ae
--- /dev/null
+++ b/server/src/com/cloud/storage/s3/S3Manager.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.s3;
+
+import java.util.List;
+
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.api.commands.AddS3Cmd;
+import com.cloud.api.commands.ListS3sCmd;
+import com.cloud.dc.DataCenterVO;
+import com.cloud.exception.DiscoveryException;
+import com.cloud.storage.S3;
+import com.cloud.storage.S3VO;
+import com.cloud.storage.VMTemplateS3VO;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.utils.component.Manager;
+
+public interface S3Manager extends Manager {
+
+    S3TO getS3TO();
+
+    S3TO getS3TO(Long s3Id);
+
+    S3 addS3(AddS3Cmd addS3Cmd) throws DiscoveryException;
+
+    Long chooseZoneForTemplateExtract(VMTemplateVO template);
+
+    boolean isS3Enabled();
+
+    boolean isTemplateInstalled(Long templateId);
+
+    void deleteTemplate(final Long accountId, final Long templateId);
+
+    String downloadTemplateFromS3ToSecondaryStorage(final long dcId,
+            final long templateId, final int primaryStorageDownloadWait);
+
+    List<S3VO> listS3s(ListS3sCmd listS3sCmd);
+
+    VMTemplateS3VO findByTemplateId(Long templateId);
+
+    void propagateTemplatesToZone(DataCenterVO zone);
+
+    void propagateTemplateToAllZones(VMTemplateS3VO vmTemplateS3VO);
+
+    void uploadTemplateToS3FromSecondaryStorage(final VMTemplateVO template);
+
+}


Mime
View raw message