cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject [2/3] git commit: updated refs/heads/master to 7665bdc
Date Fri, 20 Nov 2015 21:28:07 GMT
CLOUDSTACK-9062: Improve S3 implementation.

The S3 implementation is far from finished, this commit focusses on the bases.

 - Upgrade AWS SDK to latest version.
 - Rewrite S3 Template downloader.
 - Rewrite S3Utils utility class.
 - Improve addImageStoreS3 API command.
 - Split various classes for convenience.
 - Various minor improvements and code optimalisations.

A side effect of the new AWS SDK is that it, by default, uses the V4 signature. Therefore I added an option to specify the Signer, so it stays compatible with previous versions.


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/5c0366c9
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/5c0366c9
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/5c0366c9

Branch: refs/heads/master
Commit: 5c0366c99e60a1c2e3d4d03045026fce55165be1
Parents: fe2917e
Author: Boris Schrijver <boris@pcextreme.nl>
Authored: Fri Nov 13 02:19:24 2015 +0100
Committer: Boris Schrijver <boris@pcextreme.nl>
Committed: Thu Nov 19 15:29:10 2015 +0100

----------------------------------------------------------------------
 agent/conf/log4j-cloud.xml.in                   |  12 +-
 api/src/com/cloud/agent/api/storage/Proxy.java  |  72 ---
 api/src/com/cloud/agent/api/to/S3TO.java        | 215 +++----
 api/src/com/cloud/storage/StorageService.java   |  24 +-
 .../org/apache/cloudstack/api/ApiConstants.java |   3 +
 .../admin/storage/AddImageStoreS3CMD.java       | 276 +++++++++
 .../api/command/admin/storage/AddS3Cmd.java     | 264 --------
 .../api/command/admin/storage/ListS3sCmd.java   |  55 --
 .../template/RegisterTemplateCmdByAdmin.java    |   2 +-
 client/tomcatconf/commands.properties.in        |   5 +-
 client/tomcatconf/log4j-cloud.xml.in            |  10 +
 core/src/com/cloud/resource/ServerResource.java |   2 +-
 .../template/HttpTemplateDownloader.java        |   2 +-
 .../storage/template/S3TemplateDownloader.java  | 483 ++++++---------
 .../storage/template/TemplateDownloader.java    |  33 +-
 .../storage/command/DownloadCommand.java        |   2 +-
 .../api/storage/DataStoreProvider.java          |  16 +-
 .../storage/image/TemplateServiceImpl.java      |   2 +-
 .../datastore/ObjectInDataStoreManager.java     |  10 +-
 .../storage/image/BaseImageStoreDriverImpl.java |   2 +-
 framework/rest/pom.xml                          |  10 +-
 packaging/centos7/tomcat7/log4j-cloud.xml       |  10 +
 .../kvm/storage/KVMStorageProcessor.java        |  18 +-
 .../resource/XenServerStorageProcessor.java     |   4 +-
 plugins/storage/image/s3/pom.xml                |   2 +-
 .../driver/S3ImageStoreDriverImpl.java          |  31 +-
 .../lifecycle/S3ImageStoreLifeCycleImpl.java    |   6 +-
 .../provider/S3ImageStoreProviderImpl.java      |   7 +-
 pom.xml                                         |   3 +-
 scripts/vm/hypervisor/xenserver/s3xenserver     |   2 +-
 server/conf/log4j-cloud.xml.in                  |  10 +
 .../com/cloud/server/ManagementServerImpl.java  |   6 +-
 .../com/cloud/storage/StorageManagerImpl.java   |  25 +-
 .../storage/download/DownloadMonitorImpl.java   |   2 +-
 .../resource/NfsSecondaryStorageResource.java   |  30 +-
 .../resource/SecondaryStorageResource.java      |   3 +-
 .../SecondaryStorageResourceHandler.java        |   2 +
 .../storage/template/DownloadManager.java       |   2 +-
 .../storage/template/DownloadManagerImpl.java   |   2 +-
 systemvm/conf.dom0/log4j-cloud.xml.in           |  10 +
 systemvm/conf/log4j-cloud.xml                   |  10 +
 utils/pom.xml                                   |   5 +
 .../src/main/java/com/cloud/utils/S3Utils.java  | 619 -------------------
 .../java/com/cloud/utils/net/HTTPUtils.java     | 143 +++++
 .../main/java/com/cloud/utils/net/Proxy.java    |  69 +++
 .../cloud/utils/storage/S3/ClientOptions.java   |  42 ++
 .../utils/storage/S3/FileNamingStrategy.java    |  25 +
 .../utils/storage/S3/ObjectNamingStrategy.java  |  27 +
 .../com/cloud/utils/storage/S3/S3Utils.java     | 216 +++++++
 .../utils/rest/RESTServiceConnectorTest.java    |   4 +-
 50 files changed, 1300 insertions(+), 1535 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/agent/conf/log4j-cloud.xml.in
----------------------------------------------------------------------
diff --git a/agent/conf/log4j-cloud.xml.in b/agent/conf/log4j-cloud.xml.in
index 9e6c646..1222801 100644
--- a/agent/conf/log4j-cloud.xml.in
+++ b/agent/conf/log4j-cloud.xml.in
@@ -77,7 +77,17 @@ under the License.
    </category>
    
    <category name="net">
-     <priority value="INFO"/>
+      <priority value="INFO"/>
+   </category>
+
+   <!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
+   <category name="com.amazonaws">
+      <priority value="INFO"/>
+   </category>
+
+   <!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
+   <category name="httpclient.wire">
+      <priority value="INFO"/>
    </category>
 
    <!-- ======================= -->

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/com/cloud/agent/api/storage/Proxy.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/storage/Proxy.java b/api/src/com/cloud/agent/api/storage/Proxy.java
deleted file mode 100644
index 5adc114..0000000
--- a/api/src/com/cloud/agent/api/storage/Proxy.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package com.cloud.agent.api.storage;
-
-import java.net.URI;
-
-/**
- * Download Proxy
- */
-public class Proxy {
-    private String _host;
-    private int _port;
-    private String _userName;
-    private String _password;
-
-    public Proxy() {
-
-    }
-
-    public Proxy(String host, int port, String userName, String password) {
-        this._host = host;
-        this._port = port;
-        this._userName = userName;
-        this._password = password;
-    }
-
-    public Proxy(URI uri) {
-        this._host = uri.getHost();
-        this._port = uri.getPort() == -1 ? 3128 : uri.getPort();
-        String userInfo = uri.getUserInfo();
-        if (userInfo != null) {
-            String[] tokens = userInfo.split(":");
-            if (tokens.length == 1) {
-                this._userName = userInfo;
-                this._password = "";
-            } else if (tokens.length == 2) {
-                this._userName = tokens[0];
-                this._password = tokens[1];
-            }
-        }
-    }
-
-    public String getHost() {
-        return _host;
-    }
-
-    public int getPort() {
-        return _port;
-    }
-
-    public String getUserName() {
-        return _userName;
-    }
-
-    public String getPassword() {
-        return _password;
-    }
-}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/com/cloud/agent/api/to/S3TO.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java
index a46d609..ec6bc02 100644
--- a/api/src/com/cloud/agent/api/to/S3TO.java
+++ b/api/src/com/cloud/agent/api/to/S3TO.java
@@ -21,9 +21,9 @@ import java.util.Date;
 import com.cloud.agent.api.LogLevel;
 import com.cloud.agent.api.LogLevel.Log4jLevel;
 import com.cloud.storage.DataStoreRole;
-import com.cloud.utils.S3Utils;
+import com.cloud.utils.storage.S3.ClientOptions;
 
-public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
+public final class S3TO implements ClientOptions, DataStoreTO {
 
     private Long id;
     private String uuid;
@@ -33,6 +33,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
     private String secretKey;
     private String endPoint;
     private String bucketName;
+    private String signer;
     private Boolean httpsFlag;
     private Boolean useTCPKeepAlive;
     private Integer connectionTimeout;
@@ -44,17 +45,9 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
     private long maxSingleUploadSizeInBytes;
     private static final String pathSeparator = "/";
 
-    public S3TO() {
-
-        super();
-
-    }
-
     public S3TO(final Long id, final String uuid, final String accessKey, final String secretKey, final String endPoint, final String bucketName,
-            final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, final Integer socketTimeout, final Date created,
-            final boolean enableRRS, final long maxUploadSize, final Integer connectionTtl, final Boolean useTCPKeepAlive) {
-
-        super();
+            final String signer, final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, final Integer socketTimeout,
+            final Date created, final boolean enableRRS, final long maxUploadSize, final Integer connectionTtl, final Boolean useTCPKeepAlive) {
 
         this.id = id;
         this.uuid = uuid;
@@ -62,6 +55,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
         this.secretKey = secretKey;
         this.endPoint = endPoint;
         this.bucketName = bucketName;
+        this.signer = signer;
         this.httpsFlag = httpsFlag;
         this.connectionTimeout = connectionTimeout;
         this.maxErrorRetry = maxErrorRetry;
@@ -74,98 +68,6 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
 
     }
 
-    @Override
-    public boolean equals(final Object thatObject) {
-
-        if (this == thatObject) {
-            return true;
-        }
-        if (thatObject == null || getClass() != thatObject.getClass()) {
-            return false;
-        }
-
-        final S3TO thatS3TO = (S3TO)thatObject;
-
-        if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) : thatS3TO.httpsFlag != null) {
-            return false;
-        }
-
-        if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) : thatS3TO.accessKey != null) {
-            return false;
-        }
-
-        if (connectionTimeout != null ? !connectionTimeout.equals(thatS3TO.connectionTimeout) : thatS3TO.connectionTimeout != null) {
-            return false;
-        }
-
-        if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) : thatS3TO.endPoint != null) {
-            return false;
-        }
-
-        if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) {
-            return false;
-        }
-
-        if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) {
-            return false;
-        }
-
-        if (maxErrorRetry != null ? !maxErrorRetry.equals(thatS3TO.maxErrorRetry) : thatS3TO.maxErrorRetry != null) {
-            return false;
-        }
-
-        if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) : thatS3TO.secretKey != null) {
-            return false;
-        }
-
-        if (socketTimeout != null ? !socketTimeout.equals(thatS3TO.socketTimeout) : thatS3TO.socketTimeout != null) {
-            return false;
-        }
-
-        if (connectionTtl != null ? !connectionTtl.equals(thatS3TO.connectionTtl) : thatS3TO.connectionTtl != null) {
-            return false;
-        }
-
-        if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatS3TO.useTCPKeepAlive) : thatS3TO.useTCPKeepAlive != null) {
-            return false;
-        }
-
-        if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) : thatS3TO.bucketName != null) {
-            return false;
-        }
-
-        if (created != null ? !created.equals(thatS3TO.created) : thatS3TO.created != null) {
-            return false;
-        }
-
-        if (enableRRS != thatS3TO.enableRRS) {
-            return false;
-        }
-
-        return true;
-
-    }
-
-    @Override
-    public int hashCode() {
-
-        int result = id != null ? id.hashCode() : 0;
-
-        result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0);
-        result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
-        result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
-        result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
-        result = 31 * result + (httpsFlag ? 1 : 0);
-        result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
-        result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
-        result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
-        result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
-        result = 31 * result + (useTCPKeepAlive ? 1 : 0);
-
-        return result;
-
-    }
-
     public Long getId() {
         return this.id;
     }
@@ -224,6 +126,15 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
     }
 
     @Override
+    public String getSigner() {
+        return this.signer;
+    }
+
+    public void setSigner(final String signer) {
+        this.signer = signer;
+    }
+
+    @Override
     public Boolean isHttps() {
         return this.httpsFlag;
     }
@@ -327,4 +238,100 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
     public String getPathSeparator() {
         return pathSeparator;
     }
+
+    @Override
+    public boolean equals(final Object thatObject) {
+
+        if (this == thatObject) {
+            return true;
+        }
+        if (thatObject == null || getClass() != thatObject.getClass()) {
+            return false;
+        }
+
+        final S3TO thatS3TO = (S3TO)thatObject;
+
+        if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) : thatS3TO.httpsFlag != null) {
+            return false;
+        }
+
+        if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) : thatS3TO.accessKey != null) {
+            return false;
+        }
+
+        if (connectionTimeout != null ? !connectionTimeout.equals(thatS3TO.connectionTimeout) : thatS3TO.connectionTimeout != null) {
+            return false;
+        }
+
+        if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) : thatS3TO.endPoint != null) {
+            return false;
+        }
+
+        if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) {
+            return false;
+        }
+
+        if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) {
+            return false;
+        }
+
+        if (maxErrorRetry != null ? !maxErrorRetry.equals(thatS3TO.maxErrorRetry) : thatS3TO.maxErrorRetry != null) {
+            return false;
+        }
+
+        if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) : thatS3TO.secretKey != null) {
+            return false;
+        }
+
+        if (socketTimeout != null ? !socketTimeout.equals(thatS3TO.socketTimeout) : thatS3TO.socketTimeout != null) {
+            return false;
+        }
+
+        if (connectionTtl != null ? !connectionTtl.equals(thatS3TO.connectionTtl) : thatS3TO.connectionTtl != null) {
+            return false;
+        }
+
+        if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatS3TO.useTCPKeepAlive) : thatS3TO.useTCPKeepAlive != null) {
+            return false;
+        }
+
+        if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) : thatS3TO.bucketName != null) {
+            return false;
+        }
+
+        if (signer != null ? !signer.equals(thatS3TO.signer) : thatS3TO.signer != null) {
+            return false;
+        }
+
+        if (created != null ? !created.equals(thatS3TO.created) : thatS3TO.created != null) {
+            return false;
+        }
+
+        if (enableRRS != thatS3TO.enableRRS) {
+            return false;
+        }
+
+        return true;
+
+    }
+
+    @Override
+    public int hashCode() {
+
+        int result = id != null ? id.hashCode() : 0;
+
+        result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0);
+        result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
+        result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
+        result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
+        result = 31 * result + (signer != null ? signer.hashCode() : 0);
+        result = 31 * result + (httpsFlag ? 1 : 0);
+        result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
+        result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
+        result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
+        result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
+        result = 31 * result + (useTCPKeepAlive ? 1 : 0);
+
+        return result;
+    }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/com/cloud/storage/StorageService.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/storage/StorageService.java b/api/src/com/cloud/storage/StorageService.java
index 7fc1e16..e40b1e6 100644
--- a/api/src/com/cloud/storage/StorageService.java
+++ b/api/src/com/cloud/storage/StorageService.java
@@ -14,6 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
 package com.cloud.storage;
 
 import java.net.UnknownHostException;
@@ -38,14 +39,14 @@ public interface StorageService {
      * Create StoragePool based on uri
      *
      * @param cmd
-     *            the command object that specifies the zone, cluster/pod, URI, details, etc. to use to create the
+     *            The command object that specifies the zone, cluster/pod, URI, details, etc. to use to create the
      *            storage pool.
      * @return
+     *            The StoragePool created.
      * @throws ResourceInUseException
      * @throws IllegalArgumentException
      * @throws UnknownHostException
      * @throws ResourceUnavailableException
-     *             TODO
      */
     StoragePool createPool(CreateStoragePoolCmd cmd) throws ResourceInUseException, IllegalArgumentException, UnknownHostException, ResourceUnavailableException;
 
@@ -63,15 +64,13 @@ public interface StorageService {
     /**
      * Enable maintenance for primary storage
      *
-     * @param cmd
-     *            - the command specifying primaryStorageId
+     * @param primaryStorageId
+     *            - the primaryStorageId
      * @return the primary storage pool
      * @throws ResourceUnavailableException
-     *             TODO
      * @throws InsufficientCapacityException
-     *             TODO
      */
-    public StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException;
+    StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException;
 
     /**
      * Complete maintenance for primary storage
@@ -80,19 +79,18 @@ public interface StorageService {
      *            - the command specifying primaryStorageId
      * @return the primary storage pool
      * @throws ResourceUnavailableException
-     *             TODO
      */
-    public StoragePool cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException;
+    StoragePool cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException;
 
-    public StoragePool updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException;
+    StoragePool updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException;
 
-    public StoragePool getStoragePool(long id);
+    StoragePool getStoragePool(long id);
 
     boolean deleteImageStore(DeleteImageStoreCmd cmd);
 
     boolean deleteSecondaryStagingStore(DeleteSecondaryStagingStoreCmd cmd);
 
-    public ImageStore discoverImageStore(String name, String url, String providerName, Long dcId, Map details) throws IllegalArgumentException, DiscoveryException,
+    ImageStore discoverImageStore(String name, String url, String providerName, Long zoneId, Map details) throws IllegalArgumentException, DiscoveryException,
             InvalidParameterValueException;
 
 
@@ -107,7 +105,7 @@ public interface StorageService {
      * @throws DiscoveryException
      * @throws InvalidParameterValueException
      */
-    public ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException,
+    ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException,
             InvalidParameterValueException;
 
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/ApiConstants.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java
index 2728bcc..5365e14 100644
--- a/api/src/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/org/apache/cloudstack/api/ApiConstants.java
@@ -501,6 +501,9 @@ public class ApiConstants {
     public static final String S3_SECRET_KEY = "secretkey";
     public static final String S3_END_POINT = "endpoint";
     public static final String S3_BUCKET_NAME = "bucket";
+    public static final String S3_SIGNER = "s3signer";
+    public static final String S3_V3_SIGNER = "S3SignerType";
+    public static final String S3_V4_SIGNER = "AWSS3V4SignerType";
     public static final String S3_HTTPS_FLAG = "usehttps";
     public static final String S3_CONNECTION_TIMEOUT = "connectiontimeout";
     public static final String S3_CONNECTION_TTL = "connectionttl";

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
new file mode 100644
index 0000000..34ff171
--- /dev/null
+++ b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.api.command.admin.storage;
+
+import static com.cloud.user.Account.ACCOUNT_ID_SYSTEM;
+import static org.apache.cloudstack.api.ApiConstants.S3_ACCESS_KEY;
+import static org.apache.cloudstack.api.ApiConstants.S3_BUCKET_NAME;
+import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TIMEOUT;
+import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TTL;
+import static org.apache.cloudstack.api.ApiConstants.S3_END_POINT;
+import static org.apache.cloudstack.api.ApiConstants.S3_HTTPS_FLAG;
+import static org.apache.cloudstack.api.ApiConstants.S3_MAX_ERROR_RETRY;
+import static org.apache.cloudstack.api.ApiConstants.S3_SIGNER;
+import static org.apache.cloudstack.api.ApiConstants.S3_SECRET_KEY;
+import static org.apache.cloudstack.api.ApiConstants.S3_SOCKET_TIMEOUT;
+import static org.apache.cloudstack.api.ApiConstants.S3_USE_TCP_KEEPALIVE;
+import static org.apache.cloudstack.api.BaseCmd.CommandType.BOOLEAN;
+import static org.apache.cloudstack.api.BaseCmd.CommandType.INTEGER;
+import static org.apache.cloudstack.api.BaseCmd.CommandType.STRING;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.cloud.utils.storage.S3.ClientOptions;
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.ImageStoreResponse;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.DiscoveryException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.storage.ImageStore;
+
+@APICommand(name = "addImageStoreS3", description = "Adds S3 Image Store", responseObject = ImageStoreResponse.class, since = "4.7.0",
+        requestHasSensitiveInfo = true, responseHasSensitiveInfo = false)
+public final class AddImageStoreS3CMD extends BaseCmd implements ClientOptions {
+    public static final Logger s_logger = Logger.getLogger(AddImageStoreS3CMD.class.getName());
+
+    private static final String s_name = "addImageStoreS3Response";
+
+    @Parameter(name = S3_ACCESS_KEY, type = STRING, required = true, description = "S3 access key")
+    private String accessKey;
+
+    @Parameter(name = S3_SECRET_KEY, type = STRING, required = true, description = "S3 secret key")
+    private String secretKey;
+
+    @Parameter(name = S3_END_POINT, type = STRING, required = true, description = "S3 endpoint")
+    private String endPoint;
+
+    @Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, description = "Name of the storage bucket")
+    private String bucketName;
+
+    @Parameter(name = S3_SIGNER, type = STRING, required = false, description = "Signer Algorithm to use, either S3SignerType or AWSS3V4SignerType")
+    private String signer;
+
+    @Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, description = "Use HTTPS instead of HTTP")
+    private Boolean httpsFlag;
+
+    @Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, description = "Connection timeout (milliseconds)")
+    private Integer connectionTimeout;
+
+    @Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, description = "Maximum number of times to retry on error")
+    private Integer maxErrorRetry;
+
+    @Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, description = "Socket timeout (milliseconds)")
+    private Integer socketTimeout;
+
+    @Parameter(name = S3_CONNECTION_TTL, type = INTEGER, required = false, description = "Connection TTL (milliseconds)")
+    private Integer connectionTtl;
+
+    @Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, required = false, description = "Whether TCP keep-alive is used")
+    private Boolean useTCPKeepAlive;
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
+        ResourceAllocationException, NetworkRuleConflictException {
+
+        Map<String, String> dm = new HashMap();
+
+        dm.put(ApiConstants.S3_ACCESS_KEY, getAccessKey());
+        dm.put(ApiConstants.S3_SECRET_KEY, getSecretKey());
+        dm.put(ApiConstants.S3_END_POINT, getEndPoint());
+        dm.put(ApiConstants.S3_BUCKET_NAME, getBucketName());
+
+        if (getSigner() != null && (getSigner().equals(ApiConstants.S3_V3_SIGNER) || getSigner().equals(ApiConstants.S3_V4_SIGNER))) {
+            dm.put(ApiConstants.S3_SIGNER, getSigner());
+        }
+        if (isHttps() != null) {
+            dm.put(ApiConstants.S3_HTTPS_FLAG, isHttps().toString());
+        }
+        if (getConnectionTimeout() != null) {
+            dm.put(ApiConstants.S3_CONNECTION_TIMEOUT, getConnectionTimeout().toString());
+        }
+        if (getMaxErrorRetry() != null) {
+            dm.put(ApiConstants.S3_MAX_ERROR_RETRY, getMaxErrorRetry().toString());
+        }
+        if (getSocketTimeout() != null) {
+            dm.put(ApiConstants.S3_SOCKET_TIMEOUT, getSocketTimeout().toString());
+        }
+        if (getConnectionTtl() != null) {
+            dm.put(ApiConstants.S3_CONNECTION_TTL, getConnectionTtl().toString());
+        }
+        if (getUseTCPKeepAlive() != null) {
+            dm.put(ApiConstants.S3_USE_TCP_KEEPALIVE, getUseTCPKeepAlive().toString());
+        }
+
+        try{
+            ImageStore result = _storageService.discoverImageStore(null, null, "S3", null, dm);
+            ImageStoreResponse storeResponse;
+            if (result != null) {
+                storeResponse = _responseGenerator.createImageStoreResponse(result);
+                storeResponse.setResponseName(getCommandName());
+                storeResponse.setObjectName("imagestore");
+                setResponseObject(storeResponse);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 Image Store.");
+            }
+        } catch (DiscoveryException ex) {
+            s_logger.warn("Exception: ", ex);
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
+        }
+    }
+
+    @Override
+    public String getCommandName() {
+        return s_name;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return ACCOUNT_ID_SYSTEM;
+    }
+
+    public String getAccessKey() {
+        return accessKey;
+    }
+
+    public String getSecretKey() {
+        return secretKey;
+    }
+
+    public String getEndPoint() {
+        return endPoint;
+    }
+
+    public String getBucketName() {
+        return bucketName;
+    }
+
+    public String getSigner() {
+        return signer;
+    }
+
+    public Boolean isHttps() {
+        return httpsFlag;
+    }
+
+    public Integer getConnectionTimeout() {
+        return connectionTimeout;
+    }
+
+    public Integer getMaxErrorRetry() {
+        return maxErrorRetry;
+    }
+
+    public Integer getSocketTimeout() {
+        return socketTimeout;
+    }
+
+    public Integer getConnectionTtl() {
+        return connectionTtl;
+    }
+
+    public Boolean getUseTCPKeepAlive() {
+        return useTCPKeepAlive;
+    }
+
+    @Override
+    public boolean equals(final Object thatObject) {
+
+        if (this == thatObject) {
+            return true;
+        }
+
+        if (thatObject == null || this.getClass() != thatObject.getClass()) {
+            return false;
+        }
+
+        final AddImageStoreS3CMD thatAddS3Cmd = (AddImageStoreS3CMD)thatObject;
+
+        if (httpsFlag != null ? !httpsFlag.equals(thatAddS3Cmd.httpsFlag) : thatAddS3Cmd.httpsFlag != null) {
+            return false;
+        }
+
+        if (accessKey != null ? !accessKey.equals(thatAddS3Cmd.accessKey) : thatAddS3Cmd.accessKey != null) {
+            return false;
+        }
+
+        if (connectionTimeout != null ? !connectionTimeout.equals(thatAddS3Cmd.connectionTimeout) : thatAddS3Cmd.connectionTimeout != null) {
+            return false;
+        }
+
+        if (endPoint != null ? !endPoint.equals(thatAddS3Cmd.endPoint) : thatAddS3Cmd.endPoint != null) {
+            return false;
+        }
+
+        if (maxErrorRetry != null ? !maxErrorRetry.equals(thatAddS3Cmd.maxErrorRetry) : thatAddS3Cmd.maxErrorRetry != null) {
+            return false;
+        }
+
+        if (secretKey != null ? !secretKey.equals(thatAddS3Cmd.secretKey) : thatAddS3Cmd.secretKey != null) {
+            return false;
+        }
+
+        if (socketTimeout != null ? !socketTimeout.equals(thatAddS3Cmd.socketTimeout) : thatAddS3Cmd.socketTimeout != null) {
+            return false;
+        }
+
+        if (bucketName != null ? !bucketName.equals(thatAddS3Cmd.bucketName) : thatAddS3Cmd.bucketName != null) {
+            return false;
+        }
+
+        if (connectionTtl != null ? !connectionTtl.equals(thatAddS3Cmd.connectionTtl) : thatAddS3Cmd.connectionTtl != null) {
+            return false;
+        }
+
+        if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatAddS3Cmd.useTCPKeepAlive) : thatAddS3Cmd.useTCPKeepAlive != null) {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+
+        int result = accessKey != null ? accessKey.hashCode() : 0;
+        result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
+        result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
+        result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
+        result = 31 * result + (signer != null ? signer.hashCode() : 0);
+        result = 31 * result + (httpsFlag != null && httpsFlag ? 1 : 0);
+        result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
+        result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
+        result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
+        result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
+        result = 31 * result + (useTCPKeepAlive != null && useTCPKeepAlive ? 1 : 0);
+
+        return result;
+    }
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java
deleted file mode 100644
index d54cb75..0000000
--- a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cloudstack.api.command.admin.storage;
-
-import static com.cloud.user.Account.ACCOUNT_ID_SYSTEM;
-import static org.apache.cloudstack.api.ApiConstants.S3_ACCESS_KEY;
-import static org.apache.cloudstack.api.ApiConstants.S3_BUCKET_NAME;
-import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TIMEOUT;
-import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TTL;
-import static org.apache.cloudstack.api.ApiConstants.S3_END_POINT;
-import static org.apache.cloudstack.api.ApiConstants.S3_HTTPS_FLAG;
-import static org.apache.cloudstack.api.ApiConstants.S3_MAX_ERROR_RETRY;
-import static org.apache.cloudstack.api.ApiConstants.S3_SECRET_KEY;
-import static org.apache.cloudstack.api.ApiConstants.S3_SOCKET_TIMEOUT;
-import static org.apache.cloudstack.api.ApiConstants.S3_USE_TCP_KEEPALIVE;
-import static org.apache.cloudstack.api.BaseCmd.CommandType.BOOLEAN;
-import static org.apache.cloudstack.api.BaseCmd.CommandType.INTEGER;
-import static org.apache.cloudstack.api.BaseCmd.CommandType.STRING;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.log4j.Logger;
-
-import org.apache.cloudstack.api.APICommand;
-import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.ApiErrorCode;
-import org.apache.cloudstack.api.BaseCmd;
-import org.apache.cloudstack.api.Parameter;
-import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.ImageStoreResponse;
-
-import com.cloud.exception.ConcurrentOperationException;
-import com.cloud.exception.DiscoveryException;
-import com.cloud.exception.InsufficientCapacityException;
-import com.cloud.exception.NetworkRuleConflictException;
-import com.cloud.exception.ResourceAllocationException;
-import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.storage.ImageStore;
-
-@APICommand(name = "addS3", description = "Adds S3", responseObject = ImageStoreResponse.class, since = "4.0.0",
-        requestHasSensitiveInfo = true, responseHasSensitiveInfo = false)
-public final class AddS3Cmd extends BaseCmd {
-    public static final Logger s_logger = Logger.getLogger(AddS3Cmd.class.getName());
-
-    private static final String s_name = "adds3response";
-
-    @Parameter(name = S3_ACCESS_KEY, type = STRING, required = true, description = "S3 access key")
-    private String accessKey;
-
-    @Parameter(name = S3_SECRET_KEY, type = STRING, required = true, description = "S3 secret key")
-    private String secretKey;
-
-    @Parameter(name = S3_END_POINT, type = STRING, required = false, description = "S3 host name")
-    private String endPoint;
-
-    @Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, description = "name of the template storage bucket")
-    private String bucketName;
-
-    @Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, description = "connect to the S3 endpoint via HTTPS?")
-    private Boolean httpsFlag;
-
-    @Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, description = "connection timeout (milliseconds)")
-    private Integer connectionTimeout;
-
-    @Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, description = "maximum number of times to retry on error")
-    private Integer maxErrorRetry;
-
-    @Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, description = "socket timeout (milliseconds)")
-    private Integer socketTimeout;
-
-    @Parameter(name = S3_CONNECTION_TTL, type = INTEGER, required = false, description = "connection ttl (milliseconds)")
-    private Integer connectionTtl;
-
-    @Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, required = false, description = "whether tcp keepalive is used")
-    private Boolean useTCPKeepAlive;
-
-    @Override
-    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
-        ResourceAllocationException, NetworkRuleConflictException {
-
-        Map<String, String> dm = new HashMap<String, String>();
-        dm.put(ApiConstants.S3_ACCESS_KEY, getAccessKey());
-        dm.put(ApiConstants.S3_SECRET_KEY, getSecretKey());
-        dm.put(ApiConstants.S3_END_POINT, getEndPoint());
-        dm.put(ApiConstants.S3_BUCKET_NAME, getBucketName());
-        if (getHttpsFlag() != null) {
-            dm.put(ApiConstants.S3_HTTPS_FLAG, getHttpsFlag().toString());
-        }
-        if (getConnectionTimeout() != null) {
-            dm.put(ApiConstants.S3_CONNECTION_TIMEOUT, getConnectionTimeout().toString());
-        }
-        if (getMaxErrorRetry() != null) {
-            dm.put(ApiConstants.S3_MAX_ERROR_RETRY, getMaxErrorRetry().toString());
-        }
-        if (getSocketTimeout() != null) {
-            dm.put(ApiConstants.S3_SOCKET_TIMEOUT, getSocketTimeout().toString());
-        }
-        if (getConnectionTtl() != null) {
-            dm.put(ApiConstants.S3_CONNECTION_TTL, getConnectionTtl().toString());
-        }
-        if (getUseTCPKeepAlive() != null) {
-            dm.put(ApiConstants.S3_USE_TCP_KEEPALIVE, getUseTCPKeepAlive().toString());
-        }
-
-
-        try{
-            ImageStore result = _storageService.discoverImageStore(null, null, "S3", null, dm);
-            ImageStoreResponse storeResponse = null;
-            if (result != null) {
-                storeResponse = _responseGenerator.createImageStoreResponse(result);
-                storeResponse.setResponseName(getCommandName());
-                storeResponse.setObjectName("secondarystorage");
-                setResponseObject(storeResponse);
-            } else {
-                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 secondary storage");
-            }
-        } catch (DiscoveryException ex) {
-            s_logger.warn("Exception: ", ex);
-            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
-        }
-    }
-
-    @Override
-    public boolean equals(final Object thatObject) {
-
-        if (this == thatObject) {
-            return true;
-        }
-
-        if (thatObject == null || this.getClass() != thatObject.getClass()) {
-            return false;
-        }
-
-        final AddS3Cmd thatAddS3Cmd = (AddS3Cmd)thatObject;
-
-        if (httpsFlag != null ? !httpsFlag.equals(thatAddS3Cmd.httpsFlag) : thatAddS3Cmd.httpsFlag != null) {
-            return false;
-        }
-
-        if (accessKey != null ? !accessKey.equals(thatAddS3Cmd.accessKey) : thatAddS3Cmd.accessKey != null) {
-            return false;
-        }
-
-        if (connectionTimeout != null ? !connectionTimeout.equals(thatAddS3Cmd.connectionTimeout) : thatAddS3Cmd.connectionTimeout != null) {
-            return false;
-        }
-
-        if (endPoint != null ? !endPoint.equals(thatAddS3Cmd.endPoint) : thatAddS3Cmd.endPoint != null) {
-            return false;
-        }
-
-        if (maxErrorRetry != null ? !maxErrorRetry.equals(thatAddS3Cmd.maxErrorRetry) : thatAddS3Cmd.maxErrorRetry != null) {
-            return false;
-        }
-
-        if (secretKey != null ? !secretKey.equals(thatAddS3Cmd.secretKey) : thatAddS3Cmd.secretKey != null) {
-            return false;
-        }
-
-        if (socketTimeout != null ? !socketTimeout.equals(thatAddS3Cmd.socketTimeout) : thatAddS3Cmd.socketTimeout != null) {
-            return false;
-        }
-
-        if (bucketName != null ? !bucketName.equals(thatAddS3Cmd.bucketName) : thatAddS3Cmd.bucketName != null) {
-            return false;
-        }
-
-        if (connectionTtl != null ? !connectionTtl.equals(thatAddS3Cmd.connectionTtl) : thatAddS3Cmd.connectionTtl != null) {
-            return false;
-        }
-
-        if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatAddS3Cmd.useTCPKeepAlive) : thatAddS3Cmd.useTCPKeepAlive != null) {
-            return false;
-        }
-
-        return true;
-
-    }
-
-    @Override
-    public int hashCode() {
-
-        int result = accessKey != null ? accessKey.hashCode() : 0;
-        result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
-        result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
-        result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
-        result = 31 * result + (httpsFlag != null && httpsFlag == true ? 1 : 0);
-        result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
-        result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
-        result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
-        result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
-        result = 31 * result + (useTCPKeepAlive != null && useTCPKeepAlive == true ? 1 : 0);
-
-        return result;
-
-    }
-
-    @Override
-    public String getCommandName() {
-        return s_name;
-    }
-
-    @Override
-    public long getEntityOwnerId() {
-        return ACCOUNT_ID_SYSTEM;
-    }
-
-    public String getAccessKey() {
-        return accessKey;
-    }
-
-    public String getSecretKey() {
-        return secretKey;
-    }
-
-    public String getEndPoint() {
-        return endPoint;
-    }
-
-    public String getBucketName() {
-        return bucketName;
-    }
-
-    public Boolean getHttpsFlag() {
-        return httpsFlag;
-    }
-
-    public Integer getConnectionTimeout() {
-        return connectionTimeout;
-    }
-
-    public Integer getMaxErrorRetry() {
-        return maxErrorRetry;
-    }
-
-    public Integer getSocketTimeout() {
-        return socketTimeout;
-    }
-
-    public Integer getConnectionTtl() {
-        return connectionTtl;
-    }
-
-    public Boolean getUseTCPKeepAlive() {
-        return useTCPKeepAlive;
-    }
-}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java
deleted file mode 100644
index c1889e7..0000000
--- a/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cloudstack.api.command.admin.storage;
-
-import org.apache.cloudstack.api.APICommand;
-import org.apache.cloudstack.api.BaseListCmd;
-import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.ImageStoreResponse;
-import org.apache.cloudstack.api.response.ListResponse;
-
-import com.cloud.exception.ConcurrentOperationException;
-import com.cloud.exception.InsufficientCapacityException;
-import com.cloud.exception.NetworkRuleConflictException;
-import com.cloud.exception.ResourceAllocationException;
-import com.cloud.exception.ResourceUnavailableException;
-
-@APICommand(name = "listS3s", description = "Lists S3s", responseObject = ImageStoreResponse.class, since = "4.0.0",
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class ListS3sCmd extends BaseListCmd {
-
-    private static final String COMMAND_NAME = "lists3sresponse";
-
-    @Override
-    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
-        ResourceAllocationException, NetworkRuleConflictException {
-
-        ListImageStoresCmd cmd = new ListImageStoresCmd();
-        cmd.setProvider("S3");
-        ListResponse<ImageStoreResponse> response = _queryService.searchForImageStores(cmd);
-        response.setResponseName(getCommandName());
-        this.setResponseObject(response);
-    }
-
-    @Override
-    public String getCommandName() {
-        return COMMAND_NAME;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java b/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
index 659d1c5..68d53f9 100644
--- a/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
+++ b/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
@@ -32,7 +32,7 @@ import org.apache.cloudstack.api.response.TemplateResponse;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.template.VirtualMachineTemplate;
 
-@APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud. ", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
+@APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud.", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
 public class RegisterTemplateCmdByAdmin extends RegisterTemplateCmd {
     public static final Logger s_logger = Logger.getLogger(RegisterTemplateCmdByAdmin.class.getName());

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/client/tomcatconf/commands.properties.in
----------------------------------------------------------------------
diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in
index 4f93b97..1c788a8 100644
--- a/client/tomcatconf/commands.properties.in
+++ b/client/tomcatconf/commands.properties.in
@@ -282,12 +282,9 @@ listCapacity=3
 addSwift=1
 listSwifts=1
 
-#### s3 commands
-addS3=1
-listS3s=1
-
 #### image store commands
 addImageStore=1
+addImageStoreS3=1
 listImageStores=1
 deleteImageStore=1
 createSecondaryStagingStore=1

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/client/tomcatconf/log4j-cloud.xml.in
----------------------------------------------------------------------
diff --git a/client/tomcatconf/log4j-cloud.xml.in b/client/tomcatconf/log4j-cloud.xml.in
index 587aa86..0a12e7d 100755
--- a/client/tomcatconf/log4j-cloud.xml.in
+++ b/client/tomcatconf/log4j-cloud.xml.in
@@ -162,6 +162,16 @@ under the License.
       <appender-ref ref="APISERVER"/>
    </logger>
 
+   <!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
+   <category name="com.amazonaws">
+      <priority value="INFO"/>
+   </category>
+
+   <!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
+   <category name="httpclient.wire">
+      <priority value="INFO"/>
+   </category>
+
    <!-- ============================== -->
    <!-- Add or remove these logger for SNMP, this logger is for SNMP alerts plugin -->
    <!-- ============================== -->

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/com/cloud/resource/ServerResource.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/resource/ServerResource.java b/core/src/com/cloud/resource/ServerResource.java
index 1bbcbff..9030db7 100644
--- a/core/src/com/cloud/resource/ServerResource.java
+++ b/core/src/com/cloud/resource/ServerResource.java
@@ -28,7 +28,6 @@ import com.cloud.host.Host;
 import com.cloud.utils.component.Manager;
 
 /**
- *
  * ServerResource is a generic container to execute commands sent
  */
 public interface ServerResource extends Manager {
@@ -70,4 +69,5 @@ public interface ServerResource extends Manager {
     IAgentControl getAgentControl();
 
     void setAgentControl(IAgentControl agentControl);
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
index 632a809..76d1ac9 100644
--- a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
@@ -47,10 +47,10 @@ import org.apache.log4j.Logger;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
 
-import com.cloud.agent.api.storage.Proxy;
 import com.cloud.storage.StorageLayer;
 import com.cloud.utils.Pair;
 import com.cloud.utils.UriUtils;
+import com.cloud.utils.net.Proxy;
 
 /**
  * Download a template file using HTTP

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/com/cloud/storage/template/S3TemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
index ac47dec..d584cdf 100644
--- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
@@ -19,303 +19,235 @@
 
 package com.cloud.storage.template;
 
-import static com.cloud.utils.StringUtils.join;
-import static java.util.Arrays.asList;
-
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Date;
-
+import com.amazonaws.event.ProgressEvent;
+import com.amazonaws.event.ProgressEventType;
+import com.amazonaws.event.ProgressListener;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.StorageClass;
+import com.amazonaws.services.s3.transfer.Upload;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.utils.net.HTTPUtils;
+import com.cloud.utils.net.Proxy;
+import com.cloud.utils.storage.S3.S3Utils;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
-import org.apache.commons.httpclient.ChunkedInputStream;
-import org.apache.commons.httpclient.Credentials;
 import org.apache.commons.httpclient.Header;
 import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpMethodRetryHandler;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
-import org.apache.commons.httpclient.NoHttpResponseException;
-import org.apache.commons.httpclient.UsernamePasswordCredentials;
-import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.params.HttpMethodParams;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.ProgressEvent;
-import com.amazonaws.services.s3.model.ProgressListener;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.StorageClass;
-import com.cloud.agent.api.storage.Proxy;
-import com.cloud.agent.api.to.S3TO;
-import com.cloud.utils.Pair;
-import com.cloud.utils.S3Utils;
-import com.cloud.utils.UriUtils;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Date;
+
+import static com.cloud.utils.StringUtils.join;
+import static java.util.Arrays.asList;
 
 /**
  * Download a template file using HTTP(S)
+ *
+ * This class, once instantiated, has the purpose to download a single Template to an S3 Image Store.
+ *
+ * Execution of the instance is started when runInContext() is called.
  */
 public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader {
-    private static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
-    private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
-
-    private String downloadUrl;
-    private String installPath;
-    private String s3Key;
-    private String fileName;
-    private String fileExtension;
-    private String errorString = " ";
-
+    private static final Logger LOGGER = Logger.getLogger(S3TemplateDownloader.class.getName());
+
+    private final String downloadUrl;
+    private final String s3Key;
+    private final String fileExtension;
+    private final HttpClient httpClient;
+    private final GetMethod getMethod;
+    private final DownloadCompleteCallback downloadCompleteCallback;
+    private final S3TO s3TO;
+    private String errorString = "";
     private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
     private ResourceType resourceType = ResourceType.TEMPLATE;
-    private final HttpClient client;
-    private final HttpMethodRetryHandler myretryhandler;
-    private GetMethod request;
-    private DownloadCompleteCallback completionCallback;
-    private S3TO s3to;
-
-    private long remoteSize = 0;
-    private long downloadTime = 0;
+    private long remoteSize;
+    private long downloadTime;
     private long totalBytes;
     private long maxTemplateSizeInByte;
 
     private boolean resume = false;
-    private boolean inited = true;
 
-    public S3TemplateDownloader(S3TO s3to, String downloadUrl, String installPath, DownloadCompleteCallback callback,
-            long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) {
-        this.s3to = s3to;
+    public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
+            long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
         this.downloadUrl = downloadUrl;
-        this.installPath = installPath;
-        this.status = TemplateDownloader.Status.NOT_STARTED;
+        this.s3TO = s3TO;
         this.resourceType = resourceType;
         this.maxTemplateSizeInByte = maxTemplateSizeInBytes;
+        this.httpClient = HTTPUtils.getHTTPClient();
+        this.downloadCompleteCallback = downloadCompleteCallback;
 
-        this.totalBytes = 0;
-        this.client = new HttpClient(s_httpClientManager);
+        // Create a GET method for the download url.
+        this.getMethod = new GetMethod(downloadUrl);
 
-        this.myretryhandler = new HttpMethodRetryHandler() {
-            @Override
-            public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
-                if (executionCount >= 2) {
-                    // Do not retry if over max retry count
-                    return false;
-                }
-                if (exception instanceof NoHttpResponseException) {
-                    // Retry if the server dropped connection on us
-                    return true;
-                }
-                if (!method.isRequestSent()) {
-                    // Retry if the request has not been sent fully or
-                    // if it's OK to retry methods that have been sent
-                    return true;
-                }
-                // otherwise do not retry
-                return false;
-            }
-        };
+        // Set the retry handler, default to retry 5 times.
+        this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
 
-        try {
-            request = new GetMethod(downloadUrl);
-            request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
-            completionCallback = callback;
-
-            Pair<String, Integer> hostAndPort = UriUtils.validateUrl(downloadUrl);
-            fileName = StringUtils.substringAfterLast(downloadUrl, "/");
-            fileExtension = StringUtils.substringAfterLast(fileName, ".");
-
-            if (proxy != null) {
-                client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
-                if (proxy.getUserName() != null) {
-                    Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword());
-                    client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds);
-                }
-            }
-            if ((user != null) && (password != null)) {
-                client.getParams().setAuthenticationPreemptive(true);
-                Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
-                client.getState().setCredentials(
-                        new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
-                s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first()
-                        + ":" + hostAndPort.second());
-            } else {
-                s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
-            }
-        } catch (IllegalArgumentException iae) {
-            errorString = iae.getMessage();
-            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-            inited = false;
-        } catch (Exception ex) {
-            errorString = "Unable to start download -- check url? ";
-            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-            s_logger.warn("Exception in constructor -- " + ex.toString());
-        } catch (Throwable th) {
-            s_logger.warn("throwable caught ", th);
-        }
+        // Follow redirects
+        this.getMethod.setFollowRedirects(true);
+
+        // Set file extension.
+        this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
+
+        // Calculate and set S3 Key.
+        this.s3Key = join(asList(installPath, StringUtils.substringAfterLast(downloadUrl, "/")), S3Utils.SEPARATOR);
+
+        // Set proxy if available.
+        HTTPUtils.setProxy(proxy, this.httpClient);
+
+        // Set credentials if available.
+        HTTPUtils.setCredentials(username, password, this.httpClient);
     }
 
     @Override
     public long download(boolean resume, DownloadCompleteCallback callback) {
-        switch (status) {
-        case ABORTED:
-        case UNRECOVERABLE_ERROR:
-        case DOWNLOAD_FINISHED:
+        if (!status.equals(Status.NOT_STARTED)) {
+            // Only start downloading if we haven't started yet.
+            LOGGER.debug("Template download is already started, not starting again. Template: " + downloadUrl);
+
+            return 0;
+        }
+
+        int responseCode;
+        if ((responseCode = HTTPUtils.executeMethod(httpClient, getMethod)) == -1) {
+            errorString = "Exception while executing HttpMethod " + getMethod.getName() + " on URL " + downloadUrl;
+            LOGGER.warn(errorString);
+
+            status = Status.UNRECOVERABLE_ERROR;
             return 0;
-        default:
+        }
+
+        if (!HTTPUtils.verifyResponseCode(responseCode)) {
+            errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
+            LOGGER.warn(errorString);
 
+            status = Status.UNRECOVERABLE_ERROR;
+            return 0;
         }
 
+        // Headers
+        Header contentLengthHeader = getMethod.getResponseHeader("Content-Length");
+        Header contentTypeHeader = getMethod.getResponseHeader("Content-Type");
+
+        // Check the contentLengthHeader and transferEncodingHeader.
+        if (contentLengthHeader == null) {
+            errorString = "The ContentLengthHeader of " + downloadUrl + " isn't supplied";
+            LOGGER.warn(errorString);
+
+            status = Status.UNRECOVERABLE_ERROR;
+            return 0;
+        } else {
+            // The ContentLengthHeader is supplied, parse it's value.
+            remoteSize = Long.parseLong(contentLengthHeader.getValue());
+        }
+
+        if (remoteSize > maxTemplateSizeInByte) {
+            errorString = "Remote size is too large for template " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxTemplateSizeInByte;
+            LOGGER.warn(errorString);
+
+            status = Status.UNRECOVERABLE_ERROR;
+            return 0;
+        }
+
+        InputStream inputStream;
+
         try {
-            // execute get method
-            int responseCode = HttpStatus.SC_OK;
-            if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
-                status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-                errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
-                return 0; // FIXME: retry?
-            }
-            // get the total size of file
-            Header contentLengthHeader = request.getResponseHeader("Content-Length");
-            boolean chunked = false;
-            long remoteSize2 = 0;
-            if (contentLengthHeader == null) {
-                Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
-                if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
-                    status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-                    errorString = " Failed to receive length of download ";
-                    return 0; // FIXME: what status do we put here? Do we retry?
-                } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
-                    chunked = true;
-                }
-            } else {
-                remoteSize2 = Long.parseLong(contentLengthHeader.getValue());
-            }
+            inputStream = new BufferedInputStream(getMethod.getResponseBodyAsStream());
+        } catch (IOException e) {
+            errorString = "Exception occurred while opening InputStream for template " + downloadUrl;
+            LOGGER.warn(errorString);
 
-            if (remoteSize == 0) {
-                remoteSize = remoteSize2;
-            }
+            status = Status.UNRECOVERABLE_ERROR;
+            return 0;
+        }
 
-            if (remoteSize > maxTemplateSizeInByte) {
-                s_logger.info("Remote size is too large: " + remoteSize + " , max=" + maxTemplateSizeInByte);
-                status = Status.UNRECOVERABLE_ERROR;
-                errorString = "Download file size is too large";
-                return 0;
-            }
+        LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + remoteSize + " bytes");
 
-            if (remoteSize == 0) {
-                remoteSize = maxTemplateSizeInByte;
-            }
+        // Time the upload starts.
+        final Date start = new Date();
 
-            // get content type
-            String contentType = null;
-            Header contentTypeHeader = request.getResponseHeader("Content-Type");
-            if (contentTypeHeader != null) {
-                contentType = contentTypeHeader.getValue();
-            }
+        ObjectMetadata objectMetadata = new ObjectMetadata();
+        objectMetadata.setContentLength(remoteSize);
 
-            InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream())
-                    : new ChunkedInputStream(request.getResponseBodyAsStream());
+        if (contentTypeHeader.getValue() != null) {
+            objectMetadata.setContentType(contentTypeHeader.getValue());
+        }
 
-            s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3to.getBucketName()
-                    + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte);
+        // Create the PutObjectRequest.
+        PutObjectRequest putObjectRequest = new PutObjectRequest(s3TO.getBucketName(), s3Key, inputStream, objectMetadata);
 
-            Date start = new Date();
-            // compute s3 key
-            s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR);
+        // If reduced redundancy is enabled, set it.
+        if (s3TO.getEnableRRS()) {
+            putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
+        }
 
-            // download using S3 API
-            ObjectMetadata metadata = new ObjectMetadata();
-            metadata.setContentLength(remoteSize);
-            if (contentType != null) {
-                metadata.setContentType(contentType);
-            }
-            PutObjectRequest putObjectRequest = new PutObjectRequest(s3to.getBucketName(), s3Key, in, metadata);
-            // check if RRS is enabled
-            if (s3to.getEnableRRS()) {
-                putObjectRequest = putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
-            }
-            // register progress listenser
-            putObjectRequest.setProgressListener(new ProgressListener() {
-                @Override
-                public void progressChanged(ProgressEvent progressEvent) {
-                    // s_logger.debug(progressEvent.getBytesTransfered()
-                    // + " number of byte transferd "
-                    // + new Date());
-                    totalBytes += progressEvent.getBytesTransfered();
-                    if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) {
-                        s_logger.info("download completed");
-                        status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
-                    } else if (progressEvent.getEventCode() == ProgressEvent.FAILED_EVENT_CODE) {
-                        status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-                    } else if (progressEvent.getEventCode() == ProgressEvent.CANCELED_EVENT_CODE) {
-                        status = TemplateDownloader.Status.ABORTED;
-                    } else {
-                        status = TemplateDownloader.Status.IN_PROGRESS;
-                    }
-                }
+        Upload upload = S3Utils.putObject(s3TO, putObjectRequest);
+
+        upload.addProgressListener(new ProgressListener() {
+            @Override
+            public void progressChanged(ProgressEvent progressEvent) {
+
+                // Record the amount of bytes transferred.
+                totalBytes += progressEvent.getBytesTransferred();
+
+                LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + totalBytes + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds");
 
-            });
-
-            if (!s3to.getSingleUpload(remoteSize)) {
-                // use TransferManager to do multipart upload
-                S3Utils.mputObject(s3to, putObjectRequest);
-            } else {
-                // single part upload, with 5GB limit in Amazon
-                S3Utils.putObject(s3to, putObjectRequest);
-                while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED
-                        && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR
-                        && status != TemplateDownloader.Status.ABORTED) {
-                    // wait for completion
+                if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) {
+                    status = Status.IN_PROGRESS;
+                } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
+                    status = Status.DOWNLOAD_FINISHED;
+                } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_CANCELED_EVENT) {
+                    status = Status.ABORTED;
+                } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT) {
+                    status = Status.UNRECOVERABLE_ERROR;
                 }
             }
+        });
 
-            // finished or aborted
-            Date finish = new Date();
-            String downloaded = "(incomplete download)";
-            if (totalBytes >= remoteSize) {
-                status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
-                downloaded = "(download complete remote=" + remoteSize + "bytes)";
-            } else {
-                errorString = "Downloaded " + totalBytes + " bytes " + downloaded;
-            }
-            downloadTime += finish.getTime() - start.getTime();
-            return totalBytes;
-        } catch (HttpException hte) {
-            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-            errorString = hte.getMessage();
-        } catch (IOException ioe) {
-            // probably a file write error
-            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-            errorString = ioe.getMessage();
-        } catch (AmazonClientException ex) {
-            // S3 api exception
-            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-            errorString = ex.getMessage();
+        try {
+            // Wait for the upload to complete.
+            upload.waitForCompletion();
         } catch (InterruptedException e) {
-            // S3 upload api exception
-            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-            errorString = e.getMessage();
-        } finally {
-            // close input stream
-            request.releaseConnection();
-            if (callback != null) {
-                callback.downloadComplete(status);
-            }
+            // Interruption while waiting for the upload to complete.
+            LOGGER.warn("Interruption occurred while waiting for upload of " + downloadUrl + " to complete");
         }
-        return 0;
+
+        downloadTime = new Date().getTime() - start.getTime();
+
+        if (status == Status.DOWNLOAD_FINISHED) {
+            LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!");
+        } else {
+            LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
+        }
+
+        // Close input stream
+        getMethod.releaseConnection();
+
+        // Call the callback!
+        if (callback != null) {
+            callback.downloadComplete(status);
+        }
+
+        return totalBytes;
     }
 
     public String getDownloadUrl() {
-        return downloadUrl;
+        try {
+            return getMethod.getURI().toString();
+        } catch (URIException e) {
+            return null;
+        }
     }
 
     @Override
-    public TemplateDownloader.Status getStatus() {
+    public Status getStatus() {
         return status;
     }
 
@@ -342,47 +274,38 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
             return null;
         }
 
-        return S3Utils.getObjectStream(s3to, s3to.getBucketName(), s3Key);
+        return S3Utils.getObjectStream(s3TO, s3TO.getBucketName(), s3Key);
     }
 
     public void cleanupAfterError() {
-        if (status != Status.UNRECOVERABLE_ERROR) {
-            s_logger.debug("S3Template downloader does not have state UNRECOVERABLE_ERROR, no cleanup neccesarry.");
-            return;
-        }
-
-        s_logger.info("Cleanup after UNRECOVERABLE_ERROR, trying to remove object: " + s3Key);
+        LOGGER.warn("Cleanup after error, trying to remove object: " + s3Key);
 
-        S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
+        S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key);
     }
 
     @Override
-    @SuppressWarnings("fallthrough")
     public boolean stopDownload() {
-        switch (getStatus()) {
-        case IN_PROGRESS:
-            if (request != null) {
-                request.abort();
-            }
-            status = TemplateDownloader.Status.ABORTED;
-            return true;
-        case UNKNOWN:
-        case NOT_STARTED:
-        case RECOVERABLE_ERROR:
-        case UNRECOVERABLE_ERROR:
-        case ABORTED:
-            status = TemplateDownloader.Status.ABORTED;
-        case DOWNLOAD_FINISHED:
-            try {
-                S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
-            } catch (Exception ex) {
-                // ignore delete exception if it is not there
-            }
-            return true;
-
-        default:
-            return true;
+        switch (status) {
+            case IN_PROGRESS:
+                if (getMethod != null) {
+                    getMethod.abort();
+                }
+                break;
+            case UNKNOWN:
+            case NOT_STARTED:
+            case RECOVERABLE_ERROR:
+            case UNRECOVERABLE_ERROR:
+            case ABORTED:
+            case DOWNLOAD_FINISHED:
+                // Remove the object if it already has been uploaded.
+                S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key);
+                break;
+            default:
+                break;
         }
+
+        status = TemplateDownloader.Status.ABORTED;
+        return true;
     }
 
     @Override
@@ -396,18 +319,12 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
 
     @Override
     protected void runInContext() {
-        try {
-            download(resume, completionCallback);
-        } catch (Throwable t) {
-            s_logger.warn("Caught exception during download " + t.getMessage(), t);
-            errorString = "Failed to install: " + t.getMessage();
-            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
-        }
-
+        // Start the download!
+        download(resume, downloadCompleteCallback);
     }
 
     @Override
-    public void setStatus(TemplateDownloader.Status status) {
+    public void setStatus(Status status) {
         this.status = status;
     }
 
@@ -442,7 +359,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
 
     @Override
     public boolean isInited() {
-        return inited;
+        return true;
     }
 
     public ResourceType getResourceType() {

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/com/cloud/storage/template/TemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/TemplateDownloader.java b/core/src/com/cloud/storage/template/TemplateDownloader.java
index da9787d..5db3d24 100644
--- a/core/src/com/cloud/storage/template/TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/TemplateDownloader.java
@@ -25,16 +25,16 @@ public interface TemplateDownloader extends Runnable {
      * Callback used to notify completion of download
      *
      */
-    public interface DownloadCompleteCallback {
+    interface DownloadCompleteCallback {
         void downloadComplete(Status status);
 
     }
 
-    public static enum Status {
+    enum Status {
         UNKNOWN, NOT_STARTED, IN_PROGRESS, ABORTED, UNRECOVERABLE_ERROR, RECOVERABLE_ERROR, DOWNLOAD_FINISHED, POST_DOWNLOAD_FINISHED
     }
 
-    public static long DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES = 50L * 1024L * 1024L * 1024L;
+    long DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES = 50L * 1024L * 1024L * 1024L;
 
     /**
      * Initiate download, resuming a previous one if required
@@ -42,55 +42,54 @@ public interface TemplateDownloader extends Runnable {
      * @param callback completion callback to be called after download is complete
      * @return bytes downloaded
      */
-    public long download(boolean resume, DownloadCompleteCallback callback);
+    long download(boolean resume, DownloadCompleteCallback callback);
 
     /**
      * @return
      */
-    public boolean stopDownload();
+    boolean stopDownload();
 
     /**
      * @return percent of file downloaded
      */
-    public int getDownloadPercent();
+    int getDownloadPercent();
 
     /**
      * Get the status of the download
      * @return status of download
      */
-    public TemplateDownloader.Status getStatus();
+    TemplateDownloader.Status getStatus();
 
     /**
      * Get time taken to download so far
      * @return time in seconds taken to download
      */
-    public long getDownloadTime();
+    long getDownloadTime();
 
     /**
      * Get bytes downloaded
      * @return bytes downloaded so far
      */
-    public long getDownloadedBytes();
+    long getDownloadedBytes();
 
     /**
      * Get the error if any
      * @return error string if any
      */
-    public String getDownloadError();
+    String getDownloadError();
 
     /** Get local path of the downloaded file
      * @return local path of the file downloaded
      */
-    public String getDownloadLocalPath();
+    String getDownloadLocalPath();
 
-    public void setStatus(TemplateDownloader.Status status);
+    void setStatus(TemplateDownloader.Status status);
 
-    public void setDownloadError(String string);
+    void setDownloadError(String string);
 
-    public void setResume(boolean resume);
+    void setResume(boolean resume);
 
-    public boolean isInited();
-
-    public long getMaxTemplateSizeInBytes();
+    boolean isInited();
 
+    long getMaxTemplateSizeInBytes();
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
----------------------------------------------------------------------
diff --git a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
index 5dfc22e..29d737f 100644
--- a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
+++ b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
@@ -25,7 +25,7 @@ import org.apache.cloudstack.storage.to.VolumeObjectTO;
 
 import com.cloud.agent.api.storage.AbstractDownloadCommand;
 import com.cloud.agent.api.storage.PasswordAuth;
-import com.cloud.agent.api.storage.Proxy;
+import com.cloud.utils.net.Proxy;
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.NfsTO;
 import com.cloud.storage.Storage.ImageFormat;

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
index 7b5f8d9..3e5761e 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
@@ -23,14 +23,14 @@ import java.util.Set;
 
 public interface DataStoreProvider {
     // constants for provider names
-    static final String NFS_IMAGE = "NFS";
-    static final String S3_IMAGE = "S3";
-    static final String SWIFT_IMAGE = "Swift";
-    static final String SAMPLE_IMAGE = "Sample";
-    static final String SMB = "NFS";
-    static final String DEFAULT_PRIMARY = "DefaultPrimary";
-
-    static enum DataStoreProviderType {
+    String NFS_IMAGE = "NFS";
+    String S3_IMAGE = "S3";
+    String SWIFT_IMAGE = "Swift";
+    String SAMPLE_IMAGE = "Sample";
+    String SMB = "NFS";
+    String DEFAULT_PRIMARY = "DefaultPrimary";
+
+    enum DataStoreProviderType {
         PRIMARY, IMAGE, ImageCache
     }
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index 69dc7c7..9ab3595 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -196,7 +196,7 @@ public class TemplateServiceImpl implements TemplateService {
 
     @Override
     public void downloadBootstrapSysTemplate(DataStore store) {
-        Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
+        Set<VMTemplateVO> toBeDownloaded = new HashSet();
 
         List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
index 3dc6ac4..8f081d3 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
@@ -27,15 +27,15 @@ import com.cloud.storage.DataStoreRole;
 import com.cloud.utils.fsm.NoTransitionException;
 
 public interface ObjectInDataStoreManager {
-    public DataObject create(DataObject dataObj, DataStore dataStore);
+    DataObject create(DataObject dataObj, DataStore dataStore);
 
-    public boolean delete(DataObject dataObj);
+    boolean delete(DataObject dataObj);
 
-    public boolean deleteIfNotReady(DataObject dataObj);
+    boolean deleteIfNotReady(DataObject dataObj);
 
-    public DataObject get(DataObject dataObj, DataStore store);
+    DataObject get(DataObject dataObj, DataStore store);
 
-    public boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException;
+    boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException;
 
     DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role);
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
index 380040b..0068c2d 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
@@ -47,7 +47,6 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.DownloadAnswer;
-import com.cloud.agent.api.storage.Proxy;
 import com.cloud.agent.api.to.DataObjectType;
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.alert.AlertManager;
@@ -58,6 +57,7 @@ import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VMTemplateZoneDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.download.DownloadMonitor;
+import com.cloud.utils.net.Proxy;
 
 public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
     private static final Logger s_logger = Logger.getLogger(BaseImageStoreDriverImpl.class);


Mime
View raw message