cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mc...@apache.org
Subject git commit: updated refs/heads/master to 25acfba
Date Thu, 17 Oct 2013 17:31:07 GMT
Updated Branches:
  refs/heads/master 85f83a4e8 -> 25acfbad7


CLOUDSTACK-4816:provide configurable option to choose single vs
multipart upload to S3 object storage based on object size.



Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/25acfbad
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/25acfbad
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/25acfbad

Branch: refs/heads/master
Commit: 25acfbad78d0adc4ccf798f31d685be9c068c254
Parents: 85f83a4
Author: Min Chen <min.chen@citrix.com>
Authored: Thu Oct 17 10:28:59 2013 -0700
Committer: Min Chen <min.chen@citrix.com>
Committed: Thu Oct 17 10:28:59 2013 -0700

----------------------------------------------------------------------
 api/src/com/cloud/agent/api/to/S3TO.java        | 31 +++++++++++++++-----
 .../storage/template/S3TemplateDownloader.java  |  3 +-
 .../driver/S3ImageStoreDriverImpl.java          | 12 ++++++--
 server/src/com/cloud/configuration/Config.java  |  3 +-
 .../resource/NfsSecondaryStorageResource.java   |  6 ++--
 setup/db/db/schema-420to421.sql                 |  3 +-
 utils/src/com/cloud/utils/S3Utils.java          |  4 +++
 7 files changed, 47 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25acfbad/api/src/com/cloud/agent/api/to/S3TO.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java
index ab08a69..ea7564d 100644
--- a/api/src/com/cloud/agent/api/to/S3TO.java
+++ b/api/src/com/cloud/agent/api/to/S3TO.java
@@ -40,6 +40,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
     private Date created;
     private boolean enableRRS;
     private boolean multipartEnabled;
+    private long maxSingleUploadSizeInBytes;
 
     public S3TO() {
 
@@ -51,7 +52,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
             final String secretKey, final String endPoint,
             final String bucketName, final Boolean httpsFlag,
             final Integer connectionTimeout, final Integer maxErrorRetry,
-            final Integer socketTimeout, final Date created, final boolean enableRRS, final
boolean multipart) {
+            final Integer socketTimeout, final Date created, final boolean enableRRS, final
long maxUploadSize) {
 
         super();
 
@@ -67,7 +68,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
         this.socketTimeout = socketTimeout;
         this.created = created;
         this.enableRRS = enableRRS;
-        this.multipartEnabled = multipart;
+        this.maxSingleUploadSizeInBytes = maxUploadSize;
 
     }
 
@@ -278,14 +279,28 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO
{
         this.enableRRS = enableRRS;
     }
 
-    public boolean isMultipartEnabled() {
-        return multipartEnabled;
+    public long getMaxSingleUploadSizeInBytes() {
+        return maxSingleUploadSizeInBytes;
     }
 
-    public void setMultipartEnabled(boolean multipartEnabled) {
-        this.multipartEnabled = multipartEnabled;
+    public void setMaxSingleUploadSizeInBytes(long maxSingleUploadSizeInBytes) {
+        this.maxSingleUploadSizeInBytes = maxSingleUploadSizeInBytes;
     }
 
-
-
+    public boolean getSingleUpload(long objSize){
+        if ( maxSingleUploadSizeInBytes < 0 ){
+            // always use single part upload
+            return true;
+        } else if ( maxSingleUploadSizeInBytes == 0 ){
+            // always use multi part upload
+            return false;
+        } else {
+            // check object size to set flag
+            if (objSize < maxSingleUploadSizeInBytes){
+                return true;
+            } else{
+                return false;
+            }
+        }        
+    }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25acfbad/core/src/com/cloud/storage/template/S3TemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
index 462b21b..9dacbd3 100644
--- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
@@ -258,7 +258,8 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements
Temp
 
             });
             
-            if ( s3.isMultipartEnabled()){
+
+            if ( !s3.getSingleUpload(remoteSize) ){
                 // use TransferManager to do multipart upload
                 S3Utils.mputObject(s3, putObjectRequest);
             } else{

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25acfbad/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
index f31aea3..6b5175e 100644
--- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
+++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
@@ -67,13 +67,21 @@ public class S3ImageStoreDriverImpl extends  BaseImageStoreDriverImpl
{
                         .get(ApiConstants.S3_SOCKET_TIMEOUT)), imgStore.getCreated(),
                 _configDao.getValue(Config.S3EnableRRS.toString()) == null ? false : Boolean.parseBoolean(_configDao
                         .getValue(Config.S3EnableRRS.toString())),
-                _configDao.getValue(Config.S3EnableMultiPartUpload.toString()) == null ?
true : Boolean.parseBoolean(_configDao
-                                .getValue(Config.S3EnableMultiPartUpload.toString()))   
                    
+                getMaxSingleUploadSizeInBytes()                      
                 );
 
     }
 
 
+    private long getMaxSingleUploadSizeInBytes() {
+        try {
+            return Long.parseLong(_configDao.getValue(Config.S3MaxSingleUploadSize.toString()))
* 1024L * 1024L * 1024L;
+        } catch (NumberFormatException e) {
+            // use default 5GB
+            return 5L * 1024L * 1024L * 1024L;
+        }
+    }
+    
     @Override
     public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat
format, DataObject dataObject) {
         // for S3, no need to do anything, just return template url for

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25acfbad/server/src/com/cloud/configuration/Config.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java
index 9d94bba..c2aa2e4 100755
--- a/server/src/com/cloud/configuration/Config.java
+++ b/server/src/com/cloud/configuration/Config.java
@@ -377,7 +377,8 @@ public enum Config {
 
     // object store
     S3EnableRRS("Advanced", ManagementServer.class, Boolean.class, "s3.rrs.enabled", "false",
"enable s3 reduced redundancy storage", null),
-    S3EnableMultiPartUpload("Advanced", ManagementServer.class, Boolean.class, "s3.multipart.enabled",
"true", "enable s3 multipart upload", null),
+    S3MaxSingleUploadSize("Advanced",  ManagementServer.class, Integer.class, "s3.singleupload.max.size",
"5", "The maximum size limit for S3 single part upload API(in GB). If it is set to 0, then
it means always use multi-part upload to upload object to S3. " +
+            "If it is set to -1, then it means always use single-part upload to upload object
to S3. ", null),
 
     // Ldap
     LdapBasedn("Advanced", ManagementServer.class, String.class, "ldap.basedn", null, "Sets
the basedn for LDAP", null),

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25acfbad/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index 85d25f9..75d959b 100755
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -857,10 +857,12 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
                     }
                 }
             }
+
+            long srcSize = srcFile.length();
             ImageFormat format = getTemplateFormat(srcFile.getName());
             String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName();
-            if (s3.isMultipartEnabled()){
-                mputFile(s3, srcFile, bucket, key);
+            if (!s3.getSingleUpload(srcSize)){
+                mputFile(s3, srcFile, bucket, key); 
             } else{
                 putFile(s3, srcFile, bucket, key);
             }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25acfbad/setup/db/db/schema-420to421.sql
----------------------------------------------------------------------
diff --git a/setup/db/db/schema-420to421.sql b/setup/db/db/schema-420to421.sql
index 4dfe4e6..abe97a4 100644
--- a/setup/db/db/schema-420to421.sql
+++ b/setup/db/db/schema-420to421.sql
@@ -20,7 +20,8 @@
 --;
 
 
-INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server',
's3.multipart.enabled', 'true', 'enable s3 multipart upload');
+INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server',
's3.singleupload.max.size', '5', 
+    'The maximum size limit for S3 single part upload API(in GB). If it is set to 0, then
it means always use multi-part upload to upload object to S3. If it is set to -1, then it
means always use single-part upload to upload object to S3.');
 
 INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Storage", 'DEFAULT', 'management-server',
"enable.ha.storage.migration", "true", "Enable/disable storage migration across primary storage
during HA"); 
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25acfbad/utils/src/com/cloud/utils/S3Utils.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java
index ce4d4b7..7918a20 100644
--- a/utils/src/com/cloud/utils/S3Utils.java
+++ b/utils/src/com/cloud/utils/S3Utils.java
@@ -171,6 +171,10 @@ public final class S3Utils {
         assert clientOptions != null;
         assert req != null;
 
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(format("Sending stream as S3 object using PutObjectRequest"));
+        }
+        
         acquireClient(clientOptions).putObject(req);
 
     }


Mime
View raw message