Updated Branches: refs/heads/4.2 1a907eb4e -> d17024c09
CLOUDSTACK-4816:provide configurable option to choose single vs multipart upload to S3 object storage based on object size. Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/d17024c0 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/d17024c0 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/d17024c0 Branch: refs/heads/4.2 Commit: d17024c09baf4ecb564674dc272268cdc26bcca2 Parents: 1a907eb Author: Min Chen <[email protected]> Authored: Wed Oct 16 17:56:19 2013 -0700 Committer: Min Chen <[email protected]> Committed: Wed Oct 16 17:56:19 2013 -0700 ---------------------------------------------------------------------- api/src/com/cloud/agent/api/to/S3TO.java | 31 +++++++++++++++----- .../storage/template/S3TemplateDownloader.java | 3 +- .../driver/S3ImageStoreDriverImpl.java | 12 ++++++-- server/src/com/cloud/configuration/Config.java | 3 +- .../resource/NfsSecondaryStorageResource.java | 4 ++- setup/db/db/schema-420to421.sql | 3 +- utils/src/com/cloud/utils/S3Utils.java | 4 +++ 7 files changed, 46 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/api/src/com/cloud/agent/api/to/S3TO.java ---------------------------------------------------------------------- diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java index ab08a69..ea7564d 100644 --- a/api/src/com/cloud/agent/api/to/S3TO.java +++ b/api/src/com/cloud/agent/api/to/S3TO.java @@ -40,6 +40,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO { private Date created; private boolean enableRRS; private boolean multipartEnabled; + private long maxSingleUploadSizeInBytes; public S3TO() { @@ -51,7 +52,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO { final String secretKey, final String endPoint, final String bucketName, final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, - final Integer socketTimeout, final Date created, final boolean enableRRS, final boolean multipart) { + final Integer socketTimeout, final Date created, final boolean enableRRS, final long maxUploadSize) { super(); @@ -67,7 +68,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO { this.socketTimeout = socketTimeout; this.created = created; this.enableRRS = enableRRS; - this.multipartEnabled = multipart; + this.maxSingleUploadSizeInBytes = maxUploadSize; } @@ -278,14 +279,28 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO { this.enableRRS = enableRRS; } - public boolean isMultipartEnabled() { - return multipartEnabled; + public long getMaxSingleUploadSizeInBytes() { + return maxSingleUploadSizeInBytes; } - public void setMultipartEnabled(boolean multipartEnabled) { - this.multipartEnabled = multipartEnabled; + public void setMaxSingleUploadSizeInBytes(long maxSingleUploadSizeInBytes) { + this.maxSingleUploadSizeInBytes = maxSingleUploadSizeInBytes; } - - + public boolean getSingleUpload(long objSize){ + if ( maxSingleUploadSizeInBytes < 0 ){ + // always use single part upload + return true; + } else if ( maxSingleUploadSizeInBytes == 0 ){ + // always use multi part upload + return false; + } else { + // check object size to set flag + if (objSize < maxSingleUploadSizeInBytes){ + return true; + } else{ + return false; + } + } + } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/core/src/com/cloud/storage/template/S3TemplateDownloader.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java index c394368..9c2b628 100644 --- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java +++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java @@ -255,7 +255,8 @@ public class S3TemplateDownloader implements TemplateDownloader { }); - if ( s3.isMultipartEnabled()){ + + if ( !s3.getSingleUpload(remoteSize) ){ // use TransferManager to do multipart upload S3Utils.mputObject(s3, putObjectRequest); } else{ http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java ---------------------------------------------------------------------- diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java index 117beee..e0afe51 100644 --- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java +++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java @@ -65,13 +65,21 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl { .get(ApiConstants.S3_SOCKET_TIMEOUT)), imgStore.getCreated(), _configDao.getValue(Config.S3EnableRRS.toString()) == null ? false : Boolean.parseBoolean(_configDao .getValue(Config.S3EnableRRS.toString())), - _configDao.getValue(Config.S3EnableMultiPartUpload.toString()) == null ? true : Boolean.parseBoolean(_configDao - .getValue(Config.S3EnableMultiPartUpload.toString())) + getMaxSingleUploadSizeInBytes() ); } + private long getMaxSingleUploadSizeInBytes() { + try { + return Long.parseLong(_configDao.getValue(Config.S3MaxSingleUploadSize.toString())) * 1024L * 1024L * 1024L; + } catch (NumberFormatException e) { + // use default 5GB + return 5L * 1024L * 1024L * 1024L; + } + } + @Override public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject) { // for S3, no need to do anything, just return template url for http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/server/src/com/cloud/configuration/Config.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index eac3b75..72fe5dd 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -426,7 +426,8 @@ public enum Config { // object store S3EnableRRS("Advanced", ManagementServer.class, Boolean.class, "s3.rrs.enabled", "false", "enable s3 reduced redundancy storage", null), - S3EnableMultiPartUpload("Advanced", ManagementServer.class, Boolean.class, "s3.multipart.enabled", "true", "enable s3 multipart upload", null), + S3MaxSingleUploadSize("Advanced", ManagementServer.class, Integer.class, "s3.singleupload.max.size", "5", "The maximum size limit for S3 single part upload API(in GB). If it is set to 0, then it means always use multi-part upload to upload object to S3. " + + "If it is set to -1, then it means always use single-part upload to upload object to S3. ", null), // VMSnapshots VMSnapshotMax("Advanced", VMSnapshotManager.class, Integer.class, "vmsnapshot.max", "10", "Maximum vm snapshots for a vm", null), http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index a5a9f2b..bd2378c 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -820,9 +820,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } } + + long srcSize = srcFile.length(); ImageFormat format = this.getTemplateFormat(srcFile.getName()); String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName(); - if (s3.isMultipartEnabled()){ + if (!s3.getSingleUpload(srcSize)){ mputFile(s3, srcFile, bucket, key); } else{ putFile(s3, srcFile, bucket, key); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/setup/db/db/schema-420to421.sql ---------------------------------------------------------------------- diff --git a/setup/db/db/schema-420to421.sql b/setup/db/db/schema-420to421.sql index 4dfe4e6..abe97a4 100644 --- a/setup/db/db/schema-420to421.sql +++ b/setup/db/db/schema-420to421.sql @@ -20,7 +20,8 @@ --; -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.multipart.enabled', 'true', 'enable s3 multipart upload'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.singleupload.max.size', '5', + 'The maximum size limit for S3 single part upload API(in GB). If it is set to 0, then it means always use multi-part upload to upload object to S3. If it is set to -1, then it means always use single-part upload to upload object to S3.'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Storage", 'DEFAULT', 'management-server', "enable.ha.storage.migration", "true", "Enable/disable storage migration across primary storage during HA"); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/utils/src/com/cloud/utils/S3Utils.java ---------------------------------------------------------------------- diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java index ce4d4b7..7918a20 100644 --- a/utils/src/com/cloud/utils/S3Utils.java +++ b/utils/src/com/cloud/utils/S3Utils.java @@ -171,6 +171,10 @@ public final class S3Utils { assert clientOptions != null; assert req != null; + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format("Sending stream as S3 object using PutObjectRequest")); + } + acquireClient(clientOptions).putObject(req); }
