Increase multipart upload default part size to 8MB (#4032)

Author: philasmar

generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json

@@ -0,0 +1,11 @@
+{
+  "services": [
+    {
+      "serviceName": "S3",
+      "type": "minor",
+      "changeLogMessages": [
+        "Increasing the default part size for S3 multipart upload from 5MB to 8MB when no part size is specified. This will reduce the number of API calls for multipart uploads."
+      ]
+    }
+  ]
+}

sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs

@@ -81,11 +81,12 @@ internal MultipartUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config
             this._s3Client = s3Client;
             this._fileTransporterRequest = fileTransporterRequest;
             this._contentLength = this._fileTransporterRequest.ContentLength;
+            
+            long targetPartSize = fileTransporterRequest.IsSetPartSize() 
+                ? fileTransporterRequest.PartSize 
+                : S3Constants.DefaultPartSize;
 
-            if (fileTransporterRequest.IsSetPartSize())
-                this._partSize = fileTransporterRequest.PartSize;
-            else
-                this._partSize = calculatePartSize(this._contentLength);
+            this._partSize = calculatePartSize(this._contentLength, targetPartSize);
 
             if (fileTransporterRequest.InputStream != null)
             {
@@ -98,15 +99,9 @@ internal MultipartUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config
             Logger.DebugFormat("Upload part size {0}.", this._partSize);
         }
 
-        private static long calculatePartSize(long fileSize)
+        private static long calculatePartSize(long contentLength, long targetPartSize)
         {
-            double partSize = Math.Ceiling((double)fileSize / S3Constants.MaxNumberOfParts);
-            if (partSize < S3Constants.MinPartSize)
-            {
-                partSize = S3Constants.MinPartSize;
-            }
-
-            return (long)partSize;
+            return Math.Max(targetPartSize, contentLength / S3Constants.MaxNumberOfParts);
         }
 
         private string determineContentType()

sdk/src/Services/S3/Custom/Util/S3Constants.cs

@@ -33,6 +33,7 @@ internal static class S3Constants
         internal const int PutObjectDefaultTimeout = 20 * 60 * 1000;
 
         internal static readonly long MinPartSize = 5 * (long)Math.Pow(2, 20);
+        internal static readonly long DefaultPartSize = 8 * (long)Math.Pow(2, 20);
         internal const int MaxNumberOfParts = 10000;
 
         internal const int DefaultBufferSize = 8192;

sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs

@@ -663,6 +663,25 @@ public void MultipartGetNumberTest()
             }
         }
 
+        [TestMethod]
+        [TestCategory("S3")]
+        public void MultipartValidatePartSize8MbTest()
+        {
+            string key = "MultipartValidatePartSizeTest";
+
+            Upload(key, 20 * MEG_SIZE, null, Client);
+            
+            var objectMetadataResponse = Client.GetObjectMetadata(new GetObjectMetadataRequest
+            {
+                BucketName = bucketName,
+                Key = key,
+                PartNumber = 1,
+            });
+
+            Assert.AreEqual(3, objectMetadataResponse.PartsCount);
+            Assert.AreEqual(8 * MEG_SIZE, objectMetadataResponse.ContentLength);
+        }
+
         void Upload(string fileName, long size,
             TransferProgressValidator<UploadProgressArgs> progressValidator, AmazonS3Client client = null)
         {