Merge branch 'sh-fog-aws-set-multipart-chunk-size' into 'master'

Enable multipart transfers for S3 copying

See merge request gitlab-org/gitlab!51267

Gemfile

@@ -113,7 +113,7 @@ gem 'carrierwave', '~> 1.3'
 gem 'mini_magick', '~> 4.10.1'
 
 # for backups
-gem 'fog-aws', '~> 3.7'
+gem 'fog-aws', '~> 3.8'
 # Locked until fog-google resolves https://github.com/fog/fog-google/issues/421.
 # Also see config/initializers/fog_core_patch.rb.
 gem 'fog-core', '= 2.1.0'

Gemfile.lock

@@ -360,7 +360,7 @@ GEM
       fog-json
       ipaddress (~> 0.8)
       xml-simple (~> 1.1)
-    fog-aws (3.7.0)
+    fog-aws (3.8.0)
       fog-core (~> 2.1)
       fog-json (~> 1.1)
       fog-xml (~> 0.1)
@@ -1346,7 +1346,7 @@ DEPENDENCIES
   flipper-active_support_cache_store (~> 0.17.1)
   flowdock (~> 0.7)
   fog-aliyun (~> 0.3)
-  fog-aws (~> 3.7)
+  fog-aws (~> 3.8)
   fog-core (= 2.1.0)
   fog-google (~> 1.12)
   fog-local (~> 0.6)

config/initializers/carrierwave_patch.rb

@@ -23,7 +23,9 @@ module CarrierWave
           # Multithreaded uploads are essential for copying large amounts of data
           # within the request timeout.
           if ::Feature.enabled?(:s3_multithreaded_uploads, default_enabled: true) && fog_provider == 'AWS'
-            file.concurrency = 10 # AWS SDK uses 10 threads by default
+            # AWS SDK uses 10 threads by default and a multipart chunk size of 10 MB
+            file.concurrency = 10
+            file.multipart_chunk_size = 10485760
             file.copy(@uploader.fog_directory, new_path, copy_to_options)
           else
             # Some Fog providers may issue a GET request (https://github.com/fog/fog-google/issues/512)

spec/initializers/carrierwave_patch_spec.rb

@@ -39,6 +39,14 @@ RSpec.describe 'CarrierWave::Storage::Fog::File' do
       let(:dest_filename) { 'copied.txt'}
 
       it 'copies the file' do
+        fog_file = subject.send(:file)
+
+        expect(fog_file).to receive(:concurrency=).with(10).and_call_original
+        # multipart_chunk_size must be explicitly set in order to leverage
+        # multithreaded, multipart transfers for files below 5GB.
+        expect(fog_file).to receive(:multipart_chunk_size=).with(10.megabytes).and_call_original
+        expect(fog_file).to receive(:copy).with(bucket_name, dest_filename, anything).and_call_original
+
         result = subject.copy_to(dest_filename)
 
         expect(result.exists?).to be true