Support presigned multipart uploads

app/controllers/projects/lfs_storage_controller.rb

@@ -18,7 +18,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
   def upload_authorize
     set_workhorse_internal_api_content_type
 
-    authorized = LfsObjectUploader.workhorse_authorize
+    authorized = LfsObjectUploader.workhorse_authorize(has_length: true)
     authorized.merge!(LfsOid: oid, LfsSize: size)
 
     render json: authorized

app/uploaders/object_storage.rb

@@ -10,8 +10,6 @@ module ObjectStorage
   UnknownStoreError = Class.new(StandardError)
   ObjectStorageUnavailable = Class.new(StandardError)
 
-  DIRECT_UPLOAD_TIMEOUT = 4.hours
-  DIRECT_UPLOAD_EXPIRE_OFFSET = 15.minutes
   TMP_UPLOAD_PATH = 'tmp/uploads'.freeze
 
   module Store
@@ -157,9 +155,9 @@ module ObjectStorage
         model_class.uploader_options.dig(mount_point, :mount_on) || mount_point
       end
 
-      def workhorse_authorize
+      def workhorse_authorize(has_length:, maximum_size: nil)
         {
-          RemoteObject: workhorse_remote_upload_options,
+          RemoteObject: workhorse_remote_upload_options(has_length: has_length, maximum_size: maximum_size),
           TempPath: workhorse_local_upload_path
         }.compact
       end
@@ -168,23 +166,16 @@ module ObjectStorage
         File.join(self.root, TMP_UPLOAD_PATH)
       end
 
-      def workhorse_remote_upload_options
+      def workhorse_remote_upload_options(has_length:, maximum_size: nil)
         return unless self.object_store_enabled?
         return unless self.direct_upload_enabled?
 
         id = [CarrierWave.generate_cache_id, SecureRandom.hex].join('-')
         upload_path = File.join(TMP_UPLOAD_PATH, id)
-        connection = ::Fog::Storage.new(self.object_store_credentials)
-        expire_at = Time.now + DIRECT_UPLOAD_TIMEOUT + DIRECT_UPLOAD_EXPIRE_OFFSET
-        options = { 'Content-Type' => 'application/octet-stream' }
+        direct_upload = ObjectStorage::DirectUpload.new(self.object_store_credentials, remote_store_path, upload_path,
+          has_length: has_length, maximum_size: maximum_size)
 
-        {
-          ID: id,
-          Timeout: DIRECT_UPLOAD_TIMEOUT,
-          GetURL: connection.get_object_url(remote_store_path, upload_path, expire_at),
-          DeleteURL: connection.delete_object_url(remote_store_path, upload_path, expire_at),
-          StoreURL: connection.put_object_url(remote_store_path, upload_path, expire_at, options)
-        }
+        direct_upload.to_hash.merge(ID: id)
       end
     end
 

changelogs/unreleased/presigned-multipart-uploads.yml

+---
+title: Support direct_upload with S3 Multipart uploads
+merge_request:
+author:
+type: added

config/initializers/artifacts_direct_upload_support.rb

-artifacts_object_store = Gitlab.config.artifacts.object_store
-
-if artifacts_object_store.enabled &&
-    artifacts_object_store.direct_upload &&
-    artifacts_object_store.connection&.provider.to_s != 'Google'
-  raise "Only 'Google' is supported as a object storage provider when 'direct_upload' of artifacts is used"
-end

config/initializers/direct_upload_support.rb

+SUPPORTED_DIRECT_UPLOAD_PROVIDERS = %w(Google AWS).freeze
+
+def verify_provider_support!(object_store)
+  return unless object_store.enabled
+  return unless object_store.direct_upload
+  return if SUPPORTED_DIRECT_UPLOAD_PROVIDERS.include?(object_store.connection&.provider.to_s)
+
+  raise "Only #{SUPPORTED_DIRECT_UPLOAD_PROVIDERS.join(',')} are supported as a object storage provider when 'direct_upload' is used"
+end
+
+verify_provider_support!(Gitlab.config.artifacts.object_store)
+verify_provider_support!(Gitlab.config.uploads.object_store)
+verify_provider_support!(Gitlab.config.lfs.object_store)

doc/administration/job_artifacts.md

@@ -94,6 +94,7 @@ _The artifacts are stored by default in
 > Available in [GitLab Premium](https://about.gitlab.com/products/) and
 [GitLab.com Silver](https://about.gitlab.com/gitlab-com/).
 > Since version 10.6, available in [GitLab CE](https://about.gitlab.com/products/)
+> Since version 11.0, we support direct_upload to S3.
 
 If you don't want to use the local disk where GitLab is installed to store the
 artifacts, you can use an object storage like AWS S3 instead.
@@ -108,7 +109,7 @@ For source installations the following settings are nested under `artifacts:` an
 |---------|-------------|---------|
 | `enabled` | Enable/disable object storage | `false` |
 | `remote_directory` | The bucket name where Artifacts will be stored| |
-| `direct_upload` | Set to true to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. Currently only `Google` provider is supported | `false` |
+| `direct_upload` | Set to true to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | `false` |
 | `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` |
 | `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` |
 | `connection` | Various connection options described below | |

lib/api/runner.rb

@@ -205,7 +205,7 @@ module API
 
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-        JobArtifactUploader.workhorse_authorize
+        JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_artifacts_size)
       end
 
       desc 'Upload artifacts for job' do

lib/object_storage/direct_upload.rb

+module ObjectStorage
+  #
+  # The DirectUpload c;ass generates a set of presigned URLs
+  # that can be used to upload data to object storage from untrusted component: Workhorse, Runner?
+  # 
+  # For Google it assumes that the platform supports variable Content-Length.
+  #
+  # For AWS it initiates Multipart Upload and presignes a set of part uploads.
+  #   Class calculates the best part size to be able to upload up to asked maximum size.
+  #   The number of generated parts will never go above 100,
+  #   but we will always try to reduce amount of generated parts.
+  #   The part size is rounded-up to 5MB.
+  #
+  class DirectUpload
+    include Gitlab::Utils::StrongMemoize
+
+    TIMEOUT = 4.hours
+    EXPIRE_OFFSET = 15.minutes
+
+    MAXIMUM_MULTIPART_PARTS = 100
+    MINIMUM_MULTIPART_SIZE = 5.megabytes
+
+    attr_reader :credentials, :bucket_name, :object_name
+    attr_reader :has_length, :maximum_size
+
+    def initialize(credentials, bucket_name, object_name, has_length:, maximum_size: nil)
+      unless has_length
+        raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size
+      end
+
+      @credentials = credentials
+      @bucket_name = bucket_name
+      @object_name = object_name
+      @has_length = has_length
+      @maximum_size = maximum_size
+    end
+
+    def to_hash
+      {
+        Timeout: TIMEOUT,
+        GetURL: get_url,
+        StoreURL: store_url,
+        DeleteURL: delete_url,
+        MultipartUpload: multipart_upload_hash
+      }.compact
+    end
+
+    def multipart_upload_hash
+      return unless requires_multipart_upload?
+
+      {
+        PartSize: rounded_multipart_part_size,
+        PartURLs: multipart_part_urls,
+        CompleteURL: multipart_complete_url,
+        AbortURL: multipart_abort_url
+      }
+    end
+
+    def provider
+      credentials[:provider].to_s
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
+    def get_url
+      connection.get_object_url(bucket_name, object_name, expire_at)
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
+    def delete_url
+      connection.delete_object_url(bucket_name, object_name, expire_at)
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
+    def store_url
+      connection.put_object_url(bucket_name, object_name, expire_at, upload_options)
+    end
+
+    def multipart_part_urls
+      Array.new(number_of_multipart_parts) do |part_index|
+        multipart_part_upload_url(part_index + 1)
+      end
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
+    def multipart_part_upload_url(part_number)
+      connection.signed_url({
+        method: 'PUT',
+        bucket_name: bucket_name,
+        object_name: object_name,
+        query: { uploadId: upload_id, partNumber: part_number },
+        headers: upload_options
+      }, expire_at)
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
+    def multipart_complete_url
+      connection.signed_url({
+        method: 'POST',
+        bucket_name: bucket_name,
+        object_name: object_name,
+        query: { uploadId: upload_id },
+        headers: { 'Content-Type' => 'application/xml' }
+      }, expire_at)
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadAbort.html
+    def multipart_abort_url
+      connection.signed_url({
+        method: 'DELETE',
+        bucket_name: bucket_name,
+        object_name: object_name,
+        query: { uploadId: upload_id }
+      }, expire_at)
+    end
+
+    private
+
+    def rounded_multipart_part_size
+      # round multipart_part_size up to minimum_mulitpart_size
+      (multipart_part_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE * MINIMUM_MULTIPART_SIZE
+    end
+
+    def multipart_part_size
+      maximum_size / number_of_multipart_parts
+    end
+
+    def number_of_multipart_parts
+      [
+        # round maximum_size up to minimum_mulitpart_size
+        (maximum_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE,
+        MAXIMUM_MULTIPART_PARTS
+      ].min
+    end
+
+    def aws?
+      provider == 'AWS'
+    end
+
+    def requires_multipart_upload?
+      aws? && !has_length
+    end
+
+    def upload_id
+      return unless requires_multipart_upload?
+
+      strong_memoize(:upload_id) do
+        new_upload = connection.initiate_multipart_upload(bucket_name, object_name)
+        new_upload.body["UploadId"]
+      end
+    end
+
+    def expire_at
+      strong_memoize(:expire_at) do
+        Time.now + TIMEOUT + EXPIRE_OFFSET
+      end
+    end
+
+    def upload_options
+      { 'Content-Type' => 'application/octet-stream' }
+    end
+
+    def connection
+      @connection ||= ::Fog::Storage.new(credentials)
+    end
+  end
+end

spec/initializers/artifacts_direct_upload_support_spec.rb

-require 'spec_helper'
-
-describe 'Artifacts direct upload support' do
-  subject do
-    load Rails.root.join('config/initializers/artifacts_direct_upload_support.rb')
-  end
-
-  let(:connection) do
-    { provider: provider }
-  end
-
-  before do
-    stub_artifacts_setting(
-      object_store: {
-        enabled: enabled,
-        direct_upload: direct_upload,
-        connection: connection
-      })
-  end
-
-  context 'when object storage is enabled' do
-    let(:enabled) { true }
-
-    context 'when direct upload is enabled' do
-      let(:direct_upload) { true }
-
-      context 'when provider is Google' do
-        let(:provider) { 'Google' }
-
-        it 'succeeds' do
-          expect { subject }.not_to raise_error
-        end
-      end
-
-      context 'when connection is empty' do
-        let(:connection) { nil }
-
-        it 'raises an error' do
-          expect { subject }.to raise_error /object storage provider when 'direct_upload' of artifacts is used/
-        end
-      end
-
-      context 'when other provider is used' do
-        let(:provider) { 'AWS' }
-
-        it 'raises an error' do
-          expect { subject }.to raise_error /object storage provider when 'direct_upload' of artifacts is used/
-        end
-      end
-    end
-
-    context 'when direct upload is disabled' do
-      let(:direct_upload) { false }
-      let(:provider) { 'AWS' }
-
-      it 'succeeds' do
-        expect { subject }.not_to raise_error
-      end
-    end
-  end
-
-  context 'when object storage is disabled' do
-    let(:enabled) { false }
-    let(:direct_upload) { false }
-    let(:provider) { 'AWS' }
-
-    it 'succeeds' do
-      expect { subject }.not_to raise_error
-    end
-  end
-end

spec/initializers/direct_upload_support_spec.rb

+require 'spec_helper'
+
+describe 'Direct upload support' do
+  subject do
+    load Rails.root.join('config/initializers/direct_upload_support.rb')
+  end
+
+  where(:config_name) do
+    ['lfs', 'artifacts', 'uploads']
+  end
+
+  with_them do
+    let(:connection) do
+      { provider: provider }
+    end
+
+    let(:object_store) do
+      {
+        enabled: enabled,
+        direct_upload: direct_upload,
+        connection: connection
+      }
+    end
+
+    before do
+      allow(Gitlab.config).to receive_messages(to_settings(
+        config_name => { object_store: object_store }))
+    end
+
+    context 'when object storage is enabled' do
+      let(:enabled) { true }
+
+      context 'when direct upload is enabled' do
+        let(:direct_upload) { true }
+
+        context 'when provider is AWS' do
+          let(:provider) { 'AWS' }
+
+          it 'succeeds' do
+            expect { subject }.not_to raise_error
+          end
+        end
+
+        context 'when provider is Google' do
+          let(:provider) { 'Google' }
+
+          it 'succeeds' do
+            expect { subject }.not_to raise_error
+          end
+        end
+
+        context 'when connection is empty' do
+          let(:connection) { nil }
+
+          it 'raises an error' do
+            expect { subject }.to raise_error /are supported as a object storage provider when 'direct_upload' is used/
+          end
+        end
+
+        context 'when other provider is used' do
+          let(:provider) { 'Rackspace' }
+
+          it 'raises an error' do
+            expect { subject }.to raise_error /are supported as a object storage provider when 'direct_upload' is used/
+          end
+        end
+      end
+
+      context 'when direct upload is disabled' do
+        let(:direct_upload) { false }
+        let(:provider) { 'AWS' }
+
+        it 'succeeds' do
+          expect { subject }.not_to raise_error
+        end
+      end
+    end
+
+    context 'when object storage is disabled' do
+      let(:enabled) { false }
+      let(:direct_upload) { false }
+      let(:provider) { 'Rackspace' }
+
+      it 'succeeds' do
+        expect { subject }.not_to raise_error
+      end
+    end
+  end
+end

spec/lib/object_storage/direct_upload_spec.rb

+require 'spec_helper'
+
+describe ObjectStorage::DirectUpload do
+  let(:credentials) do
+    {
+      provider: 'AWS',
+      aws_access_key_id: 'AWS_ACCESS_KEY_ID',
+      aws_secret_access_key: 'AWS_SECRET_ACCESS_KEY'
+    }
+  end
+
+  let(:storage_url) { 'https://uploads.s3.amazonaws.com/' }
+
+  let(:bucket_name) { 'uploads' }
+  let(:object_name) { 'tmp/uploads/my-file' }
+  let(:maximum_size) { 1.gigabyte }
+
+  let(:direct_upload) { described_class.new(credentials, bucket_name, object_name, has_length: has_length, maximum_size: maximum_size) }
+
+  describe '#has_length' do
+    context 'is known' do
+      let(:has_length) { true }
+      let(:maximum_size) { nil }
+
+      it "maximum size is not required" do
+        expect { direct_upload }.not_to raise_error
+      end
+    end
+
+    context 'is unknown' do
+      let(:has_length) { false }
+
+      context 'and maximum size is specified' do
+        let(:maximum_size) { 1.gigabyte }
+
+        it "does not raise an error" do
+          expect { direct_upload }.not_to raise_error
+        end
+      end
+
+      context 'and maximum size is not specified' do
+        let(:maximum_size) { nil }
+
+        it "raises an error" do
+          expect { direct_upload }.to raise_error /maximum_size has to be specified if length is unknown/
+        end
+      end
+    end
+  end
+
+  describe '#to_hash' do
+    subject { direct_upload.to_hash }
+
+    shared_examples 'a valid upload' do
+      it "returns valid structure" do
+        expect(subject).to have_key(:Timeout)
+        expect(subject[:GetURL]).to start_with(storage_url)
+        expect(subject[:StoreURL]).to start_with(storage_url)
+        expect(subject[:DeleteURL]).to start_with(storage_url)
+      end
+    end
+
+    shared_examples 'a valid upload with multipart data' do
+      before do
+        stub_object_storage_multipart_init(storage_url, "myUpload")
+      end
+
+      it_behaves_like 'a valid upload'
+
+      it "returns valid structure" do
+        expect(subject).to have_key(:MultipartUpload)
+        expect(subject[:MultipartUpload]).to have_key(:PartSize)
+        expect(subject[:MultipartUpload][:PartURLs]).to all(start_with(storage_url))
+        expect(subject[:MultipartUpload][:PartURLs]).to all(include('uploadId=myUpload'))
+        expect(subject[:MultipartUpload][:CompleteURL]).to start_with(storage_url)
+        expect(subject[:MultipartUpload][:CompleteURL]).to include('uploadId=myUpload')
+        expect(subject[:MultipartUpload][:AbortURL]).to start_with(storage_url)
+        expect(subject[:MultipartUpload][:AbortURL]).to include('uploadId=myUpload')
+      end
+    end
+
+    shared_examples 'a valid upload without multipart data' do
+      it_behaves_like 'a valid upload'
+
+      it "returns valid structure" do
+        expect(subject).not_to have_key(:MultipartUpload)
+      end
+    end
+
+    context 'when AWS is used' do
+      context 'when length is known' do
+        let(:has_length) { true }
+
+        it_behaves_like 'a valid upload without multipart data'
+      end
+
+      context 'when length is unknown' do
+        let(:has_length) { false }
+
+        it_behaves_like 'a valid upload with multipart data' do
+          context 'when maximum upload size is 10MB' do
+            let(:maximum_size) { 10.megabyte }
+
+            it 'returns only 2 parts' do
+              expect(subject[:MultipartUpload][:PartURLs].length).to eq(2)
+            end
+
+            it 'part size is mimimum, 5MB' do
+              expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte)
+            end
+          end
+
+          context 'when maximum upload size is 12MB' do
+            let(:maximum_size) { 12.megabyte }
+
+            it 'returns only 3 parts' do
+              expect(subject[:MultipartUpload][:PartURLs].length).to eq(3)
+            end
+
+            it 'part size is rounded-up to 5MB' do
+              expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte)
+            end
+          end
+
+          context 'when maximum upload size is 49GB' do
+            let(:maximum_size) { 49.gigabyte }
+
+            it 'returns maximum, 100 parts' do
+              expect(subject[:MultipartUpload][:PartURLs].length).to eq(100)
+            end
+
+            it 'part size is rounded-up to 5MB' do
+              expect(subject[:MultipartUpload][:PartSize]).to eq(505.megabyte)
+            end
+          end
+        end
+      end
+    end
+
+    context 'when Google is used' do
+      let(:credentials) do
+        {
+          provider: 'Google',
+          google_storage_access_key_id: 'GOOGLE_ACCESS_KEY_ID',
+          google_storage_secret_access_key: 'GOOGLE_SECRET_ACCESS_KEY'
+        }
+      end
+
+      let(:storage_url) { 'https://storage.googleapis.com/uploads/' }
+
+      context 'when length is known' do
+        let(:has_length) { true }
+
+        it_behaves_like 'a valid upload without multipart data'
+      end
+
+      context 'when length is unknown' do
+        let(:has_length) { false }
+
+        it_behaves_like 'a valid upload without multipart data'
+      end
+    end
+  end
+
+  describe '#get_url' do
+    # this method can only be tested with integration tests
+  end
+
+  describe '#delete_url' do
+    # this method can only be tested with integration tests
+  end
+
+  describe '#store_url' do
+    # this method can only be tested with integration tests
+  end
+
+  describe '#multipart_part_upload_url' do
+    # this method can only be tested with integration tests
+  end
+
+  describe '#multipart_complete_url' do
+    # this method can only be tested with integration tests
+  end
+
+  describe '#multipart_abort_url' do
+    # this method can only be tested with integration tests
+  end
+end

spec/requests/api/runner_spec.rb

@@ -1101,6 +1101,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
                   expect(json_response['RemoteObject']).to have_key('GetURL')
                   expect(json_response['RemoteObject']).to have_key('StoreURL')
                   expect(json_response['RemoteObject']).to have_key('DeleteURL')
+                  expect(json_response['RemoteObject']).to have_key('MultipartUpload')
                 end
               end
 

spec/requests/lfs_http_spec.rb

@@ -1021,6 +1021,7 @@ describe 'Git LFS API and storage' do
                     expect(json_response['RemoteObject']).to have_key('GetURL')
                     expect(json_response['RemoteObject']).to have_key('StoreURL')
                     expect(json_response['RemoteObject']).to have_key('DeleteURL')
+                    expect(json_response['RemoteObject']).not_to have_key('MultipartUpload')
                     expect(json_response['LfsOid']).to eq(sample_oid)
                     expect(json_response['LfsSize']).to eq(sample_size)
                   end

spec/support/helpers/stub_object_storage.rb

@@ -45,4 +45,16 @@ module StubObjectStorage
                                  remote_directory: 'uploads',
                                  **params)
   end
+
+  def stub_object_storage_multipart_init(endpoint, upload_id = "upload_id")
+    stub_request(:post, %r{\A#{endpoint}tmp/uploads/[a-z0-9-]*\?uploads\z}).
+      to_return status: 200, body: <<-EOS.strip_heredoc
+        <?xml version="1.0" encoding="UTF-8"?>
+        <InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+          <Bucket>example-bucket</Bucket>
+          <Key>example-object</Key>
+          <UploadId>#{upload_id}</UploadId>
+        </InitiateMultipartUploadResult>
+      EOS
+  end
 end

spec/uploaders/object_storage_spec.rb

@@ -355,7 +355,10 @@ describe ObjectStorage do
   end
 
   describe '.workhorse_authorize' do
-    subject { uploader_class.workhorse_authorize }
+    let(:has_length) { true }
+    let(:maximum_size) { nil }
+
+    subject { uploader_class.workhorse_authorize(has_length: has_length, maximum_size: maximum_size) }
 
     before do
       # ensure that we use regular Fog libraries
@@ -371,10 +374,6 @@ describe ObjectStorage do
         expect(subject[:TempPath]).to start_with(uploader_class.root)
         expect(subject[:TempPath]).to include(described_class::TMP_UPLOAD_PATH)
       end
-
-      it "does not return remote store" do
-        is_expected.not_to have_key('RemoteObject')
-      end
     end
 
     shared_examples 'uses remote storage' do
@@ -383,7 +382,7 @@ describe ObjectStorage do
 
         expect(subject[:RemoteObject]).to have_key(:ID)
         expect(subject[:RemoteObject]).to include(Timeout: a_kind_of(Integer))
-        expect(subject[:RemoteObject][:Timeout]).to be(ObjectStorage::DIRECT_UPLOAD_TIMEOUT)
+        expect(subject[:RemoteObject][:Timeout]).to be(ObjectStorage::DirectUpload::TIMEOUT)
         expect(subject[:RemoteObject]).to have_key(:GetURL)
         expect(subject[:RemoteObject]).to have_key(:DeleteURL)
         expect(subject[:RemoteObject]).to have_key(:StoreURL)
@@ -391,9 +390,31 @@ describe ObjectStorage do
         expect(subject[:RemoteObject][:DeleteURL]).to include(described_class::TMP_UPLOAD_PATH)
         expect(subject[:RemoteObject][:StoreURL]).to include(described_class::TMP_UPLOAD_PATH)
       end
+    end
 
-      it "does not return local store" do
-        is_expected.not_to have_key('TempPath')
+    shared_examples 'uses remote storage with multipart uploads' do
+      it_behaves_like 'uses remote storage' do
+        it "returns multipart upload" do
+          is_expected.to have_key(:RemoteObject)
+
+          expect(subject[:RemoteObject]).to have_key(:MultipartUpload)
+          expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:PartSize)
+          expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:PartURLs)
+          expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:CompleteURL)
+          expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:AbortURL)
+          expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(include(described_class::TMP_UPLOAD_PATH))
+          expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to include(described_class::TMP_UPLOAD_PATH)
+          expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to include(described_class::TMP_UPLOAD_PATH)
+        end
+      end
+    end
+
+    shared_examples 'uses remote storage without multipart uploads' do
+      it_behaves_like 'uses remote storage' do
+        it "does not return multipart upload" do
+          is_expected.to have_key(:RemoteObject)
+          expect(subject[:RemoteObject]).not_to have_key(:MultipartUpload)
+        end
       end
     end
 
@@ -416,6 +437,8 @@ describe ObjectStorage do
         end
 
         context 'uses AWS' do
+          let(:storage_url) { "https://uploads.s3-eu-central-1.amazonaws.com/" }
+
           before do
             expect(uploader_class).to receive(:object_store_credentials) do
               { provider: "AWS",
@@ -425,18 +448,40 @@ describe ObjectStorage do
             end
           end
 
-          it_behaves_like 'uses remote storage' do
-            let(:storage_url) { "https://uploads.s3-eu-central-1.amazonaws.com/" }
+          context 'for known length' do
+            it_behaves_like 'uses remote storage without multipart uploads' do
+              it 'returns links for S3' do
+                expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+              end
+            end
+          end
+
+          context 'for unknown length' do
+            let(:has_length) { false }
+            let(:maximum_size) { 1.gigabyte }
 
-            it 'returns links for S3' do
-              expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
-              expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
-              expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+            before do
+              stub_object_storage_multipart_init(storage_url)
+            end
+
+            it_behaves_like 'uses remote storage with multipart uploads' do
+              it 'returns links for S3' do
+                expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(start_with(storage_url))
+                expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to start_with(storage_url)
+              end
             end
           end
         end
 
         context 'uses Google' do
+          let(:storage_url) { "https://storage.googleapis.com/uploads/" }
+
           before do
             expect(uploader_class).to receive(:object_store_credentials) do
               { provider: "Google",
@@ -445,36 +490,71 @@ describe ObjectStorage do
             end
           end
 
-          it_behaves_like 'uses remote storage' do
-            let(:storage_url) { "https://storage.googleapis.com/uploads/" }
+          context 'for known length' do
+            it_behaves_like 'uses remote storage without multipart uploads' do
+              it 'returns links for Google Cloud' do
+                expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+              end
+            end
+          end
+
+          context 'for unknown length' do
+            let(:has_length) { false }
+            let(:maximum_size) { 1.gigabyte }
 
-            it 'returns links for Google Cloud' do
-              expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
-              expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
-              expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+            it_behaves_like 'uses remote storage without multipart uploads' do
+              it 'returns links for Google Cloud' do
+                expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+              end
             end
           end
         end
 
         context 'uses GDK/minio' do
+          let(:storage_url) { "http://minio:9000/uploads/" }
+
           before do
             expect(uploader_class).to receive(:object_store_credentials) do
               { provider: "AWS",
                 aws_access_key_id: "AWS_ACCESS_KEY_ID",
                 aws_secret_access_key: "AWS_SECRET_ACCESS_KEY",
-                endpoint: 'http://127.0.0.1:9000',
+                endpoint: 'http://minio:9000',
                 path_style: true,
                 region: "gdk" }
             end
           end
 
-          it_behaves_like 'uses remote storage' do
-            let(:storage_url) { "http://127.0.0.1:9000/uploads/" }
+          context 'for known length' do
+            it_behaves_like 'uses remote storage without multipart uploads' do
+              it 'returns links for S3' do
+                expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+              end
+            end
+          end
+
+          context 'for unknown length' do
+            let(:has_length) { false }
+            let(:maximum_size) { 1.gigabyte }
 
-            it 'returns links for S3' do
-              expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
-              expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
-              expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+            before do
+              stub_object_storage_multipart_init(storage_url)
+            end
+
+            it_behaves_like 'uses remote storage with multipart uploads' do
+              it 'returns links for S3' do
+                expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(start_with(storage_url))
+                expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to start_with(storage_url)
+                expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to start_with(storage_url)
+              end
             end
           end
         end