Merge branch 'ab-205166-s3-recursive-delete' into 'master'

Add helpers to s3 client for image upload delete and list

See merge request gitlab-org/gitlab!29801

ee/lib/status_page/storage.rb

@@ -8,6 +8,10 @@ module StatusPage
     MAX_RECENT_INCIDENTS = 20
     # Limit the amount of comments per incident
     MAX_COMMENTS = 100
+    # Limit on paginated responses
+    MAX_KEYS_PER_PAGE = 1_000
+    MAX_PAGES = 5
+    MAX_IMAGE_UPLOADS = MAX_KEYS_PER_PAGE * MAX_PAGES
 
     def self.details_path(id)
       "data/incident/#{id}.json"

ee/lib/status_page/storage/s3_client.rb

@@ -16,7 +16,7 @@ module StatusPage
       #
       # Note: We are making sure that
       # * we control +content+ (not the user)
-      # * this upload is done a background job (not in a web request)
+      # * this upload is done as a background job (not in a web request)
       def upload_object(key, content)
         wrap_errors(key: key) do
           client.put_object(bucket: bucket_name, key: key, body: content)
@@ -25,7 +25,7 @@ module StatusPage
         true
       end
 
-      # Deletes +key+ from storage
+      # Deletes object at +key+ from storage
       #
       # Note, this operation succeeds even if +key+ does not exist in storage.
       def delete_object(key)
@@ -36,10 +36,41 @@ module StatusPage
         true
       end
 
+      # Delete all objects whose key has a given +prefix+
+      def recursive_delete(prefix)
+        wrap_errors(prefix: prefix) do
+          # Aws::S3::Types::ListObjectsV2Output is paginated and Enumerable
+          list_objects(prefix).each.with_index do |response, index|
+            break if index >= StatusPage::Storage::MAX_PAGES
+
+            objects = response.contents.map { |obj| { key: obj.key } }
+            # Batch delete in sets determined by default max_key argument that can be passed to list_objects_v2
+            client.delete_objects({ bucket: bucket_name, delete: { objects: objects } })
+          end
+        end
+
+        true
+      end
+
+      # Return a Set of all keys with a given prefix
+      def list_object_keys(prefix)
+        wrap_errors(prefix: prefix) do
+          list_objects(prefix).reduce(Set.new) do |objects, (response, index)|
+            break objects if objects.size >= StatusPage::Storage::MAX_IMAGE_UPLOADS
+
+            objects | response.contents.map(&:key)
+          end
+        end
+      end
+
       private
 
       attr_reader :client, :bucket_name
 
+      def list_objects(prefix)
+        client.list_objects_v2(bucket: bucket_name, prefix: prefix, max_keys: StatusPage::Storage::MAX_KEYS_PER_PAGE)
+      end
+
       def wrap_errors(**args)
         yield
       rescue Aws::Errors::ServiceError => e

ee/spec/lib/status_page/storage/s3_client_spec.rb

@@ -64,6 +64,83 @@ describe StatusPage::Storage::S3Client, :aws_s3 do
     end
   end
 
+  describe '#recursive_delete' do
+    let(:key_prefix) { 'key_prefix/' }
+    let(:aws_client) { client.send('client') }
+
+    subject(:result) { client.recursive_delete(key_prefix) }
+
+    context 'when successful' do
+      include_context 'list_objects_v2 result'
+
+      it 'sends keys for batch delete' do
+        expect(aws_client).to receive(:delete_objects).with(delete_objects_data(key_list_1))
+        expect(aws_client).to receive(:delete_objects).with(delete_objects_data(key_list_2))
+
+        result
+      end
+
+      it 'returns true' do
+        expect(result).to eq(true)
+      end
+    end
+
+    context 'list_object exeeds upload limit' do
+      include_context 'oversized list_objects_v2 result'
+
+      it 'respects upload limit' do
+        expect(aws_client).to receive(:delete_objects).with(delete_objects_data(keys_page_1))
+        expect(aws_client).not_to receive(:delete_objects).with(delete_objects_data(keys_page_2))
+
+        result
+      end
+    end
+
+    context 'when failed' do
+      let(:aws_error) { 'SomeError' }
+
+      it 'raises an error' do
+        stub_responses(:list_objects_v2, aws_error)
+
+        msg = error_message(aws_error, prefix: key_prefix)
+        expect { result }.to raise_error(StatusPage::Storage::Error, msg)
+      end
+    end
+  end
+
+  describe '#list_object_keys' do
+    let(:key_prefix) { 'key_prefix/' }
+
+    subject(:result) { client.list_object_keys(key_prefix) }
+
+    context 'when successful' do
+      include_context 'list_objects_v2 result'
+
+      it 'returns keys from bucket' do
+        expect(result).to eq(Set.new(key_list_1 + key_list_2))
+      end
+    end
+
+    context 'when exceeds upload limits' do
+      include_context 'oversized list_objects_v2 result'
+
+      it 'returns result at max size' do
+        expect(result.count).to eq(StatusPage::Storage::MAX_IMAGE_UPLOADS)
+      end
+    end
+
+    context 'when failed' do
+      let(:aws_error) { 'SomeError' }
+
+      it 'raises an error' do
+        stub_responses(:list_objects_v2, aws_error)
+
+        msg = error_message(aws_error, prefix: key_prefix)
+        expect { result }.to raise_error(StatusPage::Storage::Error, msg)
+      end
+    end
+  end
+
   private
 
   def stub_responses(*args)
@@ -75,4 +152,23 @@ describe StatusPage::Storage::S3Client, :aws_s3 do
     %{Error occured "Aws::S3::Errors::#{error_class}" } \
       "for bucket #{bucket_name.inspect}. Arguments: #{args.inspect}"
   end
+
+  def delete_objects_data(keys)
+    objects = keys.map { |key| { key: key } }
+    {
+      bucket: bucket_name,
+      delete: {
+        objects: objects
+      }
+    }
+  end
+
+  def list_objects_data(key_list:, next_continuation_token:, is_truncated: )
+    contents = key_list.map { |key| Aws::S3::Types::Object.new(key: key) }
+    Aws::S3::Types::ListObjectsV2Output.new(
+      contents: contents,
+      next_continuation_token: next_continuation_token,
+      is_truncated: is_truncated
+    )
+  end
 end

ee/spec/lib/status_page/storage_spec.rb

@@ -14,4 +14,10 @@ describe StatusPage::Storage do
 
     it { is_expected.to eq('data/list.json') }
   end
+
+  it 'MAX_KEYS_PER_PAGE times MAX_PAGES establishes upload limit' do
+    # spec intended to fail if page related MAX constants change
+    # In order to ensure change to documented MAX_IMAGE_UPLOADS is considered
+    expect(StatusPage::Storage::MAX_KEYS_PER_PAGE * StatusPage::Storage::MAX_PAGES).to eq(5000)
+  end
 end

ee/spec/support/shared_contexts/status_page/status_page_list_objects.rb

+# frozen_string_literal: true
+
+RSpec.shared_context 'list_objects_v2 result' do
+  let(:key_list_1) { ['key_prefix/1', 'key_prefix/2'] }
+  let(:key_list_2) { ['key_prefix/3'] }
+
+  before do
+    # AWS s3 client responses for list_objects is paginated
+    # stub_responses allows multiple responses as arguments and they will be returned in sequence
+    stub_responses(
+      :list_objects_v2,
+      list_objects_data(key_list: key_list_1, next_continuation_token: '12345', is_truncated: true),
+      list_objects_data(key_list: key_list_2, next_continuation_token: nil, is_truncated: false)
+    )
+  end
+end
+
+RSpec.shared_context 'oversized list_objects_v2 result' do
+  let(:keys_page_1) { random_keys(desired_size: StatusPage::Storage::MAX_KEYS_PER_PAGE) }
+  let(:keys_page_2) { random_keys(desired_size: StatusPage::Storage::MAX_KEYS_PER_PAGE) }
+
+  before do
+    stub_const("StatusPage::Storage::MAX_KEYS_PER_PAGE", 2)
+    stub_const("StatusPage::Storage::MAX_PAGES", 1)
+    stub_const("StatusPage::Storage::MAX_IMAGE_UPLOADS", StatusPage::Storage::MAX_PAGES * StatusPage::Storage::MAX_KEYS_PER_PAGE)
+    # AWS s3 client responses for list_objects is paginated
+    # stub_responses allows multiple responses as arguments and they will be returned in sequence
+    stub_responses(
+      :list_objects_v2,
+      list_objects_data(key_list: keys_page_1, next_continuation_token: '12345', is_truncated: true),
+      list_objects_data(key_list: keys_page_2, next_continuation_token: nil, is_truncated: true)
+    )
+  end
+
+  def random_keys(desired_size:)
+    (0...desired_size).to_a.map { |_| SecureRandom.hex }
+  end
+end