Send images to s3 on publish

Add support for sending images to s3 on
the publication of issue details

app/finders/upload_finder.rb

+# frozen_string_literal: true
+
+class UploadFinder
+  def initialize(project, secret, file_path)
+    @project = project
+    @secret = secret
+    @file_path = file_path
+  end
+
+  def execute
+    Gitlab::Utils.check_path_traversal!(@file_path)
+    uploader = FileUploader.new(@project, secret: @secret)
+    uploader.retrieve_from_store!(@file_path)
+
+    uploader
+  end
+end

doc/user/project/status_page/index.md

@@ -71,7 +71,8 @@ The incident detail page shows detailed information about a particular incident.
 
 - Status on the incident, including when the incident was last updated.
 - The incident title, including any emojis.
-- The description of the incident, including emojis and static images.
+- The description of the incident, including emojis.
+- Including any file attachments that are image type.[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205166) in GitLab 13.0.
 - A chronological ordered list of updates to the incident.
 
 ![Status Page detail](../img/status_page_detail_v12_10.png)
@@ -108,3 +109,9 @@ Anyone with access to view the Issue can add an Emoji Award to a comment, so you
 ### Changing the Incident status
 
 To change the incident status from `open` to `closed`, close the incident issue within GitLab. This will then be updated shortly on the Status Page website.
+
+## Attachment storage
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205166) in GitLab 13.0.
+
+Starting with GitLab 13.0, any file that is uploaded to incident issue descriptions or comments will be published and unpublished to the status page s3 bucket, as part of the publication flow described in the [How it works](#how-it-works) section.

ee/app/services/status_page/publish_base_service.rb

@@ -44,7 +44,7 @@ module StatusPage
       project.status_page_setting&.enabled?
     end
 
-    def upload(key, json)
+    def upload_json(key, json)
       return error_limit_exceeded(key) if limit_exceeded?(json)
 
       content = json.to_json
@@ -53,6 +53,10 @@ module StatusPage
       success(object_key: key)
     end
 
+    def multipart_upload(key, uploader)
+      storage_client.multipart_upload(key, uploader)
+    end
+
     def delete_object(key)
       storage_client.delete_object(key)
     end

ee/app/services/status_page/publish_details_service.rb

@@ -11,22 +11,68 @@ module StatusPage
     private
 
     def process(issue, user_notes)
+      publish_json_response = publish_json(issue, user_notes)
+      return publish_json_response if publish_json_response.error?
+
+      image_object_keys = publish_images(issue, user_notes)
+
+      success_payload = publish_json_response.payload.merge({ image_object_keys: image_object_keys })
+
+      success(success_payload)
+    end
+
+    # Publish Json
+
+    def publish_json(issue, user_notes)
       json = serialize(issue, user_notes)
-      key = object_key(json)
+      key = json_object_key(json)
       return error('Missing object key') unless key
 
-      upload(key, json)
+      upload_json(key, json)
     end
 
     def serialize(issue, user_notes)
       serializer.represent_details(issue, user_notes)
     end
 
-    def object_key(json)
+    def json_object_key(json)
       id = json[:id]
       return unless id
 
       StatusPage::Storage.details_path(id)
     end
+
+    # Publish Images
+    def publish_images(issue, user_notes)
+      existing_image_keys = storage_client.list_object_keys(StatusPage::Storage.uploads_path(issue.iid))
+
+      # Send all description images to s3
+      publish_markdown_uploads(
+        markdown_field: issue.description,
+        issue_iid: issue.iid,
+        existing_image_keys: existing_image_keys
+      )
+
+      # Send all comment images to s3
+      user_notes.each do |user_note|
+        publish_markdown_uploads(
+          markdown_field: user_note.note,
+          issue_iid: issue.iid,
+          existing_image_keys: existing_image_keys
+        )
+      end
+    end
+
+    def publish_markdown_uploads(markdown_field:, issue_iid:, existing_image_keys:)
+      markdown_field.scan(FileUploader::MARKDOWN_PATTERN).map do |md|
+        key = StatusPage::Storage.upload_path(issue_iid, $~[:secret], $~[:file])
+        next if existing_image_keys.include? key
+
+        uploader = UploadFinder.new(@project, $~[:secret], $~[:file]).execute
+        uploader.open do |f|
+          storage_client.multipart_upload(key, f)
+        end
+      end
+    end
   end
 end

ee/changelogs/unreleased/ab-205166-send-images-to-s3-bucket.yml

+---
+title: Support transferring and displaying image uploads on Status Page
+merge_request: 31269
+author:
+type: added

ee/lib/status_page/storage/s3_client.rb

@@ -63,10 +63,72 @@ module StatusPage
         end
       end
 
+      # Stores +file+ as +key+ in storage using multipart upload
+      #
+      # key: s3 key at which file is stored
+      # file: An open file or file-like io object
+      def multipart_upload(key, file)
+        # AWS sdk v2 has upload_file which supports multipart
+        # However Gitlab::HttpIO used when objectStorage is enabled
+        # cannot be used with upload_file
+        wrap_errors(key: key) do
+          upload_id = client.create_multipart_upload({ bucket: bucket_name, key: key }).to_h[:upload_id]
+          parts = upload_in_parts(key, file, upload_id)
+          complete_multipart_upload(key, upload_id, parts)
+        end
+      # Rescue on Exception since even on keyboard inturrupt we want to abor the upload and re-raise
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        abort_multipart_upload(key, upload_id)
+        raise e
+      end
+
       private
 
       attr_reader :client, :bucket_name
 
+      def upload_in_parts(key, file, upload_id)
+        parts = []
+        part_number = 1
+        part_size = 5.megabytes
+
+        file.seek(0)
+        until file.eof?
+          part = client.upload_part({
+            body: file.read(part_size),
+            bucket: bucket_name,
+            key: key,
+            part_number: part_number, # required
+            upload_id: upload_id
+          })
+          parts << part.to_h.merge(part_number: part_number)
+          part_number += 1
+        end
+        file.seek(0)
+
+        parts
+      end
+
+      def complete_multipart_upload(key, upload_id, parts)
+        client.complete_multipart_upload({
+          bucket: bucket_name,
+          key: key,
+          multipart_upload: {
+            parts: parts
+          },
+          upload_id: upload_id
+        })
+      end
+
+      def abort_multipart_upload(key, upload_id)
+        if upload_id
+          client.abort_multipart_upload(
+            bucket: bucket_name,
+            key: key,
+            upload_id: upload_id
+          )
+        end
+      end
+
       def list_objects(prefix)
         client.list_objects_v2(bucket: bucket_name, prefix: prefix, max_keys: StatusPage::Storage::MAX_KEYS_PER_PAGE)
       end

ee/spec/services/status_page/publish_details_service_spec.rb

@@ -4,9 +4,10 @@ require 'spec_helper'
 
 describe StatusPage::PublishDetailsService do
   let_it_be(:project, refind: true) { create(:project) }
-  let(:issue) { instance_double(Issue) }
-  let(:user_notes) { double(:user_notes) }
+  let(:markdown_field) { 'Hello World' }
+  let(:user_notes) { [] }
   let(:incident_id) { 1 }
+  let(:issue) { instance_double(Issue, notes: user_notes, description: markdown_field, iid: incident_id) }
   let(:key) { StatusPage::Storage.details_path(incident_id) }
   let(:content) { { id: incident_id } }
 

ee/spec/support/shared_examples/status_page/publish_shared_examples.rb

@@ -20,17 +20,15 @@ RSpec.shared_examples 'publish incidents' do
       .and_return(serializer)
   end
 
-  shared_examples 'feature is not available' do
-  end
-
-  context 'when upload succeeds' do
+  context 'when json upload succeeds' do
     before do
       allow(storage_client).to receive(:upload_object).with(key, content_json)
+      allow(storage_client).to receive(:list_object_keys).and_return(Set.new)
     end
 
     it 'publishes details as JSON' do
       expect(result).to be_success
-      expect(result.payload).to eq(object_key: key)
+      expect(result.payload[:json_object_key]).to eq(key)
     end
   end
 
@@ -79,4 +77,80 @@ RSpec.shared_examples 'publish incidents' do
       expect(result.message).to eq('Feature not available')
     end
   end
+
+  context 'publishes image uploads' do
+    before do
+      allow(storage_client).to receive(:upload_object).with("data/incident/1.json", "{\"id\":1}")
+      allow(storage_client).to receive(:list_object_keys).and_return(Set.new)
+    end
+
+    context 'no upload in markdown' do
+      it 'publishes no images' do
+        expect(result).to be_success
+        expect(result.payload[:image_object_keys]).to eq([])
+      end
+    end
+
+    context 'upload in markdown' do
+      let(:upload_secret) { '734b8524a16d44eb0ff28a2c2e4ff3c0' }
+      let(:image_file_name) { 'tanuki.png'}
+      let(:upload_path) { "/uploads/#{upload_secret}/#{image_file_name}" }
+      let(:markdown_field) { "![tanuki](#{upload_path})" }
+      let(:status_page_upload_path) { StatusPage::Storage.upload_path(issue.iid, upload_secret, image_file_name) }
+
+      let(:open_file) { instance_double(File) }
+      let(:upload) { double(file: double(:file, file: upload_path)) }
+
+      before do
+        allow_next_instance_of(FileUploader) do |uploader|
+          allow(uploader).to receive(:retrieve_from_store!).and_return(upload)
+        end
+        allow(File).to receive(:open).and_return(open_file)
+        allow(storage_client).to receive(:upload_object).with(upload_path, open_file)
+      end
+
+      it 'publishes description images' do
+        expect(result).to be_success
+        expect(result.payload[:image_object_keys]).to eq([status_page_upload_path])
+      end
+
+      context 'user notes uploads' do
+        let(:user_note) { instance_double(Note, note: markdown_field) }
+        let(:user_notes) { [user_note] }
+
+        it 'publishes images' do
+          expect(result).to be_success
+          expect(result.payload[:image_object_keys]).to eq([status_page_upload_path])
+        end
+      end
+
+      context 'when all images are in s3' do
+        before do
+          allow(storage_client).to receive(:list_object_keys).and_return(Set[status_page_upload_path])
+        end
+
+        it 'publishes no images' do
+          expect(result).to be_success
+          expect(result.payload[:image_object_keys]).to eq([])
+        end
+      end
+
+      context 'when images are already in s3' do
+        let(:upload_secret_2) { '9cb61a79ce884d5b6c1dd42728d3c159' }
+        let(:image_file_name_2) { 'tanuki_2.png' }
+        let(:upload_path_2) { "/uploads/#{upload_secret_2}/#{image_file_name_2}" }
+        let(:markdown_field) { "![tanuki](#{upload_path}) and ![tanuki_2](#{upload_path_2})" }
+        let(:status_page_upload_path_2) { StatusPage::Storage.upload_path(issue.iid, upload_secret_2, image_file_name_2) }
+
+        before do
+          allow(storage_client).to receive(:list_object_keys).and_return(Set[status_page_upload_path])
+        end
+
+        it 'publishes new images' do
+          expect(result).to be_success
+          expect(result.payload[:image_object_keys]).to eq([status_page_upload_path_2, status_page_upload_path_2])
+        end
+      end
+    end
+  end
 end

lib/gitlab/gfm/uploads_rewriter.rb

@@ -22,9 +22,8 @@ module Gitlab
         return @text unless needs_rewrite?
 
         @text.gsub(@pattern) do |markdown|
-          Gitlab::Utils.check_path_traversal!($~[:file])
+          file = UploadFinder.new(@source_project, $~[:secret], $~[:file]).execute
 
-          file = find_file(@source_project, $~[:secret], $~[:file])
           break markdown unless file.try(:exists?)
 
           klass = target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader
@@ -56,14 +55,6 @@ module Gitlab
       def was_embedded?(markdown)
         markdown.starts_with?("!")
       end
-
-      private
-
-      def find_file(project, secret, file)
-        uploader = FileUploader.new(project, secret: secret)
-        uploader.retrieve_from_store!(file)
-        uploader
-      end
     end
   end
 end