Merge branch 'sh-support-sse-encryption-ci-live-trace' into 'master'

Support S3 server side encryption in CI cloud native job logs

See merge request gitlab-org/gitlab!47536

app/models/ci/build_trace_chunks/fog.rb

@@ -14,11 +14,15 @@ module Ci
       end
 
       def set_data(model, new_data)
-        # TODO: Support AWS S3 server side encryption
-        files.create({
-          key: key(model),
-          body: new_data
-        })
+        if Feature.enabled?(:ci_live_trace_use_fog_attributes)
+          files.create(create_attributes(model, new_data))
+        else
+          # TODO: Support AWS S3 server side encryption
+          files.create({
+            key: key(model),
+            body: new_data
+          })
+        end
       end
 
       def append_data(model, new_data, offset)
@@ -57,6 +61,13 @@ module Ci
         key_raw(model.build_id, model.chunk_index)
       end
 
+      def create_attributes(model, new_data)
+        {
+          key: key(model),
+          body: new_data
+        }.merge(object_store_config.fog_attributes)
+      end
+
       def key_raw(build_id, chunk_index)
         "tmp/builds/#{build_id.to_i}/chunks/#{chunk_index.to_i}.log"
       end
@@ -84,6 +95,14 @@ module Ci
       def object_store
         Gitlab.config.artifacts.object_store
       end
+
+      def object_store_raw_config
+        object_store
+      end
+
+      def object_store_config
+        @object_store_config ||= ::ObjectStorage::Config.new(object_store_raw_config)
+      end
     end
   end
 end

changelogs/unreleased/sh-support-sse-encryption-ci-live-trace.yml

+---
+title: Support S3 server side encryption in CI cloud native job logs
+merge_request: 47536
+author:
+type: fixed

config/feature_flags/development/ci_live_trace_use_fog_attributes.yml

+---
+name: ci_live_trace_use_fog_attributes
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47536
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/285079
+milestone: '13.6'
+type: development
+group: group::testing
+default_enabled: false

lib/object_storage/config.rb

@@ -93,6 +93,11 @@ module ObjectStorage
 
     private
 
+    # This returns a Hash of HTTP encryption headers to send along to S3.
+    #
+    # They can also be passed in as Fog::AWS::Storage::File attributes, since there
+    # are aliases defined for them:
+    # https://github.com/fog/fog-aws/blob/ab288f29a0974d64fd8290db41080e5578be9651/lib/fog/aws/models/storage/file.rb#L24-L25
     def aws_server_side_encryption_headers
       {
         'x-amz-server-side-encryption' => server_side_encryption,

spec/models/ci/build_trace_chunks/fog_spec.rb

@@ -74,6 +74,52 @@ RSpec.describe Ci::BuildTraceChunks::Fog do
 
         expect(data_store.data(model)).to eq new_data
       end
+
+      context 'when S3 server side encryption is enabled' do
+        before do
+          config = Gitlab.config.artifacts.object_store.to_h
+          config[:storage_options] = { server_side_encryption: 'AES256' }
+          allow(data_store).to receive(:object_store_raw_config).and_return(config)
+        end
+
+        it 'creates a file with attributes' do
+          expect_next_instance_of(Fog::AWS::Storage::Files) do |files|
+            expect(files).to receive(:create).with(
+              hash_including(
+                key: anything,
+                body: new_data,
+                'x-amz-server-side-encryption' => 'AES256')
+            ).and_call_original
+          end
+
+          expect(data_store.data(model)).to be_nil
+
+          data_store.set_data(model, new_data)
+
+          expect(data_store.data(model)).to eq new_data
+        end
+
+        context 'when ci_live_trace_use_fog_attributes flag is disabled' do
+          before do
+            stub_feature_flags(ci_live_trace_use_fog_attributes: false)
+          end
+
+          it 'does not pass along Fog attributes' do
+            expect_next_instance_of(Fog::AWS::Storage::Files) do |files|
+              expect(files).to receive(:create).with(
+                key: anything,
+                body: new_data
+              ).and_call_original
+            end
+
+            expect(data_store.data(model)).to be_nil
+
+            data_store.set_data(model, new_data)
+
+            expect(data_store.data(model)).to eq new_data
+          end
+        end
+      end
     end
   end
 

spec/support/helpers/stub_object_storage.rb

@@ -22,6 +22,16 @@ module StubObjectStorage
         background_upload: false,
         direct_upload: false
   )
+    new_config = config.to_h.deep_symbolize_keys.merge({
+      enabled: enabled,
+      proxy_download: proxy_download,
+      background_upload: background_upload,
+      direct_upload: direct_upload
+    })
+
+    # Needed for ObjectStorage::Config compatibility
+    allow(config).to receive(:to_hash).and_return(new_config)
+    allow(config).to receive(:to_h).and_return(new_config)
     allow(config).to receive(:enabled) { enabled }
     allow(config).to receive(:proxy_download) { proxy_download }
     allow(config).to receive(:background_upload) { background_upload }