Merge branch '246742-stream-csv-file' into 'master'

Stream audit event CSV file

See merge request gitlab-org/gitlab!47268

app/controllers/application_controller.rb

@@ -266,6 +266,12 @@ class ApplicationController < ActionController::Base
     end
   end
 
+  def stream_headers
+    headers['Content-Length'] = nil
+    headers['X-Accel-Buffering'] = 'no' # Disable buffering on Nginx
+    headers['Last-Modified'] = '0' # Prevent buffering via Rack::ETag middleware
+  end
+
   def default_headers
     headers['X-Frame-Options'] = 'DENY'
     headers['X-XSS-Protection'] = '1; mode=block'

doc/administration/audit_events.md

@@ -239,7 +239,7 @@ The first row contains the headers, which are listed in the following table alon
 
 ### Limitation
 
-The Audit Log CSV file size is limited to a maximum of `15 MB`.
+The Audit Log CSV file size is limited to a maximum of `100,000` events.
 The remaining records are truncated when this limit is reached.
 
 ### Enable or disable Audit Log Export to CSV

ee/app/assets/javascripts/audit_events/components/audit_events_export_button.vue

@@ -17,7 +17,7 @@ export default {
   },
   strings: {
     buttonText: __('Export as CSV'),
-    tooltipText: __('Max size 15 MB'),
+    tooltipText: __('Max 100,000 events'),
   },
 };
 </script>

ee/app/controllers/admin/audit_log_reports_controller.rb

@@ -10,11 +10,13 @@ class Admin::AuditLogReportsController < Admin::ApplicationController
 
     respond_to do |format|
       format.csv do
-        send_data(
-          csv_data,
-          type: 'text/csv; charset=utf-8; header=present',
-          filename: csv_filename
-        )
+        no_cache_headers
+        stream_headers
+
+        headers['Content-Type'] = 'text/csv; charset=utf-8; header=present'
+        headers['Content-Disposition'] = "attachment; filename=\"#{csv_filename}\""
+
+        self.response_body = csv_data
       end
     end
   end

ee/app/services/audit_events/export_csv_service.rb

@@ -2,20 +2,18 @@
 
 module AuditEvents
   class ExportCsvService
-    TARGET_FILESIZE = 15.megabytes
-
     def initialize(params = {})
       @params = params
     end
 
     def csv_data
-      csv_builder.render(TARGET_FILESIZE)
+      csv_builder.render
     end
 
     private
 
     def csv_builder
-      @csv_builder ||= CsvBuilder.new(data, header_to_value_hash)
+      @csv_builder ||= CsvBuilders::Stream.new(data, header_to_value_hash)
     end
 
     def data

ee/spec/controllers/admin/audit_log_reports_controller_spec.rb

@@ -54,7 +54,11 @@ RSpec.describe Admin::AuditLogReportsController do
             subject
 
             expect(response).to have_gitlab_http_status(:ok)
-            expect(response.headers['Content-Type']).to eq('text/csv; charset=utf-8')
+            expect(response.headers["Content-Length"]).to be_nil
+            expect(response.headers["Cache-Control"]).to eq('no-cache, no-store')
+            expect(response.headers['Content-Type']).to eq('text/csv; charset=utf-8; header=present')
+            expect(response.headers['X-Accel-Buffering']).to eq('no')
+            expect(response.headers['Last-Modified']).to eq('0')
             expect(response.headers['Content-Disposition'])
               .to include("filename=\"audit-events-#{Time.current.to_i}.csv\"")
           end

ee/spec/frontend/audit_events/components/__snapshots__/audit_events_export_button_spec.js.snap

@@ -7,7 +7,7 @@ exports[`AuditEventsExportButton component Audit Events CSV export button matche
   href="http://example.com/audit_log_reports.csv?created_after=2020-12-12"
   icon="export"
   size="medium"
-  title="Max size 15 MB"
+  title="Max 100,000 events"
   variant="default"
 >
   

ee/spec/services/audit_events/export_csv_service_spec.rb

@@ -29,16 +29,9 @@ RSpec.describe AuditEvents::ExportCsvService do
     }
   end
 
-  subject { described_class.new(params) }
+  let(:export_csv_service) { described_class.new(params) }
 
-  it 'invokes the CSV builder with correct limit' do
-    csv_builder = instance_spy(CsvBuilder)
-    allow(CsvBuilder).to receive(:new).and_return(csv_builder)
-
-    subject.csv_data
-
-    expect(csv_builder).to have_received(:render).with(15.megabytes)
-  end
+  subject(:csv) { CSV.parse(export_csv_service.csv_data.to_a.join, headers: true) }
 
   it 'includes the appropriate headers' do
     expect(csv.headers).to eq([
@@ -98,8 +91,4 @@ RSpec.describe AuditEvents::ExportCsvService do
       expect(csv[0]['Created At (UTC)']).to eq('2020-02-20T12:00:00Z')
     end
   end
-
-  def csv
-    CSV.parse(subject.csv_data, headers: true)
-  end
 end

lib/csv_builders/stream.rb

+# frozen_string_literal: true
+
+module CsvBuilders
+  class Stream < CsvBuilder
+    def render(max_rows = 100_000)
+      max_rows_including_header = max_rows + 1
+
+      Enumerator.new do |csv|
+        csv << CSV.generate_line(headers)
+
+        each do |object|
+          csv << CSV.generate_line(row(object))
+        end
+      end.lazy.take(max_rows_including_header) # rubocop: disable CodeReuse/ActiveRecord
+    end
+  end
+end

locale/gitlab.pot

@@ -16536,6 +16536,9 @@ msgstr ""
 msgid "MattermostService|This service allows users to perform common operations on this project by entering slash commands in Mattermost."
 msgstr ""
 
+msgid "Max 100,000 events"
+msgstr ""
+
 msgid "Max Group Export Download requests per minute per user"
 msgstr ""
 
@@ -16560,9 +16563,6 @@ msgstr ""
 msgid "Max role"
 msgstr ""
 
-msgid "Max size 15 MB"
-msgstr ""
-
 msgid "MaxBuilds"
 msgstr ""
 

spec/lib/csv_builders/stream_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe CsvBuilders::Stream do
+  let(:event_1) { double(title: 'Added salt', description: 'A teaspoon') }
+  let(:event_2) { double(title: 'Added sugar', description: 'Just a pinch') }
+  let(:fake_relation) { FakeRelation.new([event_1, event_2]) }
+
+  subject(:builder) { described_class.new(fake_relation, 'Title' => 'title', 'Description' => 'description') }
+
+  describe '#render' do
+    before do
+      stub_const('FakeRelation', Array)
+
+      FakeRelation.class_eval do
+        def find_each(&block)
+          each(&block)
+        end
+      end
+    end
+
+    it 'returns a lazy enumerator' do
+      expect(builder.render).to be_an(Enumerator::Lazy)
+    end
+
+    it 'returns all rows up to default max value' do
+      expect(builder.render.to_a).to eq([
+        "Title,Description\n",
+        "Added salt,A teaspoon\n",
+        "Added sugar,Just a pinch\n"
+      ])
+    end
+
+    it 'truncates to max rows' do
+      expect(builder.render(1).to_a).to eq([
+        "Title,Description\n",
+        "Added salt,A teaspoon\n"
+      ])
+    end
+  end
+end