Merge branch '290813-export-requirements-service' into 'master'

Add Requirement export CSV service

See merge request gitlab-org/gitlab!50449

app/mailers/emails/issues.rb

@@ -106,6 +106,8 @@ module Emails
       @count = export_status.fetch(:rows_expected)
       @written_count = export_status.fetch(:rows_written)
       @truncated = export_status.fetch(:truncated)
+      @size_limit = ActiveSupport::NumberHelper
+        .number_to_human_size(Issuable::ExportCsv::BaseService::TARGET_FILESIZE)
 
       filename = "#{project.full_path.parameterize}_issues_#{Date.today.iso8601}.csv"
       attachments[filename] = { content: csv_data, mime_type: 'text/csv' }

app/mailers/emails/merge_requests.rb

@@ -115,6 +115,8 @@ module Emails
       @count = export_status.fetch(:rows_expected)
       @written_count = export_status.fetch(:rows_written)
       @truncated = export_status.fetch(:truncated)
+      @size_limit = ActiveSupport::NumberHelper
+        .number_to_human_size(Issuable::ExportCsv::BaseService::TARGET_FILESIZE)
 
       filename = "#{project.full_path.parameterize}_merge_requests_#{Date.current.iso8601}.csv"
       attachments[filename] = { content: csv_data, mime_type: 'text/csv' }

app/services/issuable/export_csv/base_service.rb

+# frozen_string_literal: true
+
+module Issuable
+  module ExportCsv
+    class BaseService
+      # Target attachment size before base64 encoding
+      TARGET_FILESIZE = 15.megabytes
+
+      def initialize(issuables_relation, project)
+        @issuables = issuables_relation
+        @project = project
+      end
+
+      def csv_data
+        csv_builder.render(TARGET_FILESIZE)
+      end
+
+      private
+
+      attr_reader :project, :issuables
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def csv_builder
+        @csv_builder ||=
+          CsvBuilder.new(issuables.preload(associations_to_preload), header_to_value_hash)
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def associations_to_preload
+        []
+      end
+
+      def header_to_value_hash
+        raise NotImplementedError
+      end
+    end
+  end
+end

app/services/issues/export_csv_service.rb

 # frozen_string_literal: true
 
 module Issues
-  class ExportCsvService
+  class ExportCsvService < Issuable::ExportCsv::BaseService
     include Gitlab::Routing.url_helpers
     include GitlabRoutingHelper
 
-    # Target attachment size before base64 encoding
-    TARGET_FILESIZE = 15000000
-
-    attr_reader :project
-
-    def initialize(issues_relation, project)
-      @issues = issues_relation
-      @labels = @issues.labels_hash
-      @project = project
-    end
-
-    def csv_data
-      csv_builder.render(TARGET_FILESIZE)
-    end
-
     def email(user)
       Notify.issues_csv_email(user, project, csv_data, csv_builder.status).deliver_now
     end
 
-    # rubocop: disable CodeReuse/ActiveRecord
-    def csv_builder
-      @csv_builder ||=
-        CsvBuilder.new(@issues.preload(associations_to_preload), header_to_value_hash)
-    end
-    # rubocop: enable CodeReuse/ActiveRecord
-
     private
 
     def associations_to_preload
@@ -63,7 +41,7 @@ module Issues
     end
 
     def issue_labels(issue)
-      @labels[issue.id].sort.join(',').presence
+      issuables.labels_hash[issue.id].sort.join(',').presence
     end
 
     # rubocop: disable CodeReuse/ActiveRecord

app/services/merge_requests/export_csv_service.rb

 # frozen_string_literal: true
 
 module MergeRequests
-  class ExportCsvService
+  class ExportCsvService < Issuable::ExportCsv::BaseService
     include Gitlab::Routing.url_helpers
     include GitlabRoutingHelper
 
-    # Target attachment size before base64 encoding
-    TARGET_FILESIZE = 15.megabytes
-
-    def initialize(merge_requests, project)
-      @project = project
-      @merge_requests = merge_requests
-    end
-
-    def csv_data
-      csv_builder.render(TARGET_FILESIZE)
-    end
-
     def email(user)
-      Notify.merge_requests_csv_email(user, @project, csv_data, csv_builder.status).deliver_now
+      Notify.merge_requests_csv_email(user, project, csv_data, csv_builder.status).deliver_now
     end
 
     private
 
-    def csv_builder
-      @csv_builder ||= CsvBuilder.new(@merge_requests.with_csv_entity_associations, header_to_value_hash)
-    end
-
     def header_to_value_hash
       {
         'MR IID' => 'iid',

app/views/notify/_issuable_csv_export.html.haml

@@ -3,4 +3,4 @@
   = _('Your CSV export of %{count} from project %{project_link} has been added to this email as an attachment.').html_safe % { count: pluralize(@written_count, type.to_s.titleize.downcase), project_link: project_link }
   - if @truncated
     %p
-      = _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{count} issues have been included. Consider re-exporting with a narrower selection of issues.') % { written_count: @written_count, count: @count }
+      = _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} %{issuables} have been included. Consider re-exporting with a narrower selection of %{issuables}.') % { written_count: @written_count, count: @count, issuables: type.to_s.pluralize, size_limit: @size_limit }

app/views/notify/issues_csv_email.text.erb

 <%= _('Your CSV export of %{written_count} from project %{project_name} (%{project_url}) has been added to this email as an attachment.') %  { written_count: pluralize(@written_count, 'issue'), project_name: @project.full_name, project_url: project_url(@project) } %>
 
 <% if @truncated %>
-  <%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{issues_count} issues have been included. Consider re-exporting with a narrower selection of issues.') % { written_count: @written_count, issues_count: @issues_count} %>
+  <%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{issues_count} issues have been included. Consider re-exporting with a narrower selection of issues.') % { written_count: @written_count, issues_count: @issues_count, size_limit: @size_limit } %>
 <% end %>

app/views/notify/merge_requests_csv_email.text.erb

 <%= _('Your CSV export of %{written_count} from project %{project_name} (%{project_url}) has been added to this email as an attachment.') %  { written_count: pluralize(@written_count, 'merge request'), project_name: @project.full_name, project_url: project_url(@project) } %>
 
 <% if @truncated %>
-  <%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{merge_requests_count} issues have been included. Consider re-exporting with a narrower selection of issues.') % { written_count: @written_count, merge_requests_count: @merge_requests_count} %>
+  <%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{merge_requests_count} merge requests have been included. Consider re-exporting with a narrower selection of merge requests.') % { written_count: @written_count, merge_requests_count: @merge_requests_count, size_limit: @size_limit} %>
 <% end %>

app/workers/issuable_export_csv_worker.rb

@@ -7,47 +7,45 @@ class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker
   worker_resource_boundary :cpu
   loggable_arguments 2
 
-  PERMITTED_TYPES = [:merge_request, :issue].freeze
-
   def perform(type, current_user_id, project_id, params)
-    @type = type.to_sym
-    check_permitted_type!
-    process_params!(params, project_id)
-
-    @current_user = User.find(current_user_id)
-    @project = Project.find(project_id)
-    @service = service(find_objects(params))
+    user = User.find(current_user_id)
+    project = Project.find(project_id)
+    finder_params = map_params(params, project_id)
 
-    @service.email(@current_user)
+    export_service(type.to_sym, user, project, finder_params).email(user)
+  rescue ActiveRecord::RecordNotFound => error
+    logger.error("Failed to export CSV (current_user_id:#{current_user_id}, project_id:#{project_id}): #{error.message}")
   end
 
   private
 
-  def find_objects(params)
-    case @type
-    when :issue
-      IssuesFinder.new(@current_user, params).execute
-    when :merge_request
-      MergeRequestsFinder.new(@current_user, params).execute
+  def map_params(params, project_id)
+    params
+      .symbolize_keys
+      .except(:sort)
+      .merge(project_id: project_id)
   end
+
+  def export_service(type, user, project, params)
+    issuable_class = service_classes_for(type)
+    issuables = issuable_class[:finder].new(user, params).execute
+    issuable_class[:service].new(issuables, project)
   end
 
-  def service(issuables)
-    case @type
+  def service_classes_for(type)
+    case type
     when :issue
-      Issues::ExportCsvService.new(issuables, @project)
+      { finder: IssuesFinder, service: Issues::ExportCsvService }
     when :merge_request
-      MergeRequests::ExportCsvService.new(issuables, @project)
-    end
+      { finder: MergeRequestsFinder, service: MergeRequests::ExportCsvService }
+    else
+      raise ArgumentError, type_error_message(type)
     end
-
-  def process_params!(params, project_id)
-    params.symbolize_keys!
-    params[:project_id] = project_id
-    params.delete(:sort)
   end
 
-  def check_permitted_type!
-    raise ArgumentError, "type parameter must be :issue or :merge_request, it was #{@type}" unless PERMITTED_TYPES.include?(@type)
+  def type_error_message(type)
+    "Type parameter must be :issue or :merge_request, it was #{type}"
   end
 end
+
+IssuableExportCsvWorker.prepend_if_ee('::EE::IssuableExportCsvWorker')

ee/app/mailers/ee/preview/notify_preview.rb

@@ -41,7 +41,14 @@ module EE
         end
 
         def import_requirements_csv_email
-          Notify.import_requirements_csv_email(user.id, project.id, { success: 3, errors: [5, 6, 7], valid_file: true })
+          ::Notify.import_requirements_csv_email(user.id, project.id, { success: 3, errors: [5, 6, 7], valid_file: true })
+        end
+
+        def requirements_csv_email
+          ::Notify.requirements_csv_email(
+            user, project, 'requirement1,requirement2,requirement3',
+            { truncated: false, rows_expected: 3, rows_written: 3 }
+          ).message
         end
       end
 

ee/app/mailers/emails/requirements.rb

@@ -10,6 +10,17 @@ module Emails
       requirement_email_with_layout(@user, @project.group, _('Imported requirements'))
     end
 
+    def requirements_csv_email(user, project, csv_data, export_status)
+      @project = project
+      @count, @written_count, @truncated = export_status.fetch_values(:rows_expected, :rows_written, :truncated)
+      @size_limit = ActiveSupport::NumberHelper.number_to_human_size(Issuable::ExportCsv::BaseService::TARGET_FILESIZE)
+
+      filename = "#{project.full_path.parameterize}_requirements_#{Date.current.iso8601}.csv"
+      attachments[filename] = { content: csv_data, mime_type: 'text/csv' }
+
+      requirement_email_with_layout(user, @project.group, _('Exported requirements'))
+    end
+
     def requirement_email_with_layout(user, group, subj)
       mail(to: user.notification_email_for(group), subject: subject(subj)) do |format|
         format.html { render layout: 'mailer' }

ee/app/services/requirements_management/export_csv_service.rb

+# frozen_string_literal: true
+
+module RequirementsManagement
+  class ExportCsvService < ::Issuable::ExportCsv::BaseService
+    def email(user)
+      Notify.requirements_csv_email(user, project, csv_data, csv_builder.status).deliver_now
+    end
+
+    private
+
+    def associations_to_preload
+      %i(author test_reports)
+    end
+
+    def header_to_value_hash
+      {
+        'Requirement ID' => 'iid',
+        'Title' => 'title',
+        'Description' => 'description',
+        'Author Username' => -> (requirement) { requirement.author&.username },
+        'Latest Test Report State' => -> (requirement) { requirement.last_test_report_state },
+        'Latest Test Report Created At (UTC)' => -> (requirement) { latest_test_report_time(requirement) }
+      }
+    end
+
+    def latest_test_report_time(requirement)
+      requirement.test_reports.last&.created_at
+    end
+  end
+end

ee/app/views/notify/requirements_csv_email.html.haml

+= render 'issuable_csv_export', type: :requirement

ee/app/views/notify/requirements_csv_email.text.erb

+<%= _('Your CSV export of %{written_count} from project %{project_name} (%{project_url}) has been added to this email as an attachment.') %  { written_count: pluralize(@written_count, 'requirement'), project_name: @project.full_name, project_url: project_url(@project) } %>
+
+<% if @truncated %>
+  <%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{requirements_count} requirements have been included. Consider re-exporting with a narrower selection of requirements.') % { written_count: @written_count, requirements_count: @requirements_count, size_limit: @size_limit} %>
+<% end %>

ee/app/workers/ee/issuable_export_csv_worker.rb

+# frozen_string_literal: true
+module EE
+  # PostReceive EE mixin
+  #
+  # This module is intended to encapsulate EE-specific model logic
+  # and be prepended in the `IssuableExportCsvWorker` worker
+  module IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker
+    extend ::Gitlab::Utils::Override
+
+    private
+
+    override :service_classes_for
+    def service_classes_for(type)
+      return super unless type == :requirement
+
+      { finder: ::RequirementsManagement::RequirementsFinder, service: ::RequirementsManagement::ExportCsvService }
+    end
+
+    override :type_error_message
+    def type_error_message(type)
+      "Type parameter must be :issue, :merge_request, or :requirements, it was #{type}"
+    end
+  end
+end

ee/changelogs/unreleased/290813-export-requirements-service.yml

+---
+title: Allow Requirements to be exported as a CSV file
+merge_request: 50449
+author:
+type: added

ee/spec/mailers/emails/requirements_spec.rb

@@ -5,9 +5,10 @@ require 'spec_helper'
 RSpec.describe Emails::Requirements do
   include EmailSpec::Matchers
 
-  describe "#import_requirements_csv_email" do
   let_it_be(:user) { create(:user) }
   let_it_be(:project) { create(:project) }
+
+  describe "#import_requirements_csv_email" do
     let(:results) { { success: 0, error_lines: [], parse_error: false } }
 
     subject { Notify.import_requirements_csv_email(user.id, project.id, results) }
@@ -39,4 +40,39 @@ RSpec.describe Emails::Requirements do
       it_behaves_like 'appearance header and footer not enabled'
     end
   end
+
+  describe '#requirements_csv_email' do
+    let_it_be(:requirements) { create_list(:requirement, 10) }
+    let(:export_status) do
+      {
+        rows_expected: 10,
+        rows_written: 10,
+        truncated: false
+      }
+    end
+
+    let_it_be(:csv_data) do
+      RequirementsManagement::ExportCsvService
+        .new(RequirementsManagement::Requirement.all, project).csv_data
+    end
+
+    subject { Notify.requirements_csv_email(user, project, csv_data, export_status) }
+
+    specify { expect(subject.subject).to eq("#{project.name} | Exported requirements") }
+    specify { expect(subject.to).to contain_exactly(user.notification_email_for(project.group)) }
+    specify { expect(subject.html_part).to have_content("Your CSV export of 10 requirements from project") }
+    specify { expect(subject.text_part).to have_content("Your CSV export of 10 requirements from project") }
+
+    context 'when truncated' do
+      let(:export_status) do
+        {
+            rows_expected: 10,
+            rows_written: 10,
+            truncated: true
+        }
+      end
+
+      specify { expect(subject).to have_content('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15 MB.') }
+    end
+  end
 end

ee/spec/services/requirements_management/export_csv_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe RequirementsManagement::ExportCsvService do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, :public, group: group) }
+  let_it_be(:requirement) { create(:requirement, state: :opened, project: project) }
+
+  subject { described_class.new(RequirementsManagement::Requirement.all, project) }
+
+  before do
+    stub_licensed_features(requirements: true)
+  end
+
+  it 'renders csv to string' do
+    expect(subject.csv_data).to be_a String
+  end
+
+  describe '#email' do
+    it 'emails csv' do
+      expect { subject.email(user) }.to change(ActionMailer::Base.deliveries, :count).from(0).to(1)
+    end
+
+    it 'renders with a target filesize' do
+      expect_next_instance_of(CsvBuilder) do |csv_builder|
+        expect(csv_builder).to receive(:render).with(described_class::TARGET_FILESIZE).once
+      end
+
+      subject.email(user)
+    end
+  end
+
+  def csv
+    CSV.parse(subject.csv_data, headers: true)
+  end
+
+  context 'includes' do
+    before do
+      create(
+        :test_report, requirement: requirement,
+        state: :failed, build: nil,
+        created_at: DateTime.new(2015, 4, 2, 2, 1, 0)
+      )
+      create(
+        :test_report, requirement: requirement,
+        state: :passed, build: nil,
+        created_at: DateTime.new(2015, 4, 3, 2, 1, 0)
+      )
+    end
+
+    it 'includes the columns required for import' do
+      expect(csv.headers).to include('Title', 'Description')
+    end
+
+    specify 'iid' do
+      expect(csv[0]['Requirement ID']).to eq requirement.iid.to_s
+    end
+
+    specify 'title' do
+      expect(csv[0]['Title']).to eq requirement.title
+    end
+
+    specify 'description' do
+      expect(csv[0]['Description']).to eq requirement.description
+    end
+
+    specify 'author username' do
+      expect(csv[0]['Author Username']).to eq requirement.author.username
+    end
+
+    specify 'latest test report state' do
+      expect(csv[0]['Latest Test Report State']).to eq "passed"
+    end
+
+    specify 'latest test report created at' do
+      expect(csv[0]['Latest Test Report Created At (UTC)']).to eq '2015-04-03 02:01:00 UTC'
+    end
+  end
+end

ee/spec/workers/ee/issuable_export_csv_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe IssuableExportCsvWorker do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, creator: user) }
+  let(:params) { {} }
+
+  subject { described_class.new.perform(issuable_type, user.id, project.id, params) }
+
+  context 'when issuable type is :requirement' do
+    let(:issuable_type) { :requirement }
+
+    it 'emails a CSV' do
+      expect { subject }.to change(ActionMailer::Base.deliveries, :size).by(1)
+    end
+
+    it 'calls the Requirements export service' do
+      expect(RequirementsManagement::ExportCsvService).to receive(:new).with(anything, project).once.and_call_original
+
+      subject
+    end
+
+    it 'calls the Requirements finder' do
+      expect(RequirementsManagement::RequirementsFinder).to receive(:new).once.and_call_original
+
+      subject
+    end
+
+    context 'with record not found' do
+      let(:logger) { described_class.new.send(:logger) }
+
+      it 'an error is logged if user not found' do
+        message = "Failed to export CSV (current_user_id:#{non_existing_record_id}, "\
+          "project_id:#{project.id}): Couldn't find User with 'id'=#{non_existing_record_id}"
+
+        expect(logger).to receive(:error).with(message).once
+
+        described_class.new.perform(issuable_type, non_existing_record_id, project.id, params)
+      end
+
+      it 'an error is logged if project not found' do
+        message = "Failed to export CSV (current_user_id:#{user.id}, "\
+          "project_id:#{non_existing_record_id}): Couldn't find Project with 'id'=#{non_existing_record_id}"
+
+        expect(logger).to receive(:error).with(message).once
+
+        described_class.new.perform(issuable_type, user.id, non_existing_record_id, params)
+      end
+    end
+  end
+
+  context 'when issuable type is not :requirement' do
+    context 'with a valid type' do
+      let(:issuable_type) { :issue }
+
+      it 'does not raise an exception' do
+        expect { subject }.not_to raise_error
+      end
+    end
+
+    context 'with an invalid type' do
+      let(:issuable_type) { :test }
+
+      it 'raises an exception with expected message' do
+        expect { subject }.to raise_error(
+          ArgumentError,
+          'Type parameter must be :issue, :merge_request, or :requirements, it was test'
+        )
+      end
+    end
+  end
+end

locale/gitlab.pot

@@ -11703,6 +11703,9 @@ msgstr ""
 msgid "Export variable to pipelines running on protected branches and tags only."
 msgstr ""
 
+msgid "Exported requirements"
+msgstr ""
+
 msgid "External Classification Policy Authorization"
 msgstr ""
 
@@ -28565,13 +28568,16 @@ msgstr ""
 msgid "This application will be able to:"
 msgstr ""
 
-msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{count} issues have been included. Consider re-exporting with a narrower selection of issues."
+msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} %{issuables} have been included. Consider re-exporting with a narrower selection of %{issuables}."
+msgstr ""
+
+msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{issues_count} issues have been included. Consider re-exporting with a narrower selection of issues."
 msgstr ""
 
-msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{issues_count} issues have been included. Consider re-exporting with a narrower selection of issues."
+msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{merge_requests_count} merge requests have been included. Consider re-exporting with a narrower selection of merge requests."
 msgstr ""
 
-msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{merge_requests_count} issues have been included. Consider re-exporting with a narrower selection of issues."
+msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{requirements_count} requirements have been included. Consider re-exporting with a narrower selection of requirements."
 msgstr ""
 
 msgid "This block is self-referential"

spec/mailers/emails/merge_requests_spec.rb

@@ -64,7 +64,7 @@ RSpec.describe Emails::MergeRequests do
         }
       end
 
-      it { expect(subject).to have_content('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB.') }
+      it { expect(subject).to have_content('attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15 MB.') }
     end
   end
 end

spec/services/issues/export_csv_service_spec.rb

@@ -20,7 +20,9 @@ RSpec.describe Issues::ExportCsvService do
     end
 
     it 'renders with a target filesize' do
-      expect(subject.csv_builder).to receive(:render).with(described_class::TARGET_FILESIZE)
+      expect_next_instance_of(CsvBuilder) do |csv_builder|
+        expect(csv_builder).to receive(:render).with(described_class::TARGET_FILESIZE).once
+      end
 
       subject.email(user)
     end