Commit dbd28981 authored by Ash McKenzie's avatar Ash McKenzie

Merge branch '290813-export-requirements-service' into 'master'

Add Requirement export CSV service

See merge request gitlab-org/gitlab!50449
parents e0c0b628 5da3a52e
......@@ -106,6 +106,8 @@ module Emails
@count = export_status.fetch(:rows_expected)
@written_count = export_status.fetch(:rows_written)
@truncated = export_status.fetch(:truncated)
@size_limit = ActiveSupport::NumberHelper
.number_to_human_size(Issuable::ExportCsv::BaseService::TARGET_FILESIZE)
filename = "#{project.full_path.parameterize}_issues_#{Date.today.iso8601}.csv"
attachments[filename] = { content: csv_data, mime_type: 'text/csv' }
......
......@@ -115,6 +115,8 @@ module Emails
@count = export_status.fetch(:rows_expected)
@written_count = export_status.fetch(:rows_written)
@truncated = export_status.fetch(:truncated)
@size_limit = ActiveSupport::NumberHelper
.number_to_human_size(Issuable::ExportCsv::BaseService::TARGET_FILESIZE)
filename = "#{project.full_path.parameterize}_merge_requests_#{Date.current.iso8601}.csv"
attachments[filename] = { content: csv_data, mime_type: 'text/csv' }
......
# frozen_string_literal: true
module Issuable
module ExportCsv
class BaseService
# Target attachment size before base64 encoding
TARGET_FILESIZE = 15.megabytes
def initialize(issuables_relation, project)
@issuables = issuables_relation
@project = project
end
def csv_data
csv_builder.render(TARGET_FILESIZE)
end
private
attr_reader :project, :issuables
# rubocop: disable CodeReuse/ActiveRecord
def csv_builder
@csv_builder ||=
CsvBuilder.new(issuables.preload(associations_to_preload), header_to_value_hash)
end
# rubocop: enable CodeReuse/ActiveRecord
def associations_to_preload
[]
end
def header_to_value_hash
raise NotImplementedError
end
end
end
end
# frozen_string_literal: true
module Issues
class ExportCsvService
class ExportCsvService < Issuable::ExportCsv::BaseService
include Gitlab::Routing.url_helpers
include GitlabRoutingHelper
# Target attachment size before base64 encoding
TARGET_FILESIZE = 15000000
attr_reader :project
def initialize(issues_relation, project)
@issues = issues_relation
@labels = @issues.labels_hash
@project = project
end
def csv_data
csv_builder.render(TARGET_FILESIZE)
end
def email(user)
Notify.issues_csv_email(user, project, csv_data, csv_builder.status).deliver_now
end
# rubocop: disable CodeReuse/ActiveRecord
def csv_builder
@csv_builder ||=
CsvBuilder.new(@issues.preload(associations_to_preload), header_to_value_hash)
end
# rubocop: enable CodeReuse/ActiveRecord
private
def associations_to_preload
......@@ -63,7 +41,7 @@ module Issues
end
def issue_labels(issue)
@labels[issue.id].sort.join(',').presence
issuables.labels_hash[issue.id].sort.join(',').presence
end
# rubocop: disable CodeReuse/ActiveRecord
......
# frozen_string_literal: true
module MergeRequests
class ExportCsvService
class ExportCsvService < Issuable::ExportCsv::BaseService
include Gitlab::Routing.url_helpers
include GitlabRoutingHelper
# Target attachment size before base64 encoding
TARGET_FILESIZE = 15.megabytes
def initialize(merge_requests, project)
@project = project
@merge_requests = merge_requests
end
def csv_data
csv_builder.render(TARGET_FILESIZE)
end
def email(user)
Notify.merge_requests_csv_email(user, @project, csv_data, csv_builder.status).deliver_now
Notify.merge_requests_csv_email(user, project, csv_data, csv_builder.status).deliver_now
end
private
def csv_builder
@csv_builder ||= CsvBuilder.new(@merge_requests.with_csv_entity_associations, header_to_value_hash)
end
def header_to_value_hash
{
'MR IID' => 'iid',
......
......@@ -3,4 +3,4 @@
= _('Your CSV export of %{count} from project %{project_link} has been added to this email as an attachment.').html_safe % { count: pluralize(@written_count, type.to_s.titleize.downcase), project_link: project_link }
- if @truncated
%p
= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{count} issues have been included. Consider re-exporting with a narrower selection of issues.') % { written_count: @written_count, count: @count }
= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} %{issuables} have been included. Consider re-exporting with a narrower selection of %{issuables}.') % { written_count: @written_count, count: @count, issuables: type.to_s.pluralize, size_limit: @size_limit }
<%= _('Your CSV export of %{written_count} from project %{project_name} (%{project_url}) has been added to this email as an attachment.') % { written_count: pluralize(@written_count, 'issue'), project_name: @project.full_name, project_url: project_url(@project) } %>
<% if @truncated %>
<%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{issues_count} issues have been included. Consider re-exporting with a narrower selection of issues.') % { written_count: @written_count, issues_count: @issues_count} %>
<%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{issues_count} issues have been included. Consider re-exporting with a narrower selection of issues.') % { written_count: @written_count, issues_count: @issues_count, size_limit: @size_limit } %>
<% end %>
<%= _('Your CSV export of %{written_count} from project %{project_name} (%{project_url}) has been added to this email as an attachment.') % { written_count: pluralize(@written_count, 'merge request'), project_name: @project.full_name, project_url: project_url(@project) } %>
<% if @truncated %>
<%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{merge_requests_count} issues have been included. Consider re-exporting with a narrower selection of issues.') % { written_count: @written_count, merge_requests_count: @merge_requests_count} %>
<%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{merge_requests_count} merge requests have been included. Consider re-exporting with a narrower selection of merge requests.') % { written_count: @written_count, merge_requests_count: @merge_requests_count, size_limit: @size_limit} %>
<% end %>
......@@ -7,47 +7,45 @@ class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker
worker_resource_boundary :cpu
loggable_arguments 2
PERMITTED_TYPES = [:merge_request, :issue].freeze
def perform(type, current_user_id, project_id, params)
@type = type.to_sym
check_permitted_type!
process_params!(params, project_id)
@current_user = User.find(current_user_id)
@project = Project.find(project_id)
@service = service(find_objects(params))
user = User.find(current_user_id)
project = Project.find(project_id)
finder_params = map_params(params, project_id)
@service.email(@current_user)
export_service(type.to_sym, user, project, finder_params).email(user)
rescue ActiveRecord::RecordNotFound => error
logger.error("Failed to export CSV (current_user_id:#{current_user_id}, project_id:#{project_id}): #{error.message}")
end
private
def find_objects(params)
case @type
when :issue
IssuesFinder.new(@current_user, params).execute
when :merge_request
MergeRequestsFinder.new(@current_user, params).execute
end
def map_params(params, project_id)
params
.symbolize_keys
.except(:sort)
.merge(project_id: project_id)
end
def service(issuables)
case @type
def export_service(type, user, project, params)
issuable_class = service_classes_for(type)
issuables = issuable_class[:finder].new(user, params).execute
issuable_class[:service].new(issuables, project)
end
def service_classes_for(type)
case type
when :issue
Issues::ExportCsvService.new(issuables, @project)
{ finder: IssuesFinder, service: Issues::ExportCsvService }
when :merge_request
MergeRequests::ExportCsvService.new(issuables, @project)
{ finder: MergeRequestsFinder, service: MergeRequests::ExportCsvService }
else
raise ArgumentError, type_error_message(type)
end
end
def process_params!(params, project_id)
params.symbolize_keys!
params[:project_id] = project_id
params.delete(:sort)
end
def check_permitted_type!
raise ArgumentError, "type parameter must be :issue or :merge_request, it was #{@type}" unless PERMITTED_TYPES.include?(@type)
def type_error_message(type)
"Type parameter must be :issue or :merge_request, it was #{type}"
end
end
IssuableExportCsvWorker.prepend_if_ee('::EE::IssuableExportCsvWorker')
......@@ -41,7 +41,14 @@ module EE
end
def import_requirements_csv_email
Notify.import_requirements_csv_email(user.id, project.id, { success: 3, errors: [5, 6, 7], valid_file: true })
::Notify.import_requirements_csv_email(user.id, project.id, { success: 3, errors: [5, 6, 7], valid_file: true })
end
def requirements_csv_email
::Notify.requirements_csv_email(
user, project, 'requirement1,requirement2,requirement3',
{ truncated: false, rows_expected: 3, rows_written: 3 }
).message
end
end
......
......@@ -10,6 +10,17 @@ module Emails
requirement_email_with_layout(@user, @project.group, _('Imported requirements'))
end
def requirements_csv_email(user, project, csv_data, export_status)
@project = project
@count, @written_count, @truncated = export_status.fetch_values(:rows_expected, :rows_written, :truncated)
@size_limit = ActiveSupport::NumberHelper.number_to_human_size(Issuable::ExportCsv::BaseService::TARGET_FILESIZE)
filename = "#{project.full_path.parameterize}_requirements_#{Date.current.iso8601}.csv"
attachments[filename] = { content: csv_data, mime_type: 'text/csv' }
requirement_email_with_layout(user, @project.group, _('Exported requirements'))
end
def requirement_email_with_layout(user, group, subj)
mail(to: user.notification_email_for(group), subject: subject(subj)) do |format|
format.html { render layout: 'mailer' }
......
# frozen_string_literal: true
module RequirementsManagement
class ExportCsvService < ::Issuable::ExportCsv::BaseService
def email(user)
Notify.requirements_csv_email(user, project, csv_data, csv_builder.status).deliver_now
end
private
def associations_to_preload
%i(author test_reports)
end
def header_to_value_hash
{
'Requirement ID' => 'iid',
'Title' => 'title',
'Description' => 'description',
'Author Username' => -> (requirement) { requirement.author&.username },
'Latest Test Report State' => -> (requirement) { requirement.last_test_report_state },
'Latest Test Report Created At (UTC)' => -> (requirement) { latest_test_report_time(requirement) }
}
end
def latest_test_report_time(requirement)
requirement.test_reports.last&.created_at
end
end
end
= render 'issuable_csv_export', type: :requirement
<%= _('Your CSV export of %{written_count} from project %{project_name} (%{project_url}) has been added to this email as an attachment.') % { written_count: pluralize(@written_count, 'requirement'), project_name: @project.full_name, project_url: project_url(@project) } %>
<% if @truncated %>
<%= _('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{requirements_count} requirements have been included. Consider re-exporting with a narrower selection of requirements.') % { written_count: @written_count, requirements_count: @requirements_count, size_limit: @size_limit} %>
<% end %>
# frozen_string_literal: true
module EE
# PostReceive EE mixin
#
# This module is intended to encapsulate EE-specific model logic
# and be prepended in the `IssuableExportCsvWorker` worker
module IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker
extend ::Gitlab::Utils::Override
private
override :service_classes_for
def service_classes_for(type)
return super unless type == :requirement
{ finder: ::RequirementsManagement::RequirementsFinder, service: ::RequirementsManagement::ExportCsvService }
end
override :type_error_message
def type_error_message(type)
"Type parameter must be :issue, :merge_request, or :requirements, it was #{type}"
end
end
end
---
title: Allow Requirements to be exported as a CSV file
merge_request: 50449
author:
type: added
......@@ -5,9 +5,10 @@ require 'spec_helper'
RSpec.describe Emails::Requirements do
include EmailSpec::Matchers
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project) }
describe "#import_requirements_csv_email" do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project) }
let(:results) { { success: 0, error_lines: [], parse_error: false } }
subject { Notify.import_requirements_csv_email(user.id, project.id, results) }
......@@ -39,4 +40,39 @@ RSpec.describe Emails::Requirements do
it_behaves_like 'appearance header and footer not enabled'
end
end
describe '#requirements_csv_email' do
let_it_be(:requirements) { create_list(:requirement, 10) }
let(:export_status) do
{
rows_expected: 10,
rows_written: 10,
truncated: false
}
end
let_it_be(:csv_data) do
RequirementsManagement::ExportCsvService
.new(RequirementsManagement::Requirement.all, project).csv_data
end
subject { Notify.requirements_csv_email(user, project, csv_data, export_status) }
specify { expect(subject.subject).to eq("#{project.name} | Exported requirements") }
specify { expect(subject.to).to contain_exactly(user.notification_email_for(project.group)) }
specify { expect(subject.html_part).to have_content("Your CSV export of 10 requirements from project") }
specify { expect(subject.text_part).to have_content("Your CSV export of 10 requirements from project") }
context 'when truncated' do
let(:export_status) do
{
rows_expected: 10,
rows_written: 10,
truncated: true
}
end
specify { expect(subject).to have_content('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15 MB.') }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe RequirementsManagement::ExportCsvService do
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, :public, group: group) }
let_it_be(:requirement) { create(:requirement, state: :opened, project: project) }
subject { described_class.new(RequirementsManagement::Requirement.all, project) }
before do
stub_licensed_features(requirements: true)
end
it 'renders csv to string' do
expect(subject.csv_data).to be_a String
end
describe '#email' do
it 'emails csv' do
expect { subject.email(user) }.to change(ActionMailer::Base.deliveries, :count).from(0).to(1)
end
it 'renders with a target filesize' do
expect_next_instance_of(CsvBuilder) do |csv_builder|
expect(csv_builder).to receive(:render).with(described_class::TARGET_FILESIZE).once
end
subject.email(user)
end
end
def csv
CSV.parse(subject.csv_data, headers: true)
end
context 'includes' do
before do
create(
:test_report, requirement: requirement,
state: :failed, build: nil,
created_at: DateTime.new(2015, 4, 2, 2, 1, 0)
)
create(
:test_report, requirement: requirement,
state: :passed, build: nil,
created_at: DateTime.new(2015, 4, 3, 2, 1, 0)
)
end
it 'includes the columns required for import' do
expect(csv.headers).to include('Title', 'Description')
end
specify 'iid' do
expect(csv[0]['Requirement ID']).to eq requirement.iid.to_s
end
specify 'title' do
expect(csv[0]['Title']).to eq requirement.title
end
specify 'description' do
expect(csv[0]['Description']).to eq requirement.description
end
specify 'author username' do
expect(csv[0]['Author Username']).to eq requirement.author.username
end
specify 'latest test report state' do
expect(csv[0]['Latest Test Report State']).to eq "passed"
end
specify 'latest test report created at' do
expect(csv[0]['Latest Test Report Created At (UTC)']).to eq '2015-04-03 02:01:00 UTC'
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe IssuableExportCsvWorker do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, creator: user) }
let(:params) { {} }
subject { described_class.new.perform(issuable_type, user.id, project.id, params) }
context 'when issuable type is :requirement' do
let(:issuable_type) { :requirement }
it 'emails a CSV' do
expect { subject }.to change(ActionMailer::Base.deliveries, :size).by(1)
end
it 'calls the Requirements export service' do
expect(RequirementsManagement::ExportCsvService).to receive(:new).with(anything, project).once.and_call_original
subject
end
it 'calls the Requirements finder' do
expect(RequirementsManagement::RequirementsFinder).to receive(:new).once.and_call_original
subject
end
context 'with record not found' do
let(:logger) { described_class.new.send(:logger) }
it 'an error is logged if user not found' do
message = "Failed to export CSV (current_user_id:#{non_existing_record_id}, "\
"project_id:#{project.id}): Couldn't find User with 'id'=#{non_existing_record_id}"
expect(logger).to receive(:error).with(message).once
described_class.new.perform(issuable_type, non_existing_record_id, project.id, params)
end
it 'an error is logged if project not found' do
message = "Failed to export CSV (current_user_id:#{user.id}, "\
"project_id:#{non_existing_record_id}): Couldn't find Project with 'id'=#{non_existing_record_id}"
expect(logger).to receive(:error).with(message).once
described_class.new.perform(issuable_type, user.id, non_existing_record_id, params)
end
end
end
context 'when issuable type is not :requirement' do
context 'with a valid type' do
let(:issuable_type) { :issue }
it 'does not raise an exception' do
expect { subject }.not_to raise_error
end
end
context 'with an invalid type' do
let(:issuable_type) { :test }
it 'raises an exception with expected message' do
expect { subject }.to raise_error(
ArgumentError,
'Type parameter must be :issue, :merge_request, or :requirements, it was test'
)
end
end
end
end
......@@ -11703,6 +11703,9 @@ msgstr ""
msgid "Export variable to pipelines running on protected branches and tags only."
msgstr ""
msgid "Exported requirements"
msgstr ""
msgid "External Classification Policy Authorization"
msgstr ""
......@@ -28565,13 +28568,16 @@ msgstr ""
msgid "This application will be able to:"
msgstr ""
msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{count} issues have been included. Consider re-exporting with a narrower selection of issues."
msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} %{issuables} have been included. Consider re-exporting with a narrower selection of %{issuables}."
msgstr ""
msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{issues_count} issues have been included. Consider re-exporting with a narrower selection of issues."
msgstr ""
msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{issues_count} issues have been included. Consider re-exporting with a narrower selection of issues."
msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{merge_requests_count} merge requests have been included. Consider re-exporting with a narrower selection of merge requests."
msgstr ""
msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB. %{written_count} of %{merge_requests_count} issues have been included. Consider re-exporting with a narrower selection of issues."
msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{requirements_count} requirements have been included. Consider re-exporting with a narrower selection of requirements."
msgstr ""
msgid "This block is self-referential"
......
......@@ -64,7 +64,7 @@ RSpec.describe Emails::MergeRequests do
}
end
it { expect(subject).to have_content('This attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15MB.') }
it { expect(subject).to have_content('attachment has been truncated to avoid exceeding the maximum allowed attachment size of 15 MB.') }
end
end
end
......@@ -20,7 +20,9 @@ RSpec.describe Issues::ExportCsvService do
end
it 'renders with a target filesize' do
expect(subject.csv_builder).to receive(:render).with(described_class::TARGET_FILESIZE)
expect_next_instance_of(CsvBuilder) do |csv_builder|
expect(csv_builder).to receive(:render).with(described_class::TARGET_FILESIZE).once
end
subject.email(user)
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment