Commit 2d1e7c20 authored by Jan Provaznik's avatar Jan Provaznik

Merge branch 'issue_323779-migrate_requirements_into_issues' into 'master'

Background migration to move requirement to work items

See merge request gitlab-org/gitlab!71778
parents db81c782 d787d486
# frozen_string_literal: true
class ScheduleRequirementsMigration < Gitlab::Database::Migration[1.0]
DOWNTIME = false
# 2021-10-05 requirements count: ~12500
#
# Using 30 as batch size and 120 seconds default interval will produce:
# ~420 jobs - taking ~14 hours to perform
BATCH_SIZE = 30
MIGRATION = 'MigrateRequirementsToWorkItems'
disable_ddl_transaction!
class Requirement < ActiveRecord::Base
include EachBatch
self.table_name = 'requirements'
end
def up
queue_background_migration_jobs_by_range_at_intervals(
Requirement.where(issue_id: nil),
MIGRATION,
2.minutes,
batch_size: BATCH_SIZE,
track_jobs: true
)
end
def down
# NO OP
end
end
6647e94d315c76629f9726e26bafd124fb2fed361568d65315e7c7557f8d9ecf
\ No newline at end of file
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
# Migrate requirements to work items(issues of requirement type)
# Eventually Requirement objects will be deprecated
# For more information check: https://gitlab.com/gitlab-org/gitlab/-/issues/323779#completion
module MigrateRequirementsToWorkItems
extend ::Gitlab::Utils::Override
SYNC_PARAMS = [:title, :description, :author_id, :project_id, :state].freeze
REQUIREMENT_ISSUE_TYPE = 3
INTERNAL_ID_ISSUES_USAGE = 0 # id for "issues" usage check Enums::InternalId.usage_resources
class Requirement < ActiveRecord::Base; end
class Issue < ActiveRecord::Base; end
# This is almost a hard copy of app/models/internal_id.rb
# with exception of some variables and functions that are not needed.
#
# Because we are generating iids manually we need to keep
# track of latest issue iid for each project like we do in the app.
# This avoids trying to create issues with already used iids and violating
# unique indexes after this migration runs.
class InternalId < ActiveRecord::Base
class << self
def generate_next(scope)
build_generator(scope).generate
end
def build_generator(scope)
ImplicitlyLockingInternalIdGenerator.new(scope)
end
end
class ImplicitlyLockingInternalIdGenerator
attr_reader :subject, :scope
RecordAlreadyExists = Class.new(StandardError)
def initialize(scope)
@scope = scope
end
def generate
next_iid = update_record!(scope, arel_table[:last_value] + 1)
return next_iid if next_iid
create_record!(scope, initial_value(scope) + 1)
end
def update_record!(scope, new_value)
stmt = Arel::UpdateManager.new
stmt.table(arel_table)
stmt.set(arel_table[:last_value] => new_value)
stmt.wheres = InternalId.where(**scope, usage: INTERNAL_ID_ISSUES_USAGE).arel.constraints
InternalId.connection.insert(stmt, 'Update InternalId', 'last_value')
end
def create_record!(scope, value)
attributes = {
project_id: scope[:project_id],
namespace_id: scope[:namespace_id],
usage: INTERNAL_ID_ISSUES_USAGE,
last_value: value
}
result = InternalId.insert(attributes)
raise RecordAlreadyExists if result.empty?
value
end
def initial_value(scope)
# Same logic from AtomicInternalId.project_init
Issue.where(project_id: scope[:project_id]).maximum(:iid) || 0
end
def arel_table
InternalId.arel_table
end
end
end
override :perform
def perform(start_id, end_id)
requirements = Requirement.where(id: start_id..end_id, issue_id: nil)
requirements.each do |requirement|
Requirement.transaction do
issue = create_issue_for(requirement)
# Updates requirement with issue_id to keep future changes in sync
requirement.update!(issue_id: issue.id)
end
end
mark_job_as_succeeded(start_id, end_id)
end
private
def create_issue_for(requirement)
params = requirement.slice(*SYNC_PARAMS)
next_iid = InternalId.generate_next({ project_id: requirement.project_id })
Issue.create! do |issue|
issue.iid = next_iid
issue.title = params[:title]
issue.description = params[:description]
issue.state_id = params[:state]
issue.author_id = params[:author_id]
issue.project_id = params[:project_id]
issue.issue_type = REQUIREMENT_ISSUE_TYPE
issue.created_at = requirement.created_at
issue.updated_at = Time.current
end
end
def mark_job_as_succeeded(*arguments)
::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
self.class.name.demodulize,
arguments
)
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::MigrateRequirementsToWorkItems, schema: 20211005194425 do
let(:issues) { table(:issues) }
let(:requirements) { table(:requirements) }
let(:namespaces) { table(:namespaces) }
let(:users) { table(:users) }
let(:projects) { table(:projects) }
let(:internal_ids) { table(:internal_ids) }
let!(:group) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
let!(:project) { projects.create!(namespace_id: group.id, name: 'gitlab', path: 'gitlab') }
let!(:project2) { projects.create!(namespace_id: group.id, name: 'gitlab2', path: 'gitlab2') }
let!(:user1) { users.create!(email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active') }
let!(:user2) { users.create!(email: 'author2@example.com', notification_email: 'author2@example.com', name: 'author2', username: 'author2', projects_limit: 10, state: 'active') }
let(:migration) { described_class::MIGRATION }
let!(:issue) { issues.create!(iid: 5, state_id: 1, project_id: project2.id) }
let!(:requirement_1) { requirements.create!(iid: 1, project_id: project.id, author_id: user1.id, title: 'r 1', state: 1, created_at: 2.days.ago, updated_at: 1.day.ago) }
# Already in sync
let!(:requirement_2) { requirements.create!(iid: 2, project_id: project2.id, author_id: user1.id, issue_id: issue.id, title: 'r 2', state: 1, created_at: Time.current, updated_at: Time.current) }
let!(:requirement_3) { requirements.create!(iid: 3, project_id: project.id, title: 'r 3', state: 1, created_at: 3.days.ago, updated_at: 2.days.ago) }
let!(:requirement_4) { requirements.create!(iid: 99, project_id: project2.id, author_id: user1.id, title: 'r 4', state: 2, created_at: 1.hour.ago, updated_at: Time.current) }
let!(:requirement_5) { requirements.create!(iid: 5, project_id: project2.id, author_id: user2.id, title: 'r 5', state: 1, created_at: 2.hours.ago, updated_at: Time.current) }
let(:now) { Time.now.utc.to_s }
around do |example|
freeze_time { example.run }
end
it 'creates work items for not synced requirements' do
expect do
described_class.new.perform(requirement_1.id, requirement_5.id)
end.to change { issues.count }.by(4)
end
it 'creates requirement work items with correct attributes' do
described_class.new.perform(requirement_1.id, requirement_5.id)
[requirement_1, requirement_3, requirement_4, requirement_5].each do |requirement|
issue = issues.find(requirement.reload.issue_id)
expect(issue.issue_type).to eq(3) # requirement work item type
expect(issue.title).to eq(requirement.title)
expect(issue.description).to eq(requirement.description)
expect(issue.project_id).to eq(requirement.project_id)
expect(issue.state_id).to eq(requirement.state)
expect(issue.author_id).to eq(requirement.author_id)
expect(issue.iid).to be_present
expect(issue.created_at).to eq(requirement.created_at)
expect(issue.updated_at.to_s).to eq(now) # issues updated_at column do not persist timezone
end
end
it 'populates iid correctly' do
described_class.new.perform(requirement_1.id, requirement_5.id)
# Projects without issues
expect(issues.find(requirement_1.reload.issue_id).iid).to eq(1)
expect(issues.find(requirement_3.reload.issue_id).iid).to eq(2)
# Project that already has one issue with iid = 5
expect(issues.find(requirement_4.reload.issue_id).iid).to eq(6)
expect(issues.find(requirement_5.reload.issue_id).iid).to eq(7)
end
it 'tracks iid greatest value' do
internal_ids.create!(project_id: issue.project_id, usage: 0, last_value: issue.iid)
described_class.new.perform(requirement_1.id, requirement_5.id)
expect(internal_ids.count).to eq(2) # Creates record for project when there is not one
expect(internal_ids.find_by_project_id(project.id).last_value).to eq(2)
expect(internal_ids.find_by_project_id(project2.id).last_value).to eq(7)
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20211005194425_schedule_requirements_migration.rb')
RSpec.describe ScheduleRequirementsMigration do
let(:issues) { table(:issues) }
let(:requirements) { table(:requirements) }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:users) { table(:users) }
let!(:group) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
let!(:project) { projects.create!(namespace_id: group.id, name: 'gitlab', path: 'gitlab') }
let(:migration) { described_class::MIGRATION }
let!(:issue) { issues.create!(state_id: 1) }
let!(:author) { users.create!(email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active') }
let!(:requirement_1) { requirements.create!(iid: 1, project_id: project.id, title: 'r 1', state: 1, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
# Already in sync
let!(:requirement_2) { requirements.create!(iid: 2, project_id: project.id, issue_id: issue.id, title: 'r 2', state: 1, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
let!(:requirement_3) { requirements.create!(iid: 3, project_id: project.id, title: 'r 3', state: 1, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
let!(:requirement_4) { requirements.create!(iid: 99, project_id: project.id, title: 'r 4', state: 2, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
let!(:requirement_5) { requirements.create!(iid: 5, project_id: project.id, title: 'r 5', state: 1, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 2)
end
context 'scheduling migrations' do
before do
Sidekiq::Worker.clear_all
end
it 'schedules jobs for all requirements without issues in sync' do
Sidekiq::Testing.fake! do
freeze_time do
migrate!
expect(migration).to be_scheduled_delayed_migration(120.seconds, requirement_1.id, requirement_3.id)
expect(migration).to be_scheduled_delayed_migration(240.seconds, requirement_4.id, requirement_5.id)
expect(BackgroundMigrationWorker.jobs.size).to eq(2)
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# No op on CE
class MigrateRequirementsToWorkItems
def perform(start_id, end_id)
end
end
end
end
Gitlab::BackgroundMigration::MigrateRequirementsToWorkItems.prepend_mod_with('Gitlab::BackgroundMigration::MigrateRequirementsToWorkItems')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment