Merge branch 'issue_323779-migrate_requirements_into_issues' into 'master'

Background migration to move requirement to work items

See merge request gitlab-org/gitlab!71778

db/post_migrate/20211005194425_schedule_requirements_migration.rb

+# frozen_string_literal: true
+
+class ScheduleRequirementsMigration < Gitlab::Database::Migration[1.0]
+  DOWNTIME = false
+
+  # 2021-10-05 requirements count: ~12500
+  #
+  # Using 30 as batch size and 120 seconds default interval will produce:
+  # ~420 jobs - taking ~14 hours to perform
+  BATCH_SIZE = 30
+
+  MIGRATION = 'MigrateRequirementsToWorkItems'
+
+  disable_ddl_transaction!
+
+  class Requirement < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'requirements'
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      Requirement.where(issue_id: nil),
+      MIGRATION,
+      2.minutes,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    # NO OP
+  end
+end

db/schema_migrations/20211005194425

+6647e94d315c76629f9726e26bafd124fb2fed361568d65315e7c7557f8d9ecf
\ No newline at end of file

ee/lib/ee/gitlab/background_migration/migrate_requirements_to_work_items.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      # Migrate requirements to work items(issues of requirement type)
+      # Eventually Requirement objects will be deprecated
+      # For more information check: https://gitlab.com/gitlab-org/gitlab/-/issues/323779#completion
+      module MigrateRequirementsToWorkItems
+        extend ::Gitlab::Utils::Override
+
+        SYNC_PARAMS = [:title, :description, :author_id, :project_id, :state].freeze
+        REQUIREMENT_ISSUE_TYPE = 3
+        INTERNAL_ID_ISSUES_USAGE = 0 # id for "issues" usage check Enums::InternalId.usage_resources
+
+        class Requirement < ActiveRecord::Base; end
+        class Issue < ActiveRecord::Base; end
+
+        # This is almost a hard copy of app/models/internal_id.rb
+        # with exception of some variables and functions that are not needed.
+        #
+        # Because we are generating iids manually we need to keep
+        # track of latest issue iid for each project like we do in the app.
+        # This avoids trying to create issues with already used iids and violating
+        # unique indexes after this migration runs.
+        class InternalId < ActiveRecord::Base
+          class << self
+            def generate_next(scope)
+              build_generator(scope).generate
+            end
+
+            def build_generator(scope)
+              ImplicitlyLockingInternalIdGenerator.new(scope)
+            end
+          end
+
+          class ImplicitlyLockingInternalIdGenerator
+            attr_reader :subject, :scope
+
+            RecordAlreadyExists = Class.new(StandardError)
+
+            def initialize(scope)
+              @scope = scope
+            end
+
+            def generate
+              next_iid = update_record!(scope, arel_table[:last_value] + 1)
+
+              return next_iid if next_iid
+
+              create_record!(scope, initial_value(scope) + 1)
+            end
+
+            def update_record!(scope, new_value)
+              stmt = Arel::UpdateManager.new
+              stmt.table(arel_table)
+              stmt.set(arel_table[:last_value] => new_value)
+              stmt.wheres = InternalId.where(**scope, usage: INTERNAL_ID_ISSUES_USAGE).arel.constraints
+
+              InternalId.connection.insert(stmt, 'Update InternalId', 'last_value')
+            end
+
+            def create_record!(scope, value)
+              attributes = {
+                project_id: scope[:project_id],
+                namespace_id: scope[:namespace_id],
+                usage: INTERNAL_ID_ISSUES_USAGE,
+                last_value: value
+              }
+
+              result = InternalId.insert(attributes)
+
+              raise RecordAlreadyExists if result.empty?
+
+              value
+            end
+
+            def initial_value(scope)
+              # Same logic from AtomicInternalId.project_init
+              Issue.where(project_id: scope[:project_id]).maximum(:iid) || 0
+            end
+
+            def arel_table
+              InternalId.arel_table
+            end
+          end
+        end
+
+        override :perform
+        def perform(start_id, end_id)
+          requirements = Requirement.where(id: start_id..end_id, issue_id: nil)
+
+          requirements.each do |requirement|
+            Requirement.transaction do
+              issue = create_issue_for(requirement)
+
+              # Updates requirement with issue_id to keep future changes in sync
+              requirement.update!(issue_id: issue.id)
+            end
+          end
+
+          mark_job_as_succeeded(start_id, end_id)
+        end
+
+        private
+
+        def create_issue_for(requirement)
+          params = requirement.slice(*SYNC_PARAMS)
+          next_iid = InternalId.generate_next({ project_id: requirement.project_id })
+
+          Issue.create! do |issue|
+            issue.iid = next_iid
+            issue.title = params[:title]
+            issue.description = params[:description]
+            issue.state_id = params[:state]
+            issue.author_id = params[:author_id]
+            issue.project_id = params[:project_id]
+            issue.issue_type = REQUIREMENT_ISSUE_TYPE
+            issue.created_at = requirement.created_at
+            issue.updated_at = Time.current
+          end
+        end
+
+        def mark_job_as_succeeded(*arguments)
+          ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+            self.class.name.demodulize,
+            arguments
+          )
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/background_migration/migrate_requirements_to_work_items_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::MigrateRequirementsToWorkItems, schema: 20211005194425 do
+  let(:issues) { table(:issues) }
+  let(:requirements) { table(:requirements) }
+  let(:namespaces) { table(:namespaces) }
+  let(:users) { table(:users) }
+  let(:projects) { table(:projects) }
+  let(:internal_ids) { table(:internal_ids) }
+  let!(:group) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+
+  let!(:project) { projects.create!(namespace_id: group.id, name: 'gitlab', path: 'gitlab') }
+  let!(:project2) { projects.create!(namespace_id: group.id, name: 'gitlab2', path: 'gitlab2') }
+
+  let!(:user1) { users.create!(email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active') }
+  let!(:user2) { users.create!(email: 'author2@example.com', notification_email: 'author2@example.com', name: 'author2', username: 'author2', projects_limit: 10, state: 'active') }
+
+  let(:migration) { described_class::MIGRATION }
+  let!(:issue) { issues.create!(iid: 5, state_id: 1, project_id: project2.id) }
+
+  let!(:requirement_1) { requirements.create!(iid: 1, project_id: project.id, author_id: user1.id, title: 'r 1', state: 1, created_at: 2.days.ago, updated_at: 1.day.ago) }
+
+  # Already in sync
+  let!(:requirement_2) { requirements.create!(iid: 2, project_id: project2.id, author_id: user1.id, issue_id: issue.id, title: 'r 2', state: 1, created_at: Time.current, updated_at: Time.current) }
+
+  let!(:requirement_3) { requirements.create!(iid: 3, project_id: project.id, title: 'r 3', state: 1, created_at: 3.days.ago, updated_at: 2.days.ago) }
+  let!(:requirement_4) { requirements.create!(iid: 99, project_id: project2.id, author_id: user1.id, title: 'r 4', state: 2, created_at: 1.hour.ago, updated_at: Time.current) }
+  let!(:requirement_5) { requirements.create!(iid: 5, project_id: project2.id, author_id: user2.id, title: 'r 5', state: 1, created_at: 2.hours.ago, updated_at: Time.current) }
+
+  let(:now) { Time.now.utc.to_s }
+
+  around do |example|
+    freeze_time { example.run }
+  end
+
+  it 'creates work items for not synced requirements' do
+    expect do
+      described_class.new.perform(requirement_1.id, requirement_5.id)
+    end.to change { issues.count }.by(4)
+  end
+
+  it 'creates requirement work items with correct attributes' do
+    described_class.new.perform(requirement_1.id, requirement_5.id)
+
+    [requirement_1, requirement_3, requirement_4, requirement_5].each do |requirement|
+      issue = issues.find(requirement.reload.issue_id)
+
+      expect(issue.issue_type).to eq(3) # requirement work item type
+      expect(issue.title).to eq(requirement.title)
+      expect(issue.description).to eq(requirement.description)
+      expect(issue.project_id).to eq(requirement.project_id)
+      expect(issue.state_id).to eq(requirement.state)
+      expect(issue.author_id).to eq(requirement.author_id)
+      expect(issue.iid).to be_present
+      expect(issue.created_at).to eq(requirement.created_at)
+      expect(issue.updated_at.to_s).to eq(now) # issues updated_at column do not persist timezone
+    end
+  end
+
+  it 'populates iid correctly' do
+    described_class.new.perform(requirement_1.id, requirement_5.id)
+
+    # Projects without issues
+    expect(issues.find(requirement_1.reload.issue_id).iid).to eq(1)
+    expect(issues.find(requirement_3.reload.issue_id).iid).to eq(2)
+    # Project that already has one issue with iid = 5
+    expect(issues.find(requirement_4.reload.issue_id).iid).to eq(6)
+    expect(issues.find(requirement_5.reload.issue_id).iid).to eq(7)
+  end
+
+  it 'tracks iid greatest value' do
+    internal_ids.create!(project_id: issue.project_id, usage: 0, last_value: issue.iid)
+
+    described_class.new.perform(requirement_1.id, requirement_5.id)
+
+    expect(internal_ids.count).to eq(2) # Creates record for project when there is not one
+    expect(internal_ids.find_by_project_id(project.id).last_value).to eq(2)
+    expect(internal_ids.find_by_project_id(project2.id).last_value).to eq(7)
+  end
+end

ee/spec/migrations/schedule_requirements_migration_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20211005194425_schedule_requirements_migration.rb')
+
+RSpec.describe ScheduleRequirementsMigration do
+  let(:issues) { table(:issues) }
+  let(:requirements) { table(:requirements) }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:users) { table(:users) }
+  let!(:group) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+  let!(:project) { projects.create!(namespace_id: group.id, name: 'gitlab', path: 'gitlab') }
+  let(:migration) { described_class::MIGRATION }
+  let!(:issue) { issues.create!(state_id: 1) }
+
+  let!(:author) { users.create!(email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active') }
+
+  let!(:requirement_1) { requirements.create!(iid: 1, project_id: project.id, title: 'r 1', state: 1, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
+
+  # Already in sync
+  let!(:requirement_2) { requirements.create!(iid: 2, project_id: project.id, issue_id: issue.id, title: 'r 2', state: 1, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
+
+  let!(:requirement_3) { requirements.create!(iid: 3, project_id: project.id, title: 'r 3', state: 1, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
+  let!(:requirement_4) { requirements.create!(iid: 99, project_id: project.id, title: 'r 4', state: 2, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
+  let!(:requirement_5) { requirements.create!(iid: 5, project_id: project.id, title: 'r 5', state: 1, created_at: Time.now, updated_at: Time.now, author_id: author.id) }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 2)
+  end
+
+  context 'scheduling migrations' do
+    before do
+      Sidekiq::Worker.clear_all
+    end
+
+    it 'schedules jobs for all requirements without issues in sync' do
+      Sidekiq::Testing.fake! do
+        freeze_time do
+          migrate!
+
+          expect(migration).to be_scheduled_delayed_migration(120.seconds, requirement_1.id, requirement_3.id)
+          expect(migration).to be_scheduled_delayed_migration(240.seconds, requirement_4.id, requirement_5.id)
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+        end
+      end
+    end
+  end
+end

lib/gitlab/background_migration/migrate_requirements_to_work_items.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # No op on CE
+    class MigrateRequirementsToWorkItems
+      def perform(start_id, end_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::MigrateRequirementsToWorkItems.prepend_mod_with('Gitlab::BackgroundMigration::MigrateRequirementsToWorkItems')