Implement background migration for `resolved_on_default_branch` column

This migration will download previous security scan artifacts to
populate `resolved_on_default_branch` for existing vulnerabilities.

db/post_migrate/20200806100713_schedule_populate_resolved_on_default_branch_column.rb

@@ -5,22 +5,16 @@ class SchedulePopulateResolvedOnDefaultBranchColumn < ActiveRecord::Migration[6.
 
   DOWNTIME = false
   BATCH_SIZE = 100
-  DELAY_INTERVAL = 2.minutes.to_i
+  DELAY_INTERVAL = 5.minutes.to_i
   MIGRATION_CLASS = 'PopulateResolvedOnDefaultBranchColumn'
 
   disable_ddl_transaction!
 
-  class Project < ActiveRecord::Base
-    include EachBatch
-
-    scope :has_vulnerabilities, -> { joins('INNER JOIN vulnerabilities v ON v.project_id = projects.id').group(:id) }
-  end
-
   def up
     return unless run_migration?
 
-    Project.has_vulnerabilities.each_batch(of: BATCH_SIZE) do |batch, index|
-      project_ids = batch.pluck(:id)
+    EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn::Vulnerability.distinct.each_batch(of: BATCH_SIZE, column: :project_id) do |batch, index|
+      project_ids = batch.pluck(:project_id)
       migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, project_ids)
     end
   end

ee/changelogs/unreleased/227114_populate_resolved_on_default_branch_column_for_existing_vulnera.yml

+---
+title: Populate `resolved_on_default_branch` column for existing vulnerabilities
+merge_request: 38795
+author:
+type: added

ee/spec/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn do
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:findings) { table(:vulnerability_occurrences) }
+  let(:builds) { table(:ci_builds) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
+
+  let(:namespace) { namespaces.create!(name: "foo", path: "bar") }
+
+  describe '#perform' do
+    let!(:project_1) { projects.create!(namespace_id: namespace.id) }
+    let!(:project_2) { projects.create!(namespace_id: namespace.id) }
+    let(:utility_class) { described_class::PopulateResolvedOnDefaultBranchColumnForProject }
+
+    subject(:populate_resolved_on_default_branch_column) { described_class.new.perform([project_1.id, project_2.id]) }
+
+    before do
+      allow(utility_class).to receive(:perform)
+    end
+
+    it 'calls `PopulateResolvedOnDefaultBranchColumnForProject.perform` for each project by given ids' do
+      populate_resolved_on_default_branch_column
+
+      expect(utility_class).to have_received(:perform).twice
+      expect(utility_class).to have_received(:perform).with(project_1.id)
+      expect(utility_class).to have_received(:perform).with(project_2.id)
+    end
+  end
+
+  describe EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn::PopulateResolvedOnDefaultBranchColumnForProject do
+    describe '.perform' do
+      let(:project_id) { 1 }
+      let(:mock_utility_object) { instance_double(described_class, perform: true) }
+
+      subject(:populate_for_project) { described_class.perform(project_id) }
+
+      before do
+        allow(described_class).to receive(:new).and_return(mock_utility_object)
+      end
+
+      it 'instantiates the utility service object and calls #perform on it' do
+        populate_for_project
+
+        expect(described_class).to have_received(:new).with(project_id)
+        expect(mock_utility_object).to have_received(:perform)
+      end
+    end
+
+    describe '#perform' do
+      let(:user) { users.create!(name: 'John Doe', email: 'test@example.com', projects_limit: 5) }
+      let(:project) { projects.create!(namespace_id: namespace.id) }
+      let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a', status: 'success') }
+      let(:utility_object) { described_class.new(project.id) }
+      let(:scanner) { scanners.create!(project_id: project.id, external_id: 'bandit', name: 'Bandit') }
+      let(:artifact_model) { EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn::JobArtifact }
+      let(:artifact_fixture_path) { Rails.root.join('ee/spec/fixtures/security_reports/master/gl-sast-report.json') }
+      let(:sha_attribute) { Gitlab::Database::ShaAttribute.new }
+      let(:vulnerability_identifier) do
+        vulnerability_identifiers.create!(
+          project_id: project.id,
+          name: 'identifier',
+          fingerprint: sha_attribute.serialize('e6dd15eda2137be0034977a85b300a94a4f243a3'),
+          external_type: 'bar',
+          external_id: 'zoo')
+      end
+
+      let(:disappeared_vulnerability) do
+        vulnerabilities.create!(
+          project_id: project.id,
+          author_id: user.id,
+          title: 'Vulnerability',
+          severity: 5,
+          confidence: 5,
+          report_type: 5
+        )
+      end
+
+      let(:existing_vulnerability) do
+        vulnerabilities.create!(
+          project_id: project.id,
+          author_id: user.id,
+          title: 'Vulnerability',
+          severity: 5,
+          confidence: 5,
+          report_type: 5
+        )
+      end
+
+      subject(:populate_for_project) { utility_object.perform }
+
+      before do
+        build = builds.create!(commit_id: pipeline.id, retried: false, type: 'Ci::Build')
+        artifact = artifact_model.new(project_id: project.id, job_id: build.id, file_type: 5, file_format: 1)
+        artifact.file = fixture_file_upload(artifact_fixture_path, 'application/json')
+        artifact.save!
+
+        findings.create!(
+          project_id: project.id,
+          vulnerability_id: existing_vulnerability.id,
+          severity: 5,
+          confidence: 5,
+          report_type: 5,
+          scanner_id: scanner.id,
+          primary_identifier_id: vulnerability_identifier.id,
+          project_fingerprint: 'foo',
+          location_fingerprint: sha_attribute.serialize('d869ba3f0b3347eb2749135a437dc07c8ae0f420'),
+          uuid: SecureRandom.uuid,
+          name: 'Solar blast vulnerability',
+          metadata_version: '1',
+          raw_metadata: '')
+
+        allow(::Gitlab::CurrentSettings).to receive(:default_branch_name).and_return(:master)
+      end
+
+      it 'sets `resolved_on_default_branch` attribute of disappeared vulnerabilities' do
+        expect { populate_for_project }.to change { disappeared_vulnerability.reload[:resolved_on_default_branch] }.from(false).to(true)
+                                       .and not_change { existing_vulnerability.reload[:resolved_on_default_branch] }
+      end
+    end
+  end
+end

ee/spec/migrations/schedule_populate_resolved_on_default_branch_column_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 require Rails.root.join('db', 'post_migrate', '20200806100713_schedule_populate_resolved_on_default_branch_column.rb')
@@ -21,13 +23,28 @@ RSpec.describe SchedulePopulateResolvedOnDefaultBranchColumn do
 
   context 'when the Gitlab instance is EE' do
     let(:ee?) { true }
-    let!(:project_1) { create(:project) }
-    let!(:project_2) { create(:project) }
-    let!(:project_3) { create(:project) }
+    let(:namespaces) { table(:namespaces) }
+    let(:projects) { table(:projects) }
+    let(:vulnerabilities) { table(:vulnerabilities) }
+    let(:users) { table(:users) }
+    let(:namespace) { namespaces.create!(name: "foo", path: "bar") }
+    let!(:project_1) { projects.create!(namespace_id: namespace.id) }
+    let!(:project_2) { projects.create!(namespace_id: namespace.id) }
+    let!(:project_3) { projects.create!(namespace_id: namespace.id) }
+    let(:user) { users.create!(name: 'John Doe', email: 'test@example.com', projects_limit: 1) }
+    let(:vulnerability_data) do
+      {
+        author_id: user.id,
+        title: 'Vulnerability',
+        severity: 5,
+        confidence: 5,
+        report_type: 5
+      }
+    end
 
     before do
-      create(:vulnerability, project: project_1)
-      create(:vulnerability, project: project_2)
+      vulnerabilities.create!(**vulnerability_data, project_id: project_1.id)
+      vulnerabilities.create!(**vulnerability_data, project_id: project_2.id)
 
       stub_const("#{described_class.name}::BATCH_SIZE", 1)
     end
@@ -36,8 +53,8 @@ RSpec.describe SchedulePopulateResolvedOnDefaultBranchColumn do
       migrate!
 
       expect(BackgroundMigrationWorker.jobs.size).to be(2)
-      expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(2.minutes, project_1.id)
-      expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(4.minutes, project_2.id)
+      expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(5.minutes, project_1.id)
+      expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(10.minutes, project_2.id)
     end
   end
 end

lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb

@@ -9,4 +9,4 @@ module Gitlab
   end
 end
 
-Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn')
+Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn')