Merge branch...

Merge branch '330882-dependency-scanning-vulnerability-reports-a-lot-of-duplicate-links-clear-table' into 'master'

Background migration removing vuln finding links

See merge request gitlab-org/gitlab!73780

db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb

+# frozen_string_literal: true
+
+class RemoveVulnerabilityFindingLinks < Gitlab::Database::Migration[1.0]
+  BATCH_SIZE = 50_000
+  MIGRATION = 'RemoveVulnerabilityFindingLinks'
+
+  disable_ddl_transaction!
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      define_batchable_model('vulnerability_finding_links'),
+      MIGRATION,
+      2.minutes,
+      batch_size: BATCH_SIZE
+    )
+  end
+
+  def down
+    # no ops
+  end
+end

db/schema_migrations/20211104165220

+52625ff0a6117724cc1d7c6417ef95fe8dbcbb394486bb4734e28d3b41d23fd2
\ No newline at end of file

lib/gitlab/background_migration/remove_vulnerability_finding_links.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Remove vulnerability finding link records
+    # The records will be repopulated from the `raw_metadata`
+    # column of `vulnerability_occurrences` once the unique
+    # index is in place.
+    class RemoveVulnerabilityFindingLinks
+      include Gitlab::Database::DynamicModelHelpers
+
+      def perform(start_id, stop_id)
+        define_batchable_model('vulnerability_finding_links').where(id: start_id..stop_id).delete_all
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211104165220 do
+  let(:vulnerability_findings) { table(:vulnerability_occurrences) }
+  let(:finding_links) { table(:vulnerability_finding_links) }
+
+  let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:project) { table(:projects).create!(namespace_id: namespace.id) }
+  let(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'scanner', name: 'scanner') }
+  let(:vulnerability_identifier) do
+    table(:vulnerability_identifiers).create!(
+      project_id: project.id,
+      external_type: 'vulnerability-identifier',
+      external_id: 'vulnerability-identifier',
+      fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a',
+      name: 'vulnerability identifier')
+  end
+
+  # vulnerability findings
+  let!(:findings) do
+    Array.new(2) do |id|
+      vulnerability_findings.create!(
+        project_id: project.id,
+        name: 'Vulnerability Name',
+        severity: 7,
+        confidence: 7,
+        report_type: 0,
+        project_fingerprint: '123qweasdzxc',
+        scanner_id: scanner.id,
+        primary_identifier_id: vulnerability_identifier.id,
+        location_fingerprint: "location_fingerprint_#{id}",
+        metadata_version: 'metadata_version',
+        raw_metadata: 'raw_metadata',
+        uuid: "uuid_#{id}"
+      )
+    end
+  end
+
+  # vulnerability finding links
+  let!(:links) do
+    {
+      findings.first => Array.new(5) { |id| finding_links.create!(vulnerability_occurrence_id: findings.first.id, name: "Link Name 1", url: "link_url1.example") },
+      findings.second => Array.new(5) { |id| finding_links.create!(vulnerability_occurrence_id: findings.second.id, name: "Link Name 2", url: "link_url2.example") }
+    }
+  end
+
+  it 'removes vulnerability links' do
+    expect do
+      subject.perform(links[findings.first].first.id, links[findings.second].last.id)
+    end.to change { finding_links.count }.from(10).to(0)
+
+    expect(finding_links.all).to be_empty
+  end
+
+  it 'only deletes vulnerability links for the current batch' do
+    expected_links = [finding_links.where(vulnerability_occurrence_id: findings.second.id)].flatten
+
+    expect do
+      subject.perform(links[findings.first].first.id, links[findings.first].last.id)
+    end.to change { finding_links.count }.from(10).to(5)
+
+    expect(finding_links.all).to match_array(expected_links)
+  end
+end