Commit 6ddc96f5 authored by rossfuhrman's avatar rossfuhrman Committed by Krasimir Angelov

Delete invalid vulnerability remediations

parent fddd1c97
# frozen_string_literal: true
class DropInvalidRemediations < Gitlab::Database::Migration[1.0]
BATCH_SIZE = 3_000
DELAY_INTERVAL = 3.minutes
MIGRATION_NAME = 'DropInvalidRemediations'
DAY_PRIOR_TO_BUG_INTRODUCTION = DateTime.new(2021, 8, 1, 0, 0, 0)
disable_ddl_transaction!
def up
return unless Gitlab.ee?
relation = Gitlab::BackgroundMigration::DropInvalidRemediations::FindingRemediation.where("created_at > ?", DAY_PRIOR_TO_BUG_INTRODUCTION)
queue_background_migration_jobs_by_range_at_intervals(relation,
MIGRATION_NAME,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true)
end
def down
# no-op
end
end
04a4b10085bae2006ac78600b3cc410d130f9ac6944103c7bd85f71e060d4a67
\ No newline at end of file
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module DropInvalidRemediations
class Finding < ActiveRecord::Base
self.table_name = 'vulnerability_occurrences'
end
class FindingRemediation < ActiveRecord::Base
include ::EachBatch
belongs_to :finding, class_name: '::Gitlab::BackgroundMigration::DropInvalidRemediations::Finding'
self.table_name = 'vulnerability_findings_remediations'
end
def perform(start_id, end_id)
finding_remediation_batch = FindingRemediation.where(id: start_id..end_id)
findings_without_remediation_ids = []
Finding.where(id: finding_remediation_batch.select(:vulnerability_occurrence_id)).each do |finding|
findings_without_remediation_ids << finding.id if ::Gitlab::Json.parse(finding[:raw_metadata])["remediations"] == [nil] rescue false
end
remediations_to_destroy = finding_remediation_batch.select {|finding_remediation| findings_without_remediation_ids.include? finding_remediation.vulnerability_occurrence_id}
FindingRemediation.where(id: remediations_to_destroy).each_batch(of: 100) do |batch|
batch.delete_all
end
mark_job_as_succeeded(start_id, end_id)
end
private
def mark_job_as_succeeded(*arguments)
::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
self.class.name.demodulize,
arguments
)
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::DropInvalidRemediations, schema: 20211118194239 do
let(:remediations) { table(:vulnerability_findings_remediations) }
let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
let(:users) { table(:users) }
let(:user) { create_user! }
let(:project) { table(:projects).create!(id: 14219619, namespace_id: namespace.id) }
let(:scanners) { table(:vulnerability_scanners) }
let!(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
let(:vulnerabilities) { table(:vulnerabilities) }
let(:vulnerability_findings) { table(:vulnerability_occurrences) }
let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
let(:vulnerability_identifier) do
vulnerability_identifiers.create!(
id: 1244459,
project_id: project.id,
external_type: 'vulnerability-identifier',
external_id: 'vulnerability-identifier',
fingerprint: '0a203e8cd5260a1948edbedc76c7cb91ad6a2e45',
name: 'vulnerability identifier')
end
let!(:vulnerability_1) do
create_vulnerability!(
project_id: project.id,
author_id: user.id
)
end
let!(:vulnerability_2) do
create_vulnerability!(
project_id: project.id,
author_id: user.id
)
end
let!(:corrupt_finding_1) do
create_finding!(
id: 5606961,
uuid: "bd95c085-71aa-51d7-9bb6-08ae669c262e",
vulnerability_id: vulnerability_1.id,
report_type: 0,
location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed74',
primary_identifier_id: vulnerability_identifier.id,
scanner_id: scanner.id,
project_id: project.id,
raw_metadata: metadata(true)
)
end
let!(:finding_2) do
create_finding!(
id: 8765432,
uuid: "5b714f58-1176-5b26-8fd5-e11dfcb031b5",
vulnerability_id: vulnerability_2.id,
report_type: 0,
location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75',
primary_identifier_id: vulnerability_identifier.id,
scanner_id: scanner.id,
project_id: project.id,
raw_metadata: metadata(false)
)
end
let!(:remediation1) { remediations.create!(vulnerability_occurrence_id: corrupt_finding_1.id) }
let!(:remediation2) { remediations.create!(vulnerability_occurrence_id: finding_2.id) }
context 'corresponding vuln has a remediation provided' do
it 'only deletes the finding_remediations without a remediation' do
expect { described_class.new.perform(remediation1.id, remediation2.id )}.to change {remediations.count}.from(2).to(1)
end
end
private
def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
vulnerabilities.create!(
project_id: project_id,
author_id: author_id,
title: title,
severity: severity,
confidence: confidence,
report_type: report_type
)
end
# rubocop:disable Metrics/ParameterLists
def create_finding!(
id: nil,
vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:,
name: "test", severity: 7, confidence: 7, report_type: 0,
project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
metadata_version: 'test', raw_metadata: 'test', uuid: 'test')
vulnerability_findings.create!(
vulnerability_id: vulnerability_id,
project_id: project_id,
name: name,
severity: severity,
confidence: confidence,
report_type: report_type,
project_fingerprint: project_fingerprint,
scanner_id: scanner.id,
primary_identifier_id: vulnerability_identifier.id,
location_fingerprint: location_fingerprint,
metadata_version: metadata_version,
raw_metadata: raw_metadata,
uuid: uuid
)
end
# rubocop:enable Metrics/ParameterLists
def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now)
users.create!(
name: name,
email: email,
username: name,
projects_limit: 0,
user_type: user_type,
confirmed_at: confirmed_at
)
end
def metadata(corrupt)
remediations = if corrupt
[nil]
else
[
{
summary: 'summary',
diff: Base64.encode64("This ain't a diff")
}
]
end
{ remediations: remediations }.to_json
end
end
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe DropInvalidRemediations, :migration do
let(:migration_name) { 'DropInvalidRemediations' }
let(:remediations) { table(:vulnerability_findings_remediations) }
let!(:old_remediation1) { remediations.create!( created_at: '1/1/2021') }
let!(:remediation1) { remediations.create! }
let!(:remediation2) { remediations.create! }
let!(:remediation3) { remediations.create! }
let!(:remediation4) { remediations.create! }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 2)
allow_next_instance_of(Gitlab::BackgroundMigration::CopyCiBuildsColumnsToSecurityScans) do |instance|
allow(instance).to receive(:mark_job_as_succeeded)
end
end
it 'correctly schedules background migrations' do
Sidekiq::Testing.fake! do
freeze_time do
migrate!
expect(BackgroundMigrationWorker.jobs.size).to eq(2)
expect(described_class::MIGRATION_NAME).to be_scheduled_migration_with_multiple_args(remediation1.id, remediation2.id)
expect(described_class::MIGRATION_NAME).to be_scheduled_migration_with_multiple_args(remediation3.id, remediation4.id)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class DropInvalidRemediations
def perform(start_id, stop_id)
end
end
# rubocop: enable Style/Documentation
end
end
Gitlab::BackgroundMigration::DropInvalidRemediations.prepend_mod_with('Gitlab::BackgroundMigration::DropInvalidRemediations')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment