Merge branch 'replace-undefined-with-unkown-vulnerabilities' into 'master'

Replace undefined severity with unknown severity for vulnerabilities

See merge request gitlab-org/gitlab!26305

changelogs/unreleased/replace-undefined-with-unkown-vulnerabilities.yml

+---
+title: Replace undefined severity with unknown severity for vulnerabilities
+merge_request: 26305
+author:
+type: other

db/post_migrate/20200302142052_update_vulnerability_severity_column.rb

+# frozen_string_literal: true
+
+class UpdateVulnerabilitySeverityColumn < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+  BATCH_SIZE = 1_000
+  INTERVAL = 2.minutes
+
+  def up
+    # create temporary index for undefined vulnerabilities
+    add_concurrent_index(:vulnerabilities, :id, where: 'severity = 0', name: 'undefined_vulnerability')
+
+    return unless Gitlab.ee?
+
+    migration = Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel
+    migration_name = migration.to_s.demodulize
+    relation = migration::Vulnerability.undefined_severity
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          migration_name,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # no-op
+    # This migration can not be reversed because we can not know which records had undefined severity
+  end
+end

db/schema.rb

@@ -4454,6 +4454,7 @@ ActiveRecord::Schema.define(version: 2020_03_09_105539) do
     t.index ["dismissed_by_id"], name: "index_vulnerabilities_on_dismissed_by_id"
     t.index ["due_date_sourcing_milestone_id"], name: "index_vulnerabilities_on_due_date_sourcing_milestone_id"
     t.index ["epic_id"], name: "index_vulnerabilities_on_epic_id"
+    t.index ["id"], name: "undefined_vulnerability", where: "(severity = 0)"
     t.index ["last_edited_by_id"], name: "index_vulnerabilities_on_last_edited_by_id"
     t.index ["milestone_id"], name: "index_vulnerabilities_on_milestone_id"
     t.index ["project_id"], name: "index_vulnerabilities_on_project_id"

ee/lib/ee/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      module RemoveUndefinedVulnerabilitySeverityLevel
+        extend ::Gitlab::Utils::Override
+        class Vulnerability < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'vulnerabilities'
+
+          SEVERITY_LEVELS = {
+            undefined: 0,
+            unknown: 2
+          }.with_indifferent_access.freeze
+
+          enum severity: SEVERITY_LEVELS
+
+          def self.undefined_severity
+            where(severity: Vulnerability.severities[:undefined])
+          end
+        end
+
+        override :perform
+        def perform(start_id, stop_id)
+          Vulnerability.undefined_severity
+            .where(id: start_id..stop_id)
+            .update_all(severity: Vulnerability.severities[:unknown])
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/remove_undefined_vulnerability_severity_level_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel, :migration, schema: 20200302142052 do
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:identifiers) { table(:vulnerability_identifiers) }
+  let(:projects) { table(:projects) }
+  let(:users) { table(:users) }
+
+  it 'updates undefined severity level to unknown' do
+    projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
+    users.create!(id: 13, email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active')
+
+    vul1 = vulnerabilities.create!(vuln_params)
+    vulnerabilities.create!(vuln_params)
+    vul3 = vulnerabilities.create!(vuln_params.merge(severity: 2))
+
+    expect(vulnerabilities.where(severity: 2).count). to eq(1)
+    expect(vulnerabilities.where(severity: 0).count). to eq(2)
+
+    described_class.new.perform(vul1.id, vul3.id)
+
+    expect(vulnerabilities.where(severity: 2).count).to eq(3)
+  end
+
+  def vuln_params
+    {
+      title: 'title',
+      state: 1,
+      severity: 0,
+      confidence: 5,
+      report_type: 2,
+      project_id: 123,
+      author_id: 13
+    }
+  end
+end

ee/spec/migrations/update_vulnerability_severity_column_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'post_migrate', '20200302142052_update_vulnerability_severity_column.rb')
+
+describe UpdateVulnerabilitySeverityColumn, :migration do
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:identifiers) { table(:vulnerability_identifiers) }
+  let(:projects) { table(:projects) }
+  let(:users) { table(:users) }
+
+  before do
+    projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
+    users.create!(id: 13, email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active')
+    stub_const("#{described_class}::BATCH_SIZE", 2)
+  end
+
+  it 'updates confidence levels for container scanning reports', :sidekiq_might_not_need_inline do
+    allow_any_instance_of(Gitlab).to receive(:ee?).and_return(true)
+
+    vulnerabilities.create!(vuln_params)
+    vulnerabilities.create!(vuln_params.merge(severity: 2))
+
+    expect(vulnerabilities.where(severity: 0).count). to eq(1)
+
+    migrate!
+
+    expect(vulnerabilities.exists?(severity: 0)).to be_falsy
+    expect(vulnerabilities.where(severity: 2).count).to eq(2)
+  end
+
+  it 'skips migration for ce' do
+    allow_any_instance_of(Gitlab).to receive(:ee?).and_return(false)
+
+    vulnerabilities.create!(vuln_params)
+
+    expect(vulnerabilities.where(severity: 0).count). to eq(1)
+
+    migrate!
+
+    expect(vulnerabilities.exists?(severity: 0)).to be_truthy
+  end
+
+  def vuln_params
+    {
+      title: 'title',
+      state: 1,
+      severity: 0,
+      confidence: 5,
+      report_type: 2,
+      project_id: 123,
+      author_id: 13
+    }
+  end
+end

lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class RemoveUndefinedVulnerabilitySeverityLevel
+      def perform(start_id, stop_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel')