Merge branch '244380_populate_has_vulnerabilities_values' into 'master'

Populate `has_vulnerabilities` values

See merge request gitlab-org/gitlab!46890

changelogs/unreleased/244380_populate_has_vulnerabilities_values.yml

+---
+title: Populate values for `has_vulnerabilities` column of `project_settings` table
+merge_request: 46890
+author:
+type: fixed

db/post_migrate/20201103192526_schedule_populate_has_vulnerabilities.rb

+# frozen_string_literal: true
+
+class SchedulePopulateHasVulnerabilities < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 1_000
+  DELAY_INTERVAL = 2.minutes
+  MIGRATION_CLASS = 'PopulateHasVulnerabilities'
+
+  disable_ddl_transaction!
+
+  def up
+    Gitlab::BackgroundMigration::PopulateHasVulnerabilities::Vulnerability.distinct.each_batch(of: BATCH_SIZE, column: :project_id) do |batch, index|
+      project_ids = batch.pluck(:project_id)
+
+      migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, project_ids)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20201103192526

+bb137c3a41a40e740f8ae65b43d7f9218f52d6d5eaf53c8a64b3336a8f16141b
\ No newline at end of file

lib/gitlab/background_migration/populate_has_vulnerabilities.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class populates missing dismissal information for
+    # vulnerability entries.
+    class PopulateHasVulnerabilities
+      class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation
+        self.table_name = 'project_settings'
+
+        UPSERT_SQL = <<~SQL
+          INSERT INTO project_settings
+          (project_id, has_vulnerabilities, created_at, updated_at)
+          VALUES
+          %{values}
+          ON CONFLICT (project_id)
+          DO UPDATE SET
+            has_vulnerabilities = true,
+            updated_at = EXCLUDED.updated_at
+        SQL
+
+        def self.upsert_for(project_ids)
+          timestamp = connection.quote(Time.now)
+          values = project_ids.map { |project_id| "(#{project_id}, true, #{timestamp}, #{timestamp})" }.join(', ')
+
+          connection.execute(UPSERT_SQL % { values: values })
+        end
+      end
+
+      class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation
+        include EachBatch
+
+        self.table_name = 'vulnerabilities'
+      end
+
+      def perform(*project_ids)
+        ProjectSetting.upsert_for(project_ids)
+      rescue StandardError => e
+        log_error(e, project_ids)
+      ensure
+        log_info(project_ids)
+      end
+
+      private
+
+      def log_error(error, project_ids)
+        ::Gitlab::BackgroundMigration::Logger.error(
+          migrator: self.class.name,
+          message: error.message,
+          project_ids: project_ids
+        )
+      end
+
+      def log_info(project_ids)
+        ::Gitlab::BackgroundMigration::Logger.info(
+          migrator: self.class.name,
+          message: 'Projects has been processed to populate `has_vulnerabilities` information',
+          count: project_ids.length
+        )
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/populate_has_vulnerabilities_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::PopulateHasVulnerabilities, schema: 20201103192526 do
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:project_settings) { table(:project_settings) }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+
+  let(:user) { users.create!(name: 'test', email: 'test@example.com', projects_limit: 5) }
+  let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+  let(:vulnerability_base_params) { { title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, author_id: user.id } }
+
+  let!(:project_1) { projects.create!(namespace_id: namespace.id, name: 'foo_1') }
+  let!(:project_2) { projects.create!(namespace_id: namespace.id, name: 'foo_2') }
+  let!(:project_3) { projects.create!(namespace_id: namespace.id, name: 'foo_3') }
+
+  before do
+    project_settings.create!(project_id: project_1.id)
+    vulnerabilities.create!(vulnerability_base_params.merge(project_id: project_1.id))
+    vulnerabilities.create!(vulnerability_base_params.merge(project_id: project_3.id))
+
+    allow(::Gitlab::BackgroundMigration::Logger).to receive_messages(info: true, error: true)
+  end
+
+  describe '#perform' do
+    it 'sets `has_vulnerabilities` attribute of project_settings' do
+      expect { subject.perform(project_1.id, project_3.id) }.to change { project_settings.count }.from(1).to(2)
+                                                            .and change { project_settings.where(has_vulnerabilities: true).count }.from(0).to(2)
+    end
+
+    it 'writes info log message' do
+      subject.perform(project_1.id, project_3.id)
+
+      expect(::Gitlab::BackgroundMigration::Logger).to have_received(:info).with(migrator: described_class.name,
+                                                                                 message: 'Projects has been processed to populate `has_vulnerabilities` information',
+                                                                                 count: 2)
+    end
+
+    context 'when an error happens' do
+      before do
+        allow(described_class::ProjectSetting).to receive(:upsert_for).and_raise('foo')
+      end
+
+      it 'writes error log message' do
+        subject.perform(project_1.id, project_3.id)
+
+        expect(::Gitlab::BackgroundMigration::Logger).to have_received(:error).with(migrator: described_class.name,
+                                                                                    message: 'foo',
+                                                                                    project_ids: [project_1.id, project_3.id])
+      end
+    end
+  end
+end

spec/migrations/schedule_populate_has_vulnerabilities_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe SchedulePopulateHasVulnerabilities do
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:user) { users.create!(name: 'test', email: 'test@example.com', projects_limit: 5) }
+  let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+  let(:vulnerability_base_params) { { title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, author_id: user.id } }
+  let!(:project_1) { projects.create!(namespace_id: namespace.id, name: 'foo_1') }
+  let!(:project_2) { projects.create!(namespace_id: namespace.id, name: 'foo_2') }
+  let!(:project_3) { projects.create!(namespace_id: namespace.id, name: 'foo_3') }
+
+  around do |example|
+    freeze_time { Sidekiq::Testing.fake! { example.run } }
+  end
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+
+    vulnerabilities.create!(vulnerability_base_params.merge(project_id: project_1.id))
+    vulnerabilities.create!(vulnerability_base_params.merge(project_id: project_3.id))
+  end
+
+  it 'schedules the background jobs', :aggregate_failures do
+    migrate!
+
+    expect(BackgroundMigrationWorker.jobs.size).to be(2)
+    expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(2.minutes, project_1.id)
+    expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(4.minutes, project_3.id)
+  end
+end