Commit be5ac14b authored by GitLab Bot's avatar GitLab Bot

Automatic merge of gitlab-org/gitlab master

parents 9fc9b1b4 da3d1ee5
For guidance on the overall deprecations, removals and breaking changes workflow, please visit https://about.gitlab.com/handbook/product/gitlab-the-product/#breaking-changes-deprecations-and-removing-features
<!-- Use this template as a starting point for deprecations. -->
### Deprecation Summary
......@@ -75,4 +77,4 @@ Use the following resources to find the appropriate labels:
<!-- Identifies that this Issue is related to deprecating a feature -->
/label ~"type::deprecation"
<!-- Add the ~"breaking change" label to this issue if necessary -->
\ No newline at end of file
<!-- Add the ~"breaking change" label to this issue if necessary -->
......@@ -6,10 +6,11 @@
**Be sure to link this MR to the relevant issue(s).**
- Removal Issue (or earlier deprecation issue):
- Deprecation issue:
- Removal issue:
- MR that removes the feature (optional):
If there is no relevant removal or deprecation issue, hit pause and:
If there is no relevant deprecation issue, hit pause and:
- Review the [process for deprecating and removing features](https://about.gitlab.com/handbook/product/gitlab-the-product/#process-for-deprecating-and-removing-a-feature).
- Connect with the Product Manager DRI.
......
......@@ -8,14 +8,14 @@ const PRODUCT_INFO = {
// eslint-disable-next-line @gitlab/require-i18n-strings
name: 'Premium',
id: '0002',
price: 228,
price: '228',
variant: 'SaaS',
},
[SKU_ULTIMATE]: {
// eslint-disable-next-line @gitlab/require-i18n-strings
name: 'Ultimate',
id: '0001',
price: 1188,
price: '1188',
variant: 'SaaS',
},
};
......@@ -220,8 +220,8 @@ export const trackTransaction = (transactionDetails) => {
id: transactionId,
affiliation: 'GitLab',
option: paymentOption,
revenue,
tax,
revenue: revenue.toString(),
tax: tax.toString(),
},
products: [product],
},
......
# frozen_string_literal: true
class PopulateVulnerabilityReads < Gitlab::Database::Migration[1.0]
BATCH_SIZE = 10_000
DELAY_INTERVAL = 2.minutes
MIGRATION_NAME = 'PopulateVulnerabilityReads'
SUB_BATCH_SIZE = 1_000
disable_ddl_transaction!
def up
queue_background_migration_jobs_by_range_at_intervals(
define_batchable_model('vulnerabilities'),
MIGRATION_NAME,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true,
other_job_arguments: [SUB_BATCH_SIZE]
)
end
def down
# no-op
end
end
c7c645787aadc95c77df6420f437f78aed4e7e862ea2b66e7824766b1d9f3cb5
\ No newline at end of file
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop:disable Style/Documentation
class PopulateVulnerabilityReads
include Gitlab::Database::DynamicModelHelpers
PAUSE_SECONDS = 0.1
def perform(start_id, end_id, sub_batch_size)
vulnerability_model.where(id: start_id..end_id).each_batch(of: sub_batch_size) do |sub_batch|
first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first
connection.execute(insert_query(first, last))
sleep PAUSE_SECONDS
end
mark_job_as_succeeded(start_id, end_id, sub_batch_size)
end
private
def vulnerability_model
define_batchable_model('vulnerabilities', connection: connection)
end
def connection
ActiveRecord::Base.connection
end
def insert_query(start_id, end_id)
<<~SQL
INSERT INTO vulnerability_reads (
vulnerability_id,
project_id,
scanner_id,
report_type,
severity,
state,
has_issues,
resolved_on_default_branch,
uuid,
location_image
)
SELECT
vulnerabilities.id,
vulnerabilities.project_id,
vulnerability_scanners.id,
vulnerabilities.report_type,
vulnerabilities.severity,
vulnerabilities.state,
CASE
WHEN
vulnerability_issue_links.vulnerability_id IS NOT NULL
THEN
true
ELSE
false
END
has_issues,
vulnerabilities.resolved_on_default_branch,
vulnerability_occurrences.uuid::uuid,
vulnerability_occurrences.location ->> 'image'
FROM
vulnerabilities
INNER JOIN vulnerability_occurrences ON vulnerability_occurrences.vulnerability_id = vulnerabilities.id
INNER JOIN vulnerability_scanners ON vulnerability_scanners.id = vulnerability_occurrences.scanner_id
LEFT JOIN vulnerability_issue_links ON vulnerability_issue_links.vulnerability_id = vulnerabilities.id
WHERE vulnerabilities.id BETWEEN #{start_id} AND #{end_id}
ON CONFLICT(vulnerability_id) DO NOTHING;
SQL
end
def mark_job_as_succeeded(*arguments)
Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
self.class.name.demodulize,
arguments
)
end
end
# rubocop:enable Style/Documentation
end
end
......@@ -234,7 +234,7 @@ describe('~/google_tag_manager/index', () => {
category: 'DevOps',
id: '0002',
name: 'Premium',
price: 228,
price: '228',
quantity: 1,
variant: 'SaaS',
},
......@@ -264,7 +264,7 @@ describe('~/google_tag_manager/index', () => {
category: 'DevOps',
id: '0001',
name: 'Ultimate',
price: 1188,
price: '1188',
quantity: 1,
variant: 'SaaS',
},
......@@ -301,7 +301,7 @@ describe('~/google_tag_manager/index', () => {
category: 'DevOps',
id: '0001',
name: 'Ultimate',
price: 1188,
price: '1188',
quantity: 5,
variant: 'SaaS',
},
......@@ -354,8 +354,8 @@ describe('~/google_tag_manager/index', () => {
id: '123',
affiliation: 'GitLab',
option: 'visa',
revenue,
tax: 10,
revenue: revenue.toString(),
tax: '10',
},
products: [
{
......@@ -363,7 +363,7 @@ describe('~/google_tag_manager/index', () => {
category: 'DevOps',
id,
name,
price: revenue,
price: revenue.toString(),
quantity: 1,
variant: 'SaaS',
},
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityReads do
let(:vulnerabilities) { table(:vulnerabilities) }
let(:vulnerability_reads) { table(:vulnerability_reads) }
let(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
let(:vulnerability_issue_links) { table(:vulnerability_issue_links) }
let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
let(:user) { table(:users).create!(email: 'author@example.com', username: 'author', projects_limit: 10) }
let(:project) { table(:projects).create!(namespace_id: namespace.id) }
let(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
let(:sub_batch_size) { 1000 }
before do
vulnerabilities_findings.connection.execute 'ALTER TABLE vulnerability_occurrences DISABLE TRIGGER "trigger_insert_or_update_vulnerability_reads_from_occurrences"'
vulnerabilities.connection.execute 'ALTER TABLE vulnerabilities DISABLE TRIGGER "trigger_update_vulnerability_reads_on_vulnerability_update"'
vulnerability_issue_links.connection.execute 'ALTER TABLE vulnerability_issue_links DISABLE TRIGGER "trigger_update_has_issues_on_vulnerability_issue_links_update"'
10.times.each do |x|
vulnerability = create_vulnerability!(
project_id: project.id,
report_type: 7,
author_id: user.id
)
identifier = table(:vulnerability_identifiers).create!(
project_id: project.id,
external_type: 'uuid-v5',
external_id: 'uuid-v5',
fingerprint: Digest::SHA1.hexdigest("#{vulnerability.id}"),
name: 'Identifier for UUIDv5')
create_finding!(
vulnerability_id: vulnerability.id,
project_id: project.id,
scanner_id: scanner.id,
primary_identifier_id: identifier.id
)
end
end
it 'creates vulnerability_reads for the given records' do
described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size)
expect(vulnerability_reads.count).to eq(10)
end
it 'does not create new records when records already exists' do
described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size)
described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size)
expect(vulnerability_reads.count).to eq(10)
end
private
def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
vulnerabilities.create!(
project_id: project_id,
author_id: author_id,
title: title,
severity: severity,
confidence: confidence,
report_type: report_type
)
end
# rubocop:disable Metrics/ParameterLists
def create_finding!(
vulnerability_id: nil, project_id:, scanner_id:, primary_identifier_id:,
name: "test", severity: 7, confidence: 7, report_type: 0,
project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, location_fingerprint: 'test',
metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid)
vulnerabilities_findings.create!(
vulnerability_id: vulnerability_id,
project_id: project_id,
name: name,
severity: severity,
confidence: confidence,
report_type: report_type,
project_fingerprint: project_fingerprint,
scanner_id: scanner_id,
primary_identifier_id: primary_identifier_id,
location: location,
location_fingerprint: location_fingerprint,
metadata_version: metadata_version,
raw_metadata: raw_metadata,
uuid: uuid
)
end
# rubocop:enable Metrics/ParameterLists
end
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe PopulateVulnerabilityReads, :migration do
let_it_be(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
let_it_be(:user) { table(:users).create!(email: 'author@example.com', username: 'author', projects_limit: 10) }
let_it_be(:project) { table(:projects).create!(namespace_id: namespace.id) }
let_it_be(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
let_it_be(:background_migration_jobs) { table(:background_migration_jobs) }
let_it_be(:vulnerabilities) { table(:vulnerabilities) }
let_it_be(:vulnerability_reads) { table(:vulnerability_reads) }
let_it_be(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
let_it_be(:vulnerability_issue_links) { table(:vulnerability_issue_links) }
let_it_be(:vulnerability_ids) { [] }
before do
stub_const("#{described_class}::BATCH_SIZE", 1)
stub_const("#{described_class}::SUB_BATCH_SIZE", 1)
5.times.each do |x|
vulnerability = create_vulnerability!(
project_id: project.id,
report_type: 7,
author_id: user.id
)
identifier = table(:vulnerability_identifiers).create!(
project_id: project.id,
external_type: 'uuid-v5',
external_id: 'uuid-v5',
fingerprint: Digest::SHA1.hexdigest("#{vulnerability.id}"),
name: 'Identifier for UUIDv5')
create_finding!(
vulnerability_id: vulnerability.id,
project_id: project.id,
scanner_id: scanner.id,
primary_identifier_id: identifier.id
)
vulnerability_ids << vulnerability.id
end
end
around do |example|
freeze_time { Sidekiq::Testing.fake! { example.run } }
end
it 'schedules background migrations' do
migrate!
expect(background_migration_jobs.count).to eq(5)
expect(background_migration_jobs.first.arguments).to match_array([vulnerability_ids.first, vulnerability_ids.first, 1])
expect(background_migration_jobs.second.arguments).to match_array([vulnerability_ids.second, vulnerability_ids.second, 1])
expect(background_migration_jobs.third.arguments).to match_array([vulnerability_ids.third, vulnerability_ids.third, 1])
expect(background_migration_jobs.fourth.arguments).to match_array([vulnerability_ids.fourth, vulnerability_ids.fourth, 1])
expect(background_migration_jobs.fifth.arguments).to match_array([vulnerability_ids.fifth, vulnerability_ids.fifth, 1])
expect(BackgroundMigrationWorker.jobs.size).to eq(5)
expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(2.minutes, vulnerability_ids.first, vulnerability_ids.first, 1)
expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(4.minutes, vulnerability_ids.second, vulnerability_ids.second, 1)
expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(6.minutes, vulnerability_ids.third, vulnerability_ids.third, 1)
expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(8.minutes, vulnerability_ids.fourth, vulnerability_ids.fourth, 1)
expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(10.minutes, vulnerability_ids.fifth, vulnerability_ids.fifth, 1)
end
private
def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
vulnerabilities.create!(
project_id: project_id,
author_id: author_id,
title: title,
severity: severity,
confidence: confidence,
report_type: report_type
)
end
# rubocop:disable Metrics/ParameterLists
def create_finding!(
id: nil,
vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:,
name: "test", severity: 7, confidence: 7, report_type: 0,
project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid)
params = {
vulnerability_id: vulnerability_id,
project_id: project_id,
name: name,
severity: severity,
confidence: confidence,
report_type: report_type,
project_fingerprint: project_fingerprint,
scanner_id: scanner_id,
primary_identifier_id: primary_identifier_id,
location_fingerprint: location_fingerprint,
metadata_version: metadata_version,
raw_metadata: raw_metadata,
uuid: uuid
}
params[:id] = id unless id.nil?
vulnerabilities_findings.create!(params)
end
# rubocop:enable Metrics/ParameterLists
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment