Commit 9f222d4c authored by Adam Hegyi's avatar Adam Hegyi

Merge branch 'pedropombeiro/354027/1-create-unique-registration-token-indices' into 'master'

Nullify duplicate project runners tokens

See merge request gitlab-org/gitlab!81991
parents 881dadc0 66c9c3b9
# frozen_string_literal: true
class ScheduleResetDuplicateCiRunnersTokenEncryptedValuesOnProjects < Gitlab::Database::Migration[1.0]
MIGRATION = 'ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects'
TOKEN_COLUMN_NAME = :runners_token_encrypted
TEMP_INDEX_NAME = "tmp_index_projects_on_id_and_#{TOKEN_COLUMN_NAME}"
BATCH_SIZE = 10_000
DELAY_INTERVAL = 2.minutes
disable_ddl_transaction!
def up
add_concurrent_index :projects, [:id, TOKEN_COLUMN_NAME], where: "#{TOKEN_COLUMN_NAME} IS NOT NULL", unique: false, name: TEMP_INDEX_NAME
queue_background_migration_jobs_by_range_at_intervals(
Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects::Project.base_query,
MIGRATION,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true
)
end
def down
remove_concurrent_index_by_name(:projects, name: TEMP_INDEX_NAME)
end
end
# frozen_string_literal: true
class ScheduleResetDuplicateCiRunnersTokenValuesOnProjects < Gitlab::Database::Migration[1.0]
MIGRATION = 'ResetDuplicateCiRunnersTokenValuesOnProjects'
TOKEN_COLUMN_NAME = :runners_token
TEMP_INDEX_NAME = "tmp_index_projects_on_id_and_#{TOKEN_COLUMN_NAME}"
BATCH_SIZE = 10_000
DELAY_INTERVAL = 2.minutes
disable_ddl_transaction!
def up
add_concurrent_index :projects, [:id, TOKEN_COLUMN_NAME], where: "#{TOKEN_COLUMN_NAME} IS NOT NULL", unique: false, name: TEMP_INDEX_NAME
queue_background_migration_jobs_by_range_at_intervals(
Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenValuesOnProjects::Project.base_query,
MIGRATION,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true
)
end
def down
remove_concurrent_index_by_name(:projects, name: TEMP_INDEX_NAME)
end
end
e18ed9e6b2a98c77190ff2ce33f4d2b1984710b438e851d6a526ec8bb1f33c80
\ No newline at end of file
0aacf46a4a5b430a718336108f52c1c0bed4283846f36c2ab1de80100dcae0b4
\ No newline at end of file
...@@ -29551,6 +29551,10 @@ CREATE UNIQUE INDEX tmp_index_on_tmp_project_id_on_namespaces ON namespaces USIN ...@@ -29551,6 +29551,10 @@ CREATE UNIQUE INDEX tmp_index_on_tmp_project_id_on_namespaces ON namespaces USIN
CREATE INDEX tmp_index_on_vulnerabilities_non_dismissed ON vulnerabilities USING btree (id) WHERE (state <> 2); CREATE INDEX tmp_index_on_vulnerabilities_non_dismissed ON vulnerabilities USING btree (id) WHERE (state <> 2);
CREATE INDEX tmp_index_projects_on_id_and_runners_token ON projects USING btree (id, runners_token) WHERE (runners_token IS NOT NULL);
CREATE INDEX tmp_index_projects_on_id_and_runners_token_encrypted ON projects USING btree (id, runners_token_encrypted) WHERE (runners_token_encrypted IS NOT NULL);
CREATE UNIQUE INDEX uniq_pkgs_deb_grp_architectures_on_distribution_id_and_name ON packages_debian_group_architectures USING btree (distribution_id, name); CREATE UNIQUE INDEX uniq_pkgs_deb_grp_architectures_on_distribution_id_and_name ON packages_debian_group_architectures USING btree (distribution_id, name);
CREATE UNIQUE INDEX uniq_pkgs_deb_grp_components_on_distribution_id_and_name ON packages_debian_group_components USING btree (distribution_id, name); CREATE UNIQUE INDEX uniq_pkgs_deb_grp_components_on_distribution_id_and_name ON packages_debian_group_components USING btree (distribution_id, name);
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# A job to nullify duplicate runners_token_encrypted values in projects table in batches
class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects
class Project < ActiveRecord::Base # rubocop:disable Style/Documentation
include ::EachBatch
self.table_name = 'projects'
scope :base_query, -> do
where.not(runners_token_encrypted: nil)
end
end
def perform(start_id, end_id)
# Reset duplicate runner tokens that would prevent creating an unique index.
duplicate_tokens = Project.base_query
.where(id: start_id..end_id)
.group(:runners_token_encrypted)
.having('COUNT(*) > 1')
.pluck(:runners_token_encrypted)
Project.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any?
mark_job_as_succeeded(start_id, end_id)
end
private
def mark_job_as_succeeded(*arguments)
Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects', arguments)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# A job to nullify duplicate ci_runners_token values in projects table in batches
class ResetDuplicateCiRunnersTokenValuesOnProjects
class Project < ActiveRecord::Base # rubocop:disable Style/Documentation
include ::EachBatch
self.table_name = 'projects'
scope :base_query, -> do
where.not(runners_token: nil)
end
end
def perform(start_id, end_id)
# Reset duplicate runner tokens that would prevent creating an unique index.
duplicate_tokens = Project.base_query
.where(id: start_id..end_id)
.group(:runners_token)
.having('COUNT(*) > 1')
.pluck(:runners_token)
Project.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any?
mark_job_as_succeeded(start_id, end_id)
end
private
def mark_job_as_succeeded(*arguments)
Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnerValuesTokensOnProjects', arguments)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:perform) { described_class.new.perform(1, 4) }
before do
namespaces.create!(id: 123, name: 'sample', path: 'sample')
projects.create!(id: 1, namespace_id: 123, runners_token_encrypted: 'duplicate')
projects.create!(id: 2, namespace_id: 123, runners_token_encrypted: 'a-runners-token')
projects.create!(id: 3, namespace_id: 123, runners_token_encrypted: 'duplicate')
projects.create!(id: 4, namespace_id: 123, runners_token_encrypted: nil)
projects.create!(id: 5, namespace_id: 123, runners_token_encrypted: 'duplicate-2')
projects.create!(id: 6, namespace_id: 123, runners_token_encrypted: 'duplicate-2')
end
describe '#up' do
before do
stub_const("#{described_class}::SUB_BATCH_SIZE", 2)
end
it 'nullifies duplicate tokens', :aggregate_failures do
perform
expect(projects.count).to eq(6)
expect(projects.all.pluck(:id, :runners_token_encrypted).to_h).to eq(
{ 1 => nil, 2 => 'a-runners-token', 3 => nil, 4 => nil, 5 => 'duplicate-2', 6 => 'duplicate-2' }
)
expect(projects.pluck(:runners_token_encrypted).uniq).to match_array [nil, 'a-runners-token', 'duplicate-2']
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenValuesOnProjects do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:perform) { described_class.new.perform(1, 4) }
before do
namespaces.create!(id: 123, name: 'sample', path: 'sample')
projects.create!(id: 1, namespace_id: 123, runners_token: 'duplicate')
projects.create!(id: 2, namespace_id: 123, runners_token: 'a-runners-token')
projects.create!(id: 3, namespace_id: 123, runners_token: 'duplicate')
projects.create!(id: 4, namespace_id: 123, runners_token: nil)
projects.create!(id: 5, namespace_id: 123, runners_token: 'duplicate-2')
projects.create!(id: 6, namespace_id: 123, runners_token: 'duplicate-2')
end
describe '#up' do
before do
stub_const("#{described_class}::SUB_BATCH_SIZE", 2)
end
it 'nullifies duplicate tokens', :aggregate_failures do
perform
expect(projects.count).to eq(6)
expect(projects.all.pluck(:id, :runners_token).to_h).to eq(
{ 1 => nil, 2 => 'a-runners-token', 3 => nil, 4 => nil, 5 => 'duplicate-2', 6 => 'duplicate-2' }
)
expect(projects.pluck(:runners_token).uniq).to match_array [nil, 'a-runners-token', 'duplicate-2']
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment