Commit 4dcd8405 authored by pbair's avatar pbair

Remove batched background job migration helpers

Remove the migration helpers that queue the background migrations, as
they can be added at a later date when the feature is ready to be used
on an actual migration.
parent eff4c099
...@@ -12,11 +12,6 @@ module Gitlab ...@@ -12,11 +12,6 @@ module Gitlab
foreign_key: :batched_background_migration_id foreign_key: :batched_background_migration_id
scope :queue_order, -> { order(created_at: :asc) } scope :queue_order, -> { order(created_at: :asc) }
scope :for_batch_configuration, -> (job_class_name, table_name, column_name) do
where(job_class_name: remove_toplevel_prefix(job_class_name),
table_name: table_name,
column_name: column_name)
end
enum status: { enum status: {
paused: 0, paused: 0,
......
...@@ -5,9 +5,7 @@ module Gitlab ...@@ -5,9 +5,7 @@ module Gitlab
module Migrations module Migrations
module BackgroundMigrationHelpers module BackgroundMigrationHelpers
BACKGROUND_MIGRATION_BATCH_SIZE = 1_000 # Number of rows to process per job BACKGROUND_MIGRATION_BATCH_SIZE = 1_000 # Number of rows to process per job
BACKGROUND_MIGRATION_SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch
BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1_000 # Number of jobs to bulk queue at a time BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1_000 # Number of jobs to bulk queue at a time
BACKGROUND_MIGRATION_BATCH_CLASS_NAME = 'Gitlab::Database::BackgroundMigration::PrimaryKeyBatchingStrategy'
# Bulk queues background migration jobs for an entire table, batched by ID range. # Bulk queues background migration jobs for an entire table, batched by ID range.
# "Bulk" meaning many jobs will be pushed at a time for efficiency. # "Bulk" meaning many jobs will be pushed at a time for efficiency.
...@@ -57,47 +55,6 @@ module Gitlab ...@@ -57,47 +55,6 @@ module Gitlab
bulk_migrate_async(jobs) unless jobs.empty? bulk_migrate_async(jobs) unless jobs.empty?
end end
def queue_batched_background_migration( # rubocop:disable Metrics/ParameterLists
job_class_name,
batch_table_name,
batch_column_name,
job_interval:,
batch_min_value: 1,
batch_max_value: nil,
batch_class_name: BACKGROUND_MIGRATION_BATCH_CLASS_NAME,
batch_size: BACKGROUND_MIGRATION_BATCH_SIZE,
sub_batch_size: BACKGROUND_MIGRATION_SUB_BATCH_SIZE,
other_job_arguments: []
)
batch_max_value ||= connection.select_value(<<~SQL)
SELECT MAX(#{connection.quote_column_name(batch_column_name)})
FROM #{connection.quote_table_name(batch_table_name)}
SQL
return if batch_max_value.nil?
Gitlab::Database::BackgroundMigration::BatchedMigration.create!(
job_class_name: job_class_name,
table_name: batch_table_name,
column_name: batch_column_name,
interval: job_interval,
min_value: batch_min_value,
max_value: batch_max_value,
batch_class_name: batch_class_name,
batch_size: batch_size,
sub_batch_size: sub_batch_size,
job_arguments: other_job_arguments,
status: :active)
end
def abort_batched_background_migrations(job_class_name, table_name, column_name)
Gitlab::Database::BackgroundMigration::BatchedMigration
.for_batch_configuration(job_class_name, table_name, column_name)
.not_finished
.update_all(status: :aborted, updated_at: Time.current)
end
# Queues background migration jobs for an entire table in batches. # Queues background migration jobs for an entire table in batches.
# The default batching column used is the standard primary key `id`. # The default batching column used is the standard primary key `id`.
# Each job is scheduled with a `delay_interval` in between. # Each job is scheduled with a `delay_interval` in between.
......
...@@ -29,29 +29,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m ...@@ -29,29 +29,6 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m
end end
end end
describe '.for_batch_configuration' do
let(:job_class_name) { 'TestClass' }
let(:table_name) { :issues }
let(:column_name) { :project_id }
let!(:migration1) { create(:batched_background_migration, job_class_name: job_class_name, table_name: table_name) }
let!(:migration2) { create(:batched_background_migration, job_class_name: job_class_name, column_name: column_name) }
let!(:migration3) { create(:batched_background_migration, job_class_name: job_class_name, table_name: table_name, column_name: column_name) }
let!(:migration4) { create(:batched_background_migration, table_name: table_name, column_name: column_name) }
it 'returns records matching the migration configuration' do
relation = described_class.for_batch_configuration(job_class_name, table_name, column_name)
expect(relation.all).to eq([migration3])
end
it 'normalizes the job class name' do
relation = described_class.for_batch_configuration("::#{job_class_name}", table_name, column_name)
expect(relation.all).to eq([migration3])
end
end
describe '#interval_elapsed?' do describe '#interval_elapsed?' do
context 'when the migration has no last_job' do context 'when the migration has no last_job' do
let(:batched_migration) { build(:batched_background_migration) } let(:batched_migration) { build(:batched_background_migration) }
......
...@@ -79,89 +79,6 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do ...@@ -79,89 +79,6 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do
end end
end end
describe '#queue_batched_background_migration' do
it 'creates the database record for the migration' do
expect do
model.queue_batched_background_migration(
'MyJobClass',
:projects,
:id,
job_interval: 5.minutes,
batch_min_value: 5,
batch_max_value: 1000,
batch_class_name: 'MyBatchClass',
batch_size: 100,
sub_batch_size: 10,
other_job_arguments: %w[my arguments])
end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.from(0).to(1)
expect(Gitlab::Database::BackgroundMigration::BatchedMigration.first).to have_attributes(
job_class_name: 'MyJobClass',
table_name: 'projects',
column_name: 'id',
interval: 300,
min_value: 5,
max_value: 1000,
batch_class_name: 'MyBatchClass',
batch_size: 100,
sub_batch_size: 10,
job_arguments: %w[my arguments],
status: 'active')
end
context 'when the max_value is not given' do
context 'when records exist in the database' do
let!(:event1) { create(:event) }
let!(:event2) { create(:event) }
let!(:event3) { create(:event) }
it 'creates the record with current max value' do
expect do
model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes)
end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.from(0).to(1)
expect(Gitlab::Database::BackgroundMigration::BatchedMigration.first).to have_attributes(
job_class_name: 'MyJobClass',
table_name: 'events',
column_name: 'id',
interval: 300,
min_value: 1,
max_value: event3.id,
batch_class_name: described_class::BACKGROUND_MIGRATION_BATCH_CLASS_NAME,
batch_size: described_class::BACKGROUND_MIGRATION_BATCH_SIZE,
sub_batch_size: described_class::BACKGROUND_MIGRATION_SUB_BATCH_SIZE,
job_arguments: [],
status: 'active')
end
end
context 'when the database is empty' do
it 'does not create the record' do
expect do
model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes)
end.not_to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }
end
end
end
end
describe '#abort_batched_background_migrations' do
let!(:migration1) { create(:batched_background_migration, :finished) }
let!(:migration2) { create(:batched_background_migration) }
let!(:migration3) { create(:batched_background_migration, job_class_name: 'SomeOtherJobClassName') }
let!(:migration4) { create(:batched_background_migration, table_name: 'some_other_table_name') }
it 'aborts unfinished migrations that match the job class and table configuration' do
job_class_name = 'Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob'
expect do
model.abort_batched_background_migrations(job_class_name, 'events', 'id')
end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.aborted.count }.from(0).to(1)
expect(migration2.reload).to be_aborted
end
end
describe '#queue_background_migration_jobs_by_range_at_intervals' do describe '#queue_background_migration_jobs_by_range_at_intervals' do
context 'when the model has an ID column' do context 'when the model has an ID column' do
let!(:id1) { create(:user).id } let!(:id1) { create(:user).id }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment