Commit b41b4d2e authored by Grzegorz Bizon's avatar Grzegorz Bizon

Use new `each_batch` helper instead of custom one

In stage_id backgrond migration.
parent b7b3aef4
...@@ -35,7 +35,7 @@ module EachBatch ...@@ -35,7 +35,7 @@ module EachBatch
start_id = start[primary_key] start_id = start[primary_key]
arel_table = self.arel_table arel_table = self.arel_table
loop do 1.step do |index|
stop = except(:select) stop = except(:select)
.select(primary_key) .select(primary_key)
.where(arel_table[primary_key].gteq(start_id)) .where(arel_table[primary_key].gteq(start_id))
...@@ -54,7 +54,7 @@ module EachBatch ...@@ -54,7 +54,7 @@ module EachBatch
# Any ORDER BYs are useless for this relation and can lead to less # Any ORDER BYs are useless for this relation and can lead to less
# efficient UPDATE queries, hence we get rid of it. # efficient UPDATE queries, hence we get rid of it.
yield relation.except(:order) yield relation.except(:order), index
break unless stop break unless stop
end end
......
...@@ -7,18 +7,20 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration ...@@ -7,18 +7,20 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration
disable_ddl_transaction! disable_ddl_transaction!
class Build < ActiveRecord::Base
self.table_name = 'ci_builds'
include ::EachBatch
end
## ##
# It will take around 3 days to process 20M ci_builds. # It will take around 3 days to process 20M ci_builds.
# #
def up def up
opts = { scope: ->(table, query) { query.where(table[:stage_id].eq(nil)) }, Build.all.each_batch(of: BATCH_SIZE) do |relation, index|
of: BATCH_SIZE } range = relation.pluck('MIN(id)', 'MAX(id)').first
walk_table_in_batches(:ci_builds, **opts) do |index, start_id, stop_id|
schedule = index * 2.minutes schedule = index * 2.minutes
BackgroundMigrationWorker BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range)
.perform_in(schedule, MIGRATION, [start_id, stop_id])
end end
end end
......
...@@ -47,9 +47,9 @@ describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do ...@@ -47,9 +47,9 @@ describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do
Timecop.freeze do Timecop.freeze do
migrate! migrate!
expect(described_class::MIGRATION).to be_scheduled_migration(2.minutes, 1, 3) expect(described_class::MIGRATION).to be_scheduled_migration(2.minutes, 1, 2)
expect(described_class::MIGRATION).to be_scheduled_migration(4.minutes, 3, 5) expect(described_class::MIGRATION).to be_scheduled_migration(4.minutes, 3, 4)
expect(described_class::MIGRATION).to be_scheduled_migration(6.minutes, 5, 0) expect(described_class::MIGRATION).to be_scheduled_migration(6.minutes, 5, 6)
expect(BackgroundMigrationWorker.jobs.size).to eq 3 expect(BackgroundMigrationWorker.jobs.size).to eq 3
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment