Commit 824ca5c1 authored by Heinrich Lee Yu's avatar Heinrich Lee Yu

Merge branch...

Merge branch '246655-hashed-storage-trigger-migration-to-hashed-storage-via-post-deployment-migration' into 'master'

Hashed Storage: Force migration to Hashed Storage  via background jobs

Closes #246655

See merge request gitlab-org/gitlab!42313
parents 4a4aa2aa fd52f5de
---
title: 'Hashed Storage: forced automatic migration of legacy projects via background jobs'
merge_request: 42313
author:
type: changed
# frozen_string_literal: true
class ScheduleMigrationToHashedStorage < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
MIGRATION = 'MigrateToHashedStorage'
disable_ddl_transaction!
def up
migrate_async(MIGRATION)
end
def down
# NO-OP
end
end
0c659c7cbbda4d2fbbbd344f0fca50860ccf54ec8b666772d53a4c6fa602d097
\ No newline at end of file
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Background migration to move any legacy project to Hashed Storage
class MigrateToHashedStorage
def perform
batch_size = helper.batch_size
legacy_projects_count = Project.with_unmigrated_storage.count
if storage_migrator.rollback_pending?
logger.warn(
migrator: 'MigrateToHashedStorage',
message: 'Aborting an storage rollback operation currently in progress'
)
storage_migrator.abort_rollback!
end
if legacy_projects_count == 0
logger.info(
migrator: 'MigrateToHashedStorage',
message: 'There are no projects requiring migration to Hashed Storage'
)
return
end
logger.info(
migrator: 'MigrateToHashedStorage',
message: "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}"
)
helper.project_id_batches_migration do |start, finish|
storage_migrator.bulk_schedule_migration(start: start, finish: finish)
logger.info(
migrator: 'MigrateToHashedStorage',
message: "Enqueuing migration of projects in batches of #{batch_size} from ID=#{start} to ID=#{finish}",
batch_from: start,
batch_to: finish
)
end
end
private
def helper
Gitlab::HashedStorage::RakeHelper
end
def storage_migrator
@storage_migrator ||= Gitlab::HashedStorage::Migrator.new
end
def logger
@logger ||= ::Gitlab::BackgroundMigration::Logger.build
end
end
end
end
......@@ -93,6 +93,14 @@ module Gitlab
any_non_empty_queue?(::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker)
end
# Remove all remaining scheduled rollback operations
#
def abort_rollback!
[::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker].each do |worker|
Sidekiq::Queue.new(worker.queue).clear
end
end
private
def any_non_empty_queue?(*workers)
......
# frozen_string_literal: true
require 'spec_helper'
# rubocop:disable RSpec/FactoriesInMigrationSpecs
RSpec.describe Gitlab::BackgroundMigration::MigrateToHashedStorage, :sidekiq, :redis do
let(:migrator) { Gitlab::HashedStorage::Migrator.new }
subject(:background_migration) { described_class.new }
describe '#perform' do
let!(:project) { create(:project, :empty_repo, :legacy_storage) }
context 'with pending rollback' do
it 'aborts rollback operation' do
Sidekiq::Testing.disable! do
Sidekiq::Client.push(
'queue' => ::HashedStorage::ProjectRollbackWorker.queue,
'class' => ::HashedStorage::ProjectRollbackWorker,
'args' => [project.id]
)
expect { background_migration.perform }.to change { migrator.rollback_pending? }.from(true).to(false)
end
end
end
it 'enqueues legacy projects to be migrated' do
Sidekiq::Testing.fake! do
expect { background_migration.perform }.to change { Sidekiq::Queues[::HashedStorage::MigratorWorker.queue].size }.by(1)
end
end
context 'when executing all jobs' do
it 'migrates legacy projects' do
Sidekiq::Testing.inline! do
expect { background_migration.perform }.to change { project.reload.legacy_storage? }.from(true).to(false)
end
end
end
end
end
# rubocop:enable RSpec/FactoriesInMigrationSpecs
......@@ -232,4 +232,16 @@ RSpec.describe Gitlab::HashedStorage::Migrator, :redis do
expect(subject.rollback_pending?).to be_falsey
end
end
describe 'abort_rollback!' do
let_it_be(:project) { create(:project, :empty_repo) }
it 'removes any rollback related scheduled job' do
Sidekiq::Testing.disable! do
::HashedStorage::RollbackerWorker.perform_async(1, 5)
expect { subject.abort_rollback! }.to change { subject.rollback_pending? }.from(true).to(false)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200915044225_schedule_migration_to_hashed_storage.rb')
RSpec.describe ScheduleMigrationToHashedStorage, :sidekiq do
describe '#up' do
it 'schedules background migration job' do
Sidekiq::Testing.fake! do
expect { migrate! }.to change { BackgroundMigrationWorker.jobs.size }.by(1)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment