Commit efce07e8 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch '320786-add-a-rake-task-for-removing-migrated-object-storage-data-2' into 'master'

Add a rake task for removing migrated object storage data

See merge request gitlab-org/gitlab!53727
parents 7fbf987c cc327600
......@@ -2,6 +2,7 @@
# PagesDeployment stores a zip archive containing GitLab Pages web-site
class PagesDeployment < ApplicationRecord
include EachBatch
include FileStoreMounter
MIGRATED_FILE_NAME = "_migrated.zip"
......
......@@ -6,7 +6,6 @@ namespace :gitlab do
namespace :pages do
desc "GitLab | Pages | Migrate legacy storage to zip format"
task migrate_legacy_storage: :gitlab_environment do
logger = Logger.new(STDOUT)
logger.info('Starting to migrate legacy pages storage to zip deployments')
result = ::Pages::MigrateFromLegacyStorageService.new(logger, migration_threads, batch_size).execute
......@@ -16,6 +15,26 @@ namespace :gitlab do
logger.info("- The #{result[:errored]} projects failed to be migrated")
end
desc "GitLab | Pages | DANGER: Removes data which was migrated from legacy storage on zip storage. Can be used if some bugs in migration are discovered and migration needs to be restarted from scratch."
task clean_migrated_zip_storage: :gitlab_environment do
destroyed_deployments = 0
logger.info("Starting to delete migrated pages deployments")
::PagesDeployment.migrated_from_legacy_storage.each_batch(of: batch_size) do |batch|
destroyed_deployments += batch.count
# we need to destroy associated files, so can't use delete_all
batch.destroy_all # rubocop: disable Cop/DestroyAll
logger.info("#{destroyed_deployments} deployments were deleted")
end
end
def logger
@logger ||= Logger.new(STDOUT)
end
def migration_threads
ENV.fetch('PAGES_MIGRATION_THREADS', '3').to_i
end
......
......@@ -4,12 +4,20 @@ FactoryBot.define do
factory :pages_deployment, class: 'PagesDeployment' do
project
after(:build) do |deployment, _evaluator|
filepath = Rails.root.join("spec/fixtures/pages.zip")
transient do
filename { nil }
end
trait(:migrated) do
filename { PagesDeployment::MIGRATED_FILE_NAME }
end
after(:build) do |deployment, evaluator|
file = UploadedFile.new("spec/fixtures/pages.zip", filename: evaluator.filename)
deployment.file = fixture_file_upload(filepath)
deployment.file_sha256 = Digest::SHA256.file(filepath).hexdigest
::Zip::File.open(filepath) do |zip_archive|
deployment.file = file
deployment.file_sha256 = Digest::SHA256.file(file.path).hexdigest
::Zip::File.open(file.path) do |zip_archive|
deployment.file_count = zip_archive.count
end
end
......
......@@ -2,11 +2,12 @@
require 'rake_helper'
RSpec.describe 'gitlab:pages:migrate_legacy_storagerake task' do
RSpec.describe 'gitlab:pages' do
before(:context) do
Rake.application.rake_require 'tasks/gitlab/pages'
end
describe 'migrate_legacy_storage task' do
subject { run_rake_task('gitlab:pages:migrate_legacy_storage') }
it 'calls migration service' do
......@@ -36,4 +37,23 @@ RSpec.describe 'gitlab:pages:migrate_legacy_storagerake task' do
subject
end
end
describe 'clean_migrated_zip_storage task' do
it 'removes only migrated deployments' do
regular_deployment = create(:pages_deployment)
migrated_deployment = create(:pages_deployment, :migrated)
regular_deployment.project.update_pages_deployment!(regular_deployment)
migrated_deployment.project.update_pages_deployment!(migrated_deployment)
expect(PagesDeployment.all).to contain_exactly(regular_deployment, migrated_deployment)
run_rake_task('gitlab:pages:clean_migrated_zip_storage')
expect(PagesDeployment.all).to contain_exactly(regular_deployment)
expect(PagesDeployment.find_by_id(regular_deployment.id)).not_to be_nil
expect(PagesDeployment.find_by_id(migrated_deployment.id)).to be_nil
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment