Commit 8306618c authored by Vladimir Shushlin's avatar Vladimir Shushlin

Add a rake task to migrate pages to deployments

The main purpose of this is to migrate pages to object storage
parent 830379e8
# frozen_string_literal: true
class ProjectPagesMetadatum < ApplicationRecord
include EachBatch
self.primary_key = :project_id
belongs_to :project, inverse_of: :pages_metadatum
......@@ -8,4 +10,5 @@ class ProjectPagesMetadatum < ApplicationRecord
belongs_to :pages_deployment
scope :deployed, -> { where(deployed: true) }
scope :only_on_legacy_storage, -> { deployed.where(pages_deployment: nil) }
end
......@@ -2,7 +2,8 @@
module Pages
class MigrateLegacyStorageToDeploymentService
ExclusiveLeaseTaken = Class.new(StandardError)
ExclusiveLeaseTakenError = Class.new(StandardError)
FailedToCreateArchiveError = Class.new(StandardError)
include ::Pages::LegacyStorageLease
......@@ -19,7 +20,7 @@ module Pages
true
end
raise ExclusiveLeaseTaken, "Can't migrate pages for project #{project.id}: exclusive lease taken" unless migrated
raise ExclusiveLeaseTakenError, "Can't migrate pages for project #{project.id}: exclusive lease taken" unless migrated
end
private
......@@ -38,13 +39,12 @@ module Pages
project.set_first_pages_deployment!(deployment)
rescue ::Pages::ZipDirectoryService::InvalidArchiveError => e
Gitlab::ErrorTracking.track_exception(e, project_id: project.id)
if !project.pages_metadatum&.reload&.pages_deployment &&
Feature.enabled?(:pages_migration_mark_as_not_deployed, project)
project.mark_pages_as_not_deployed
end
raise FailedToCreateArchiveError, e
ensure
FileUtils.rm_f(archive_path) if archive_path
end
......
......@@ -4,8 +4,9 @@ module Pages
class ZipDirectoryService
include Gitlab::Utils::StrongMemoize
InvalidArchiveError = Class.new(RuntimeError)
InvalidEntryError = Class.new(RuntimeError)
Error = Class.new(::StandardError)
InvalidArchiveError = Class.new(Error)
InvalidEntryError = Class.new(Error)
PUBLIC_DIR = 'public'
......@@ -14,7 +15,7 @@ module Pages
end
def execute
raise InvalidArchiveError unless valid_work_directory?
raise InvalidArchiveError, "Invalid work directory: #{@input_dir}" unless valid_work_directory?
output_file = File.join(real_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects
......@@ -39,7 +40,7 @@ module Pages
unless valid_path?(disk_file_path)
# archive without public directory is completelly unusable
raise InvalidArchiveError if zipfile_path == PUBLIC_DIR
raise InvalidArchiveError, "Invalid public directory: #{disk_file_path}" if zipfile_path == PUBLIC_DIR
# archive with invalid entry will just have this entry missing
raise InvalidEntryError
......
---
title: Add rake task for migrating legacy pages storage to zip deployments
merge_request: 50153
author:
type: added
require 'logger'
namespace :gitlab do
namespace :pages do
desc "GitLab | Pages | Migrate legacy storage to zip format"
task migrate_legacy_storage: :gitlab_environment do
logger = Logger.new(STDOUT)
logger.info('Starting to migrate legacy pages storage to zip deployments')
migrated_projects = 0
ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: 10) do |batch|
# count will actually be changed after migration, so increment before
batch.preload(project: [:namespace, :route, pages_metadatum: :pages_deployment]).each do |metadatum|
project = metadatum.project
time = Benchmark.realtime do
::Pages::MigrateLegacyStorageToDeploymentService.new(project).execute
end
migrated_projects += 1
logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time} seconds")
rescue => e
logger.error("#{e.message} project_id: #{project&.id}")
Gitlab::ErrorTracking.track_exception(e, project_id: project&.id)
end
logger.info("#{migrated_projects} pages projects are migrated")
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe ProjectPagesMetadatum do
describe '.only_on_legacy_storage' do
it 'returns only deployed records without deployment' do
create(:project) # without pages deployed
legacy_storage_project = create(:project)
legacy_storage_project.mark_pages_as_deployed
project_with_deployment = create(:project)
deployment = create(:pages_deployment, project: project_with_deployment)
project_with_deployment.mark_pages_as_deployed
project_with_deployment.update_pages_deployment!(deployment)
expect(described_class.only_on_legacy_storage).to eq([legacy_storage_project.pages_metadatum])
end
end
end
......@@ -9,9 +9,13 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
it 'marks pages as not deployed if public directory is absent' do
project.mark_pages_as_deployed
expect(project.pages_metadatum.reload.deployed).to eq(true)
expect do
service.execute
end.to change { project.pages_metadatum.reload.deployed }.from(true).to(false)
end.to raise_error(described_class::FailedToCreateArchiveError)
expect(project.pages_metadatum.reload.deployed).to eq(false)
end
it 'does not mark pages as not deployed if public directory is absent but pages_deployment exists' do
......@@ -19,9 +23,13 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
project.update_pages_deployment!(deployment)
project.mark_pages_as_deployed
expect(project.pages_metadatum.reload.deployed).to eq(true)
expect do
service.execute
end.not_to change { project.pages_metadatum.reload.deployed }.from(true)
end.to raise_error(described_class::FailedToCreateArchiveError)
expect(project.pages_metadatum.reload.deployed).to eq(true)
end
it 'does not mark pages as not deployed if public directory is absent but feature is disabled' do
......@@ -29,9 +37,13 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
project.mark_pages_as_deployed
expect(project.pages_metadatum.reload.deployed).to eq(true)
expect do
service.execute
end.not_to change { project.pages_metadatum.reload.deployed }.from(true)
end.to raise_error(described_class::FailedToCreateArchiveError)
expect(project.pages_metadatum.reload.deployed).to eq(true)
end
it 'removes pages archive when can not save deployment' do
......@@ -94,7 +106,7 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
described_class.new(project).try_obtain_lease do
expect do
described_class.new(project).execute
end.to raise_error(described_class::ExclusiveLeaseTaken)
end.to raise_error(described_class::ExclusiveLeaseTakenError)
end
end
end
......
# frozen_string_literal: true
require 'rake_helper'
RSpec.describe 'gitlab:pages:migrate_legacy_storagerake task' do
before(:context) do
Rake.application.rake_require 'tasks/gitlab/pages'
end
subject { run_rake_task('gitlab:pages:migrate_legacy_storage') }
let(:project) { create(:project) }
it 'does not try to migrate pages if pages are not deployed' do
expect(::Pages::MigrateLegacyStorageToDeploymentService).not_to receive(:new)
subject
end
context 'when pages are marked as deployed' do
before do
project.mark_pages_as_deployed
end
context 'when pages directory does not exist' do
it 'tries to migrate the project, but does not crash' do
expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project) do |service|
expect(service).to receive(:execute).and_call_original
end
subject
end
end
context 'when pages directory exists on disk' do
before do
FileUtils.mkdir_p File.join(project.pages_path, "public")
File.open(File.join(project.pages_path, "public/index.html"), "w") do |f|
f.write("Hello!")
end
end
it 'migrates pages projects without deployments' do
expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project) do |service|
expect(service).to receive(:execute).and_call_original
end
expect do
subject
end.to change { project.pages_metadatum.reload.pages_deployment }.from(nil)
end
context 'when deployed already exists for the project' do
before do
deployment = create(:pages_deployment, project: project)
project.set_first_pages_deployment!(deployment)
end
it 'does not try to migrate project' do
expect(::Pages::MigrateLegacyStorageToDeploymentService).not_to receive(:new)
subject
end
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment