Add a rake task to migrate pages to deployments

The main purpose of this is to migrate pages to object storage

app/models/project_pages_metadatum.rb

 # frozen_string_literal: true
 
 class ProjectPagesMetadatum < ApplicationRecord
+  include EachBatch
+
   self.primary_key = :project_id
 
   belongs_to :project, inverse_of: :pages_metadatum
@@ -8,4 +10,5 @@ class ProjectPagesMetadatum < ApplicationRecord
   belongs_to :pages_deployment
 
   scope :deployed, -> { where(deployed: true) }
+  scope :only_on_legacy_storage, -> { deployed.where(pages_deployment: nil) }
 end

app/services/pages/migrate_legacy_storage_to_deployment_service.rb

@@ -2,7 +2,8 @@
 
 module Pages
   class MigrateLegacyStorageToDeploymentService
-    ExclusiveLeaseTaken = Class.new(StandardError)
+    ExclusiveLeaseTakenError = Class.new(StandardError)
+    FailedToCreateArchiveError = Class.new(StandardError)
 
     include ::Pages::LegacyStorageLease
 
@@ -19,7 +20,7 @@ module Pages
         true
       end
 
-      raise ExclusiveLeaseTaken, "Can't migrate pages for project #{project.id}: exclusive lease taken" unless migrated
+      raise ExclusiveLeaseTakenError, "Can't migrate pages for project #{project.id}: exclusive lease taken" unless migrated
     end
 
     private
@@ -38,13 +39,12 @@ module Pages
 
       project.set_first_pages_deployment!(deployment)
     rescue ::Pages::ZipDirectoryService::InvalidArchiveError => e
-      Gitlab::ErrorTracking.track_exception(e, project_id: project.id)
-
       if !project.pages_metadatum&.reload&.pages_deployment &&
          Feature.enabled?(:pages_migration_mark_as_not_deployed, project)
         project.mark_pages_as_not_deployed
       end
 
+      raise FailedToCreateArchiveError, e
     ensure
       FileUtils.rm_f(archive_path) if archive_path
     end

app/services/pages/zip_directory_service.rb

@@ -4,8 +4,9 @@ module Pages
   class ZipDirectoryService
     include Gitlab::Utils::StrongMemoize
 
-    InvalidArchiveError = Class.new(RuntimeError)
-    InvalidEntryError = Class.new(RuntimeError)
+    Error = Class.new(::StandardError)
+    InvalidArchiveError = Class.new(Error)
+    InvalidEntryError = Class.new(Error)
 
     PUBLIC_DIR = 'public'
 
@@ -14,7 +15,7 @@ module Pages
     end
 
     def execute
-      raise InvalidArchiveError unless valid_work_directory?
+      raise InvalidArchiveError, "Invalid work directory: #{@input_dir}" unless valid_work_directory?
 
       output_file = File.join(real_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects
 
@@ -39,7 +40,7 @@ module Pages
 
       unless valid_path?(disk_file_path)
         # archive without public directory is completelly unusable
-        raise InvalidArchiveError if zipfile_path == PUBLIC_DIR
+        raise InvalidArchiveError, "Invalid public directory: #{disk_file_path}" if zipfile_path == PUBLIC_DIR
 
         # archive with invalid entry will just have this entry missing
         raise InvalidEntryError

changelogs/unreleased/pages-migration-task.yml

+---
+title: Add rake task for migrating legacy pages storage to zip deployments
+merge_request: 50153
+author:
+type: added

lib/tasks/gitlab/pages.rake

+require 'logger'
+
+namespace :gitlab do
+  namespace :pages do
+    desc "GitLab | Pages | Migrate legacy storage to zip format"
+    task migrate_legacy_storage: :gitlab_environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting to migrate legacy pages storage to zip deployments')
+      migrated_projects = 0
+
+      ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: 10) do |batch|
+        # count will actually be changed after migration, so increment before
+
+        batch.preload(project: [:namespace, :route, pages_metadatum: :pages_deployment]).each do |metadatum|
+          project = metadatum.project
+          time = Benchmark.realtime do
+            ::Pages::MigrateLegacyStorageToDeploymentService.new(project).execute
+          end
+
+          migrated_projects += 1
+
+          logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time} seconds")
+        rescue => e
+          logger.error("#{e.message} project_id: #{project&.id}")
+          Gitlab::ErrorTracking.track_exception(e, project_id: project&.id)
+        end
+
+        logger.info("#{migrated_projects} pages projects are migrated")
+      end
+    end
+  end
+end

spec/models/project_pages_metadatum_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ProjectPagesMetadatum do
+  describe '.only_on_legacy_storage' do
+    it 'returns only deployed records without deployment' do
+      create(:project) # without pages deployed
+
+      legacy_storage_project = create(:project)
+      legacy_storage_project.mark_pages_as_deployed
+
+      project_with_deployment = create(:project)
+      deployment = create(:pages_deployment, project: project_with_deployment)
+      project_with_deployment.mark_pages_as_deployed
+      project_with_deployment.update_pages_deployment!(deployment)
+
+      expect(described_class.only_on_legacy_storage).to eq([legacy_storage_project.pages_metadatum])
+    end
+  end
+end

spec/services/pages/migrate_legacy_storage_to_deployment_service_spec.rb

@@ -9,9 +9,13 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
   it 'marks pages as not deployed if public directory is absent' do
     project.mark_pages_as_deployed
 
+    expect(project.pages_metadatum.reload.deployed).to eq(true)
+
     expect do
       service.execute
-    end.to change { project.pages_metadatum.reload.deployed }.from(true).to(false)
+    end.to raise_error(described_class::FailedToCreateArchiveError)
+
+    expect(project.pages_metadatum.reload.deployed).to eq(false)
   end
 
   it 'does not mark pages as not deployed if public directory is absent but pages_deployment exists' do
@@ -19,9 +23,13 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
     project.update_pages_deployment!(deployment)
     project.mark_pages_as_deployed
 
+    expect(project.pages_metadatum.reload.deployed).to eq(true)
+
     expect do
       service.execute
-    end.not_to change { project.pages_metadatum.reload.deployed }.from(true)
+    end.to raise_error(described_class::FailedToCreateArchiveError)
+
+    expect(project.pages_metadatum.reload.deployed).to eq(true)
   end
 
   it 'does not mark pages as not deployed if public directory is absent but feature is disabled' do
@@ -29,9 +37,13 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
 
     project.mark_pages_as_deployed
 
+    expect(project.pages_metadatum.reload.deployed).to eq(true)
+
     expect do
       service.execute
-    end.not_to change { project.pages_metadatum.reload.deployed }.from(true)
+    end.to raise_error(described_class::FailedToCreateArchiveError)
+
+    expect(project.pages_metadatum.reload.deployed).to eq(true)
   end
 
   it 'removes pages archive when can not save deployment' do
@@ -94,7 +106,7 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
       described_class.new(project).try_obtain_lease do
         expect do
           described_class.new(project).execute
-        end.to raise_error(described_class::ExclusiveLeaseTaken)
+        end.to raise_error(described_class::ExclusiveLeaseTakenError)
       end
     end
   end

spec/tasks/gitlab/pages_rake_spec.rb

+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:pages:migrate_legacy_storagerake task' do
+  before(:context) do
+    Rake.application.rake_require 'tasks/gitlab/pages'
+  end
+
+  subject { run_rake_task('gitlab:pages:migrate_legacy_storage') }
+
+  let(:project) { create(:project) }
+
+  it 'does not try to migrate pages if pages are not deployed' do
+    expect(::Pages::MigrateLegacyStorageToDeploymentService).not_to receive(:new)
+
+    subject
+  end
+
+  context 'when pages are marked as deployed' do
+    before do
+      project.mark_pages_as_deployed
+    end
+
+    context 'when pages directory does not exist' do
+      it 'tries to migrate the project, but does not crash' do
+        expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project) do |service|
+          expect(service).to receive(:execute).and_call_original
+        end
+
+        subject
+      end
+    end
+
+    context 'when pages directory exists on disk' do
+      before do
+        FileUtils.mkdir_p File.join(project.pages_path, "public")
+        File.open(File.join(project.pages_path, "public/index.html"), "w") do |f|
+          f.write("Hello!")
+        end
+      end
+
+      it 'migrates pages projects without deployments' do
+        expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project) do |service|
+          expect(service).to receive(:execute).and_call_original
+        end
+
+        expect do
+          subject
+        end.to change { project.pages_metadatum.reload.pages_deployment }.from(nil)
+      end
+
+      context 'when deployed already exists for the project' do
+        before do
+          deployment = create(:pages_deployment, project: project)
+          project.set_first_pages_deployment!(deployment)
+        end
+
+        it 'does not try to migrate project' do
+          expect(::Pages::MigrateLegacyStorageToDeploymentService).not_to receive(:new)
+
+          subject
+        end
+      end
+    end
+  end
+end