Merge branch 'pages-migration-service' into 'master'

Add Pages migration service

See merge request gitlab-org/gitlab!49473

app/models/project.rb

@@ -1829,6 +1829,15 @@ class Project < ApplicationRecord
     ensure_pages_metadatum.update!(pages_deployment: deployment)
   end
 
+  def set_first_pages_deployment!(deployment)
+    ensure_pages_metadatum
+
+    # where().update_all to perform update in the single transaction with check for null
+    ProjectPagesMetadatum
+      .where(project_id: id, pages_deployment_id: nil)
+      .update_all(pages_deployment_id: deployment.id)
+  end
+
   def write_repository_config(gl_full_path: full_path)
     # We'd need to keep track of project full path otherwise directory tree
     # created with hashed storage enabled cannot be usefully imported using

app/services/pages/migrate_legacy_storage_to_deployment_service.rb

+# frozen_string_literal: true
+
+module Pages
+  class MigrateLegacyStorageToDeploymentService
+    ExclusiveLeaseTaken = Class.new(StandardError)
+
+    include ::Pages::LegacyStorageLease
+
+    attr_reader :project
+
+    def initialize(project)
+      @project = project
+    end
+
+    def execute
+      migrated = try_obtain_lease do
+        execute_unsafe
+
+        true
+      end
+
+      raise ExclusiveLeaseTaken, "Can't migrate pages for project #{project.id}: exclusive lease taken" unless migrated
+    end
+
+    private
+
+    def execute_unsafe
+      archive_path, entries_count = ::Pages::ZipDirectoryService.new(project.pages_path).execute
+
+      deployment = nil
+      File.open(archive_path) do |file|
+        deployment = project.pages_deployments.create!(
+          file: file,
+          file_count: entries_count,
+          file_sha256: Digest::SHA256.file(archive_path).hexdigest
+        )
+      end
+
+      project.set_first_pages_deployment!(deployment)
+    rescue ::Pages::ZipDirectoryService::InvalidArchiveError => e
+      Gitlab::ErrorTracking.track_exception(e, project_id: project.id)
+
+      if !project.pages_metadatum&.reload&.pages_deployment &&
+         Feature.enabled?(:pages_migration_mark_as_not_deployed, project)
+        project.mark_pages_as_not_deployed
+      end
+
+    ensure
+      FileUtils.rm_f(archive_path) if archive_path
+    end
+  end
+end

app/services/pages/zip_directory_service.rb

@@ -2,32 +2,40 @@
 
 module Pages
   class ZipDirectoryService
+    include Gitlab::Utils::StrongMemoize
+
     InvalidArchiveError = Class.new(RuntimeError)
     InvalidEntryError = Class.new(RuntimeError)
 
     PUBLIC_DIR = 'public'
 
     def initialize(input_dir)
-      @input_dir = File.realpath(input_dir)
-      @output_file = File.join(@input_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects
+      @input_dir = input_dir
     end
 
     def execute
-      FileUtils.rm_f(@output_file)
+      raise InvalidArchiveError unless valid_work_directory?
+
+      output_file = File.join(real_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects
+
+      FileUtils.rm_f(output_file)
 
       count = 0
-      ::Zip::File.open(@output_file, ::Zip::File::CREATE) do |zipfile|
+      ::Zip::File.open(output_file, ::Zip::File::CREATE) do |zipfile|
         write_entry(zipfile, PUBLIC_DIR)
         count = zipfile.entries.count
       end
 
-      [@output_file, count]
+      [output_file, count]
+    rescue => e
+      FileUtils.rm_f(output_file) if output_file
+      raise e
     end
 
     private
 
     def write_entry(zipfile, zipfile_path)
-      disk_file_path = File.join(@input_dir, zipfile_path)
+      disk_file_path = File.join(real_dir, zipfile_path)
 
       unless valid_path?(disk_file_path)
         # archive without public directory is completelly unusable
@@ -71,13 +79,27 @@ module Pages
     def valid_path?(disk_file_path)
       realpath = File.realpath(disk_file_path)
 
-      realpath == File.join(@input_dir, PUBLIC_DIR) ||
-        realpath.start_with?(File.join(@input_dir, PUBLIC_DIR + "/"))
+      realpath == File.join(real_dir, PUBLIC_DIR) ||
+        realpath.start_with?(File.join(real_dir, PUBLIC_DIR + "/"))
     # happens if target of symlink isn't there
     rescue => e
-      Gitlab::ErrorTracking.track_exception(e, input_dir: @input_dir, disk_file_path: disk_file_path)
+      Gitlab::ErrorTracking.track_exception(e, input_dir: real_dir, disk_file_path: disk_file_path)
 
       false
     end
+
+    def valid_work_directory?
+      Dir.exist?(real_dir)
+    rescue => e
+      Gitlab::ErrorTracking.track_exception(e, input_dir: @input_dir)
+
+      false
+    end
+
+    def real_dir
+      strong_memoize(:real_dir) do
+        File.realpath(@input_dir) rescue nil
+      end
+    end
   end
 end

config/feature_flags/development/pages_migration_mark_as_not_deployed.yml

+---
+name: pages_migration_mark_as_not_deployed
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49473
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/295187
+milestone: '13.8'
+type: development
+group: group::release
+default_enabled: false

spec/models/project_spec.rb

@@ -6002,6 +6002,43 @@ RSpec.describe Project, factory_default: :keep do
     end
   end
 
+  describe '#set_first_pages_deployment!' do
+    let(:project) { create(:project) }
+    let(:deployment) { create(:pages_deployment, project: project) }
+
+    it "creates new metadata record if none exists yet and sets deployment" do
+      project.pages_metadatum.destroy!
+      project.reload
+
+      project.set_first_pages_deployment!(deployment)
+
+      expect(project.pages_metadatum.reload.pages_deployment).to eq(deployment)
+    end
+
+    it "updates the existing metadara record with deployment" do
+      expect do
+        project.set_first_pages_deployment!(deployment)
+      end.to change { project.pages_metadatum.reload.pages_deployment }.from(nil).to(deployment)
+    end
+
+    it 'only updates metadata for this project' do
+      other_project = create(:project)
+
+      expect do
+        project.set_first_pages_deployment!(deployment)
+      end.not_to change { other_project.pages_metadatum.reload.pages_deployment }.from(nil)
+    end
+
+    it 'does nothing if metadata already references some deployment' do
+      existing_deployment = create(:pages_deployment, project: project)
+      project.set_first_pages_deployment!(existing_deployment)
+
+      expect do
+        project.set_first_pages_deployment!(deployment)
+      end.not_to change { project.pages_metadatum.reload.pages_deployment }.from(existing_deployment)
+    end
+  end
+
   describe '#has_pool_repsitory?' do
     it 'returns false when it does not have a pool repository' do
       subject = create(:project, :repository)

spec/services/pages/migrate_legacy_storage_to_deployment_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
+  let(:project) { create(:project, :repository) }
+  let(:service) { described_class.new(project) }
+
+  it 'marks pages as not deployed if public directory is absent' do
+    project.mark_pages_as_deployed
+
+    expect do
+      service.execute
+    end.to change { project.pages_metadatum.reload.deployed }.from(true).to(false)
+  end
+
+  it 'does not mark pages as not deployed if public directory is absent but pages_deployment exists' do
+    deployment = create(:pages_deployment, project: project)
+    project.update_pages_deployment!(deployment)
+    project.mark_pages_as_deployed
+
+    expect do
+      service.execute
+    end.not_to change { project.pages_metadatum.reload.deployed }.from(true)
+  end
+
+  it 'does not mark pages as not deployed if public directory is absent but feature is disabled' do
+    stub_feature_flags(pages_migration_mark_as_not_deployed: false)
+
+    project.mark_pages_as_deployed
+
+    expect do
+      service.execute
+    end.not_to change { project.pages_metadatum.reload.deployed }.from(true)
+  end
+
+  it 'removes pages archive when can not save deployment' do
+    archive = fixture_file_upload("spec/fixtures/pages.zip")
+    expect_next_instance_of(::Pages::ZipDirectoryService) do |zip_service|
+      expect(zip_service).to receive(:execute).and_return([archive.path, 3])
+    end
+
+    expect_next_instance_of(PagesDeployment) do |deployment|
+      expect(deployment).to receive(:save!).and_raise("Something")
+    end
+
+    expect do
+      service.execute
+    end.to raise_error("Something")
+
+    expect(File.exist?(archive.path)).to eq(false)
+  end
+
+  context 'when pages site is deployed to legacy storage' do
+    before do
+      FileUtils.mkdir_p File.join(project.pages_path, "public")
+      File.open(File.join(project.pages_path, "public/index.html"), "w") do |f|
+        f.write("Hello!")
+      end
+    end
+
+    it 'creates pages deployment' do
+      expect do
+        described_class.new(project).execute
+      end.to change { project.reload.pages_deployments.count }.by(1)
+
+      deployment = project.pages_metadatum.pages_deployment
+
+      Zip::File.open(deployment.file.path) do |zip_file|
+        expect(zip_file.glob("public").first.ftype).to eq(:directory)
+        expect(zip_file.glob("public/index.html").first.get_input_stream.read).to eq("Hello!")
+      end
+
+      expect(deployment.file_count).to eq(2)
+      expect(deployment.file_sha256).to eq(Digest::SHA256.file(deployment.file.path).hexdigest)
+    end
+
+    it 'removes tmp pages archive' do
+      described_class.new(project).execute
+
+      expect(File.exist?(File.join(project.pages_path, '@migrated.zip'))).to eq(false)
+    end
+
+    it 'does not change pages deployment if it is set' do
+      old_deployment = create(:pages_deployment, project: project)
+      project.update_pages_deployment!(old_deployment)
+
+      expect do
+        described_class.new(project).execute
+      end.not_to change { project.pages_metadatum.reload.pages_deployment_id }.from(old_deployment.id)
+    end
+
+    it 'raises exception if exclusive lease is taken' do
+      described_class.new(project).try_obtain_lease do
+        expect do
+          described_class.new(project).execute
+        end.to raise_error(described_class::ExclusiveLeaseTaken)
+      end
+    end
+  end
+end

spec/services/pages/zip_directory_service_spec.rb

@@ -3,6 +3,13 @@
 require 'spec_helper'
 
 RSpec.describe Pages::ZipDirectoryService do
+  it 'raises error if project pages dir does not exist' do
+    expect do
+      described_class.new("/tmp/not/existing/dir").execute
+    end.to raise_error(described_class::InvalidArchiveError)
+  end
+
+  context 'when work dir exists' do
     around do |example|
       Dir.mktmpdir do |dir|
         @work_dir = dir
@@ -17,8 +24,10 @@ RSpec.describe Pages::ZipDirectoryService do
     let(:archive) { result.first }
     let(:entries_count) { result.second }
 
-  it 'raises error if there is no public directory' do
+    it 'raises error if there is no public directory and does not leave archive' do
       expect { archive }.to raise_error(described_class::InvalidArchiveError)
+
+      expect(File.exist?(File.join(@work_dir, '@migrated.zip'))).to eq(false)
     end
 
     it 'raises error if public directory is a symlink' do
@@ -206,4 +215,5 @@ RSpec.describe Pages::ZipDirectoryService do
         yield zip_file
       end
     end
+  end
 end