Merge branch '295206-follow-up-from-add-pages-migration-service' into 'master'

Refactor pages migration services

See merge request gitlab-org/gitlab!50624

app/services/pages/migrate_legacy_storage_to_deployment_service.rb

@@ -3,8 +3,8 @@
 module Pages
   class MigrateLegacyStorageToDeploymentService
     ExclusiveLeaseTakenError = Class.new(StandardError)
-    FailedToCreateArchiveError = Class.new(StandardError)
 
+    include BaseServiceUtility
     include ::Pages::LegacyStorageLease
 
     attr_reader :project
@@ -14,37 +14,43 @@ module Pages
     end
 
     def execute
-      migrated = try_obtain_lease do
+      result = try_obtain_lease do
         execute_unsafe
-
-        true
       end
 
-      raise ExclusiveLeaseTakenError, "Can't migrate pages for project #{project.id}: exclusive lease taken" unless migrated
+      raise ExclusiveLeaseTakenError, "Can't migrate pages for project #{project.id}: exclusive lease taken" if result.nil?
+
+      result
     end
 
     private
 
     def execute_unsafe
-      archive_path, entries_count = ::Pages::ZipDirectoryService.new(project.pages_path).execute
+      zip_result = ::Pages::ZipDirectoryService.new(project.pages_path).execute
+
+      if zip_result[:status] == :error
+        if !project.pages_metadatum&.reload&.pages_deployment &&
+           Feature.enabled?(:pages_migration_mark_as_not_deployed, project)
+          project.mark_pages_as_not_deployed
+        end
+
+        return error("Can't create zip archive: #{zip_result[:message]}")
+      end
+
+      archive_path = zip_result[:archive_path]
 
       deployment = nil
       File.open(archive_path) do |file|
         deployment = project.pages_deployments.create!(
           file: file,
-          file_count: entries_count,
+          file_count: zip_result[:entries_count],
           file_sha256: Digest::SHA256.file(archive_path).hexdigest
         )
       end
 
       project.set_first_pages_deployment!(deployment)
-    rescue ::Pages::ZipDirectoryService::InvalidArchiveError => e
-      if !project.pages_metadatum&.reload&.pages_deployment &&
-         Feature.enabled?(:pages_migration_mark_as_not_deployed, project)
-        project.mark_pages_as_not_deployed
-      end
 
-      raise FailedToCreateArchiveError, e
+      success
     ensure
       FileUtils.rm_f(archive_path) if archive_path
     end

app/services/pages/zip_directory_service.rb

@@ -2,11 +2,11 @@
 
 module Pages
   class ZipDirectoryService
+    include BaseServiceUtility
     include Gitlab::Utils::StrongMemoize
 
-    Error = Class.new(::StandardError)
-    InvalidArchiveError = Class.new(Error)
-    InvalidEntryError = Class.new(Error)
+    # used only to track exceptions in Sentry
+    InvalidEntryError = Class.new(StandardError)
 
     PUBLIC_DIR = 'public'
 
@@ -15,19 +15,19 @@ module Pages
     end
 
     def execute
-      raise InvalidArchiveError, "Invalid work directory: #{@input_dir}" unless valid_work_directory?
+      return error("Can not find valid public dir in #{@input_dir}") unless valid_path?(public_dir)
 
       output_file = File.join(real_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects
 
       FileUtils.rm_f(output_file)
 
-      count = 0
+      entries_count = 0
       ::Zip::File.open(output_file, ::Zip::File::CREATE) do |zipfile|
         write_entry(zipfile, PUBLIC_DIR)
-        count = zipfile.entries.count
+        entries_count = zipfile.entries.count
       end
 
-      [output_file, count]
+      success(archive_path: output_file, entries_count: entries_count)
     rescue => e
       FileUtils.rm_f(output_file) if output_file
       raise e
@@ -39,9 +39,6 @@ module Pages
       disk_file_path = File.join(real_dir, zipfile_path)
 
       unless valid_path?(disk_file_path)
-        # archive without public directory is completelly unusable
-        raise InvalidArchiveError, "Invalid public directory: #{disk_file_path}" if zipfile_path == PUBLIC_DIR
-
         # archive with invalid entry will just have this entry missing
         raise InvalidEntryError
       end
@@ -80,8 +77,7 @@ module Pages
     def valid_path?(disk_file_path)
       realpath = File.realpath(disk_file_path)
 
-      realpath == File.join(real_dir, PUBLIC_DIR) ||
-        realpath.start_with?(File.join(real_dir, PUBLIC_DIR + "/"))
+      realpath == public_dir || realpath.start_with?(public_dir + "/")
     # happens if target of symlink isn't there
     rescue => e
       Gitlab::ErrorTracking.track_exception(e, input_dir: real_dir, disk_file_path: disk_file_path)
@@ -89,18 +85,16 @@ module Pages
       false
     end
 
-    def valid_work_directory?
-      Dir.exist?(real_dir)
-    rescue => e
-      Gitlab::ErrorTracking.track_exception(e, input_dir: @input_dir)
-
-      false
-    end
-
     def real_dir
       strong_memoize(:real_dir) do
         File.realpath(@input_dir) rescue nil
       end
     end
+
+    def public_dir
+      strong_memoize(:public_dir) do
+        File.join(real_dir, PUBLIC_DIR) rescue nil
+      end
+    end
   end
 end

lib/tasks/gitlab/pages.rake

@@ -6,24 +6,29 @@ namespace :gitlab do
     task migrate_legacy_storage: :gitlab_environment do
       logger = Logger.new(STDOUT)
       logger.info('Starting to migrate legacy pages storage to zip deployments')
-      migrated_projects = 0
+      processed_projects = 0
 
       ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: 10) do |batch|
         batch.preload(project: [:namespace, :route, pages_metadatum: :pages_deployment]).each do |metadatum|
           project = metadatum.project
+
+          result = nil
           time = Benchmark.realtime do
-            ::Pages::MigrateLegacyStorageToDeploymentService.new(project).execute
+            result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project).execute
           end
+          processed_projects += 1
 
-          migrated_projects += 1
-
-          logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time} seconds")
+          if result[:status] == :success
+            logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time} seconds")
+          else
+            logger.error("project_id: #{project.id} #{project.pages_path} failed to be migrated in #{time} seconds: #{result[:message]}")
+          end
         rescue => e
           logger.error("#{e.message} project_id: #{project&.id}")
           Gitlab::ErrorTracking.track_exception(e, project_id: project&.id)
         end
 
-        logger.info("#{migrated_projects} pages projects are migrated")
+        logger.info("#{processed_projects} pages projects are processed")
       end
     end
   end

spec/services/pages/migrate_legacy_storage_to_deployment_service_spec.rb

@@ -11,9 +11,10 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
 
     expect(project.pages_metadatum.reload.deployed).to eq(true)
 
-    expect do
-      service.execute
-    end.to raise_error(described_class::FailedToCreateArchiveError)
+    expect(service.execute).to(
+      eq(status: :error,
+         message: "Can't create zip archive: Can not find valid public dir in #{project.pages_path}")
+    )
 
     expect(project.pages_metadatum.reload.deployed).to eq(false)
   end
@@ -25,9 +26,10 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
 
     expect(project.pages_metadatum.reload.deployed).to eq(true)
 
-    expect do
-      service.execute
-    end.to raise_error(described_class::FailedToCreateArchiveError)
+    expect(service.execute).to(
+      eq(status: :error,
+         message: "Can't create zip archive: Can not find valid public dir in #{project.pages_path}")
+    )
 
     expect(project.pages_metadatum.reload.deployed).to eq(true)
   end
@@ -39,9 +41,10 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
 
     expect(project.pages_metadatum.reload.deployed).to eq(true)
 
-    expect do
-      service.execute
-    end.to raise_error(described_class::FailedToCreateArchiveError)
+    expect(service.execute).to(
+      eq(status: :error,
+         message: "Can't create zip archive: Can not find valid public dir in #{project.pages_path}")
+    )
 
     expect(project.pages_metadatum.reload.deployed).to eq(true)
   end
@@ -49,7 +52,9 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
   it 'removes pages archive when can not save deployment' do
     archive = fixture_file_upload("spec/fixtures/pages.zip")
     expect_next_instance_of(::Pages::ZipDirectoryService) do |zip_service|
-      expect(zip_service).to receive(:execute).and_return([archive.path, 3])
+      expect(zip_service).to receive(:execute).and_return(status: :success,
+                                                          archive_path: archive.path,
+                                                          entries_count: 3)
     end
 
     expect_next_instance_of(PagesDeployment) do |deployment|
@@ -73,7 +78,7 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
 
     it 'creates pages deployment' do
       expect do
-        described_class.new(project).execute
+        expect(described_class.new(project).execute).to eq(status: :success)
       end.to change { project.reload.pages_deployments.count }.by(1)
 
       deployment = project.pages_metadatum.pages_deployment

spec/services/pages/zip_directory_service_spec.rb

@@ -3,217 +3,223 @@
 require 'spec_helper'
 
 RSpec.describe Pages::ZipDirectoryService do
-  it 'raises error if project pages dir does not exist' do
-    expect do
-      described_class.new("/tmp/not/existing/dir").execute
-    end.to raise_error(described_class::InvalidArchiveError)
+  around do |example|
+    Dir.mktmpdir do |dir|
+      @work_dir = dir
+      example.run
+    end
   end
 
-  context 'when work dir exists' do
-    around do |example|
-      Dir.mktmpdir do |dir|
-        @work_dir = dir
-        example.run
-      end
-    end
+  let(:result) do
+    described_class.new(@work_dir).execute
+  end
 
-    let(:result) do
-      described_class.new(@work_dir).execute
-    end
+  let(:status) { result[:status] }
+  let(:message) { result[:message] }
+  let(:archive) { result[:archive_path] }
+  let(:entries_count) { result[:entries_count] }
 
-    let(:archive) { result.first }
-    let(:entries_count) { result.second }
+  it 'returns error if project pages dir does not exist' do
+    expect(
+      described_class.new("/tmp/not/existing/dir").execute
+    ).to eq(status: :error, message: "Can not find valid public dir in /tmp/not/existing/dir")
+  end
 
-    it 'raises error if there is no public directory and does not leave archive' do
-      expect { archive }.to raise_error(described_class::InvalidArchiveError)
+  it 'returns nils if there is no public directory and does not leave archive' do
+    expect(status).to eq(:error)
+    expect(message).to eq("Can not find valid public dir in #{@work_dir}")
+    expect(archive).to eq(nil)
+    expect(entries_count).to eq(nil)
 
-      expect(File.exist?(File.join(@work_dir, '@migrated.zip'))).to eq(false)
-    end
+    expect(File.exist?(File.join(@work_dir, '@migrated.zip'))).to eq(false)
+  end
 
-    it 'raises error if public directory is a symlink' do
-      create_dir('target')
-      create_file('./target/index.html', 'hello')
-      create_link("public", "./target")
+  it 'returns nils if public directory is a symlink' do
+    create_dir('target')
+    create_file('./target/index.html', 'hello')
+    create_link("public", "./target")
 
-      expect { archive }.to raise_error(described_class::InvalidArchiveError)
-    end
+    expect(status).to eq(:error)
+    expect(message).to eq("Can not find valid public dir in #{@work_dir}")
+    expect(archive).to eq(nil)
+    expect(entries_count).to eq(nil)
+  end
 
-    context 'when there is a public directory' do
-      before do
-        create_dir('public')
-      end
+  context 'when there is a public directory' do
+    before do
+      create_dir('public')
+    end
 
-      it 'creates the file next the public directory' do
-        expect(archive).to eq(File.join(@work_dir, "@migrated.zip"))
-      end
+    it 'creates the file next the public directory' do
+      expect(archive).to eq(File.join(@work_dir, "@migrated.zip"))
+    end
 
-      it 'includes public directory' do
-        with_zip_file do |zip_file|
-          entry = zip_file.get_entry("public/")
-          expect(entry.ftype).to eq(:directory)
-        end
+    it 'includes public directory' do
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/")
+        expect(entry.ftype).to eq(:directory)
       end
+    end
 
-      it 'returns number of entries' do
-        create_file("public/index.html", "hello")
-        create_link("public/link.html", "./index.html")
-        expect(entries_count).to eq(3) # + 'public' directory
-      end
+    it 'returns number of entries' do
+      create_file("public/index.html", "hello")
+      create_link("public/link.html", "./index.html")
+      expect(entries_count).to eq(3) # + 'public' directory
+    end
 
-      it 'removes the old file if it exists' do
-        # simulate the old run
-        described_class.new(@work_dir).execute
+    it 'removes the old file if it exists' do
+      # simulate the old run
+      described_class.new(@work_dir).execute
 
-        with_zip_file do |zip_file|
-          expect(zip_file.entries.count).to eq(1)
-        end
+      with_zip_file do |zip_file|
+        expect(zip_file.entries.count).to eq(1)
       end
+    end
 
-      it 'ignores other top level files and directories' do
-        create_file("top_level.html", "hello")
-        create_dir("public2")
+    it 'ignores other top level files and directories' do
+      create_file("top_level.html", "hello")
+      create_dir("public2")
 
-        with_zip_file do |zip_file|
-          expect { zip_file.get_entry("top_level.html") }.to raise_error(Errno::ENOENT)
-          expect { zip_file.get_entry("public2/") }.to raise_error(Errno::ENOENT)
-        end
+      with_zip_file do |zip_file|
+        expect { zip_file.get_entry("top_level.html") }.to raise_error(Errno::ENOENT)
+        expect { zip_file.get_entry("public2/") }.to raise_error(Errno::ENOENT)
       end
+    end
 
-      it 'includes index.html file' do
-        create_file("public/index.html", "Hello!")
+    it 'includes index.html file' do
+      create_file("public/index.html", "Hello!")
 
-        with_zip_file do |zip_file|
-          entry = zip_file.get_entry("public/index.html")
-          expect(zip_file.read(entry)).to eq("Hello!")
-        end
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/index.html")
+        expect(zip_file.read(entry)).to eq("Hello!")
       end
+    end
 
-      it 'includes hidden file' do
-        create_file("public/.hidden.html", "Hello!")
+    it 'includes hidden file' do
+      create_file("public/.hidden.html", "Hello!")
 
-        with_zip_file do |zip_file|
-          entry = zip_file.get_entry("public/.hidden.html")
-          expect(zip_file.read(entry)).to eq("Hello!")
-        end
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/.hidden.html")
+        expect(zip_file.read(entry)).to eq("Hello!")
       end
+    end
 
-      it 'includes nested directories and files' do
-        create_dir("public/nested")
-        create_dir("public/nested/nested2")
-        create_file("public/nested/nested2/nested.html", "Hello nested")
+    it 'includes nested directories and files' do
+      create_dir("public/nested")
+      create_dir("public/nested/nested2")
+      create_file("public/nested/nested2/nested.html", "Hello nested")
 
-        with_zip_file do |zip_file|
-          entry = zip_file.get_entry("public/nested")
-          expect(entry.ftype).to eq(:directory)
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/nested")
+        expect(entry.ftype).to eq(:directory)
 
-          entry = zip_file.get_entry("public/nested/nested2")
-          expect(entry.ftype).to eq(:directory)
+        entry = zip_file.get_entry("public/nested/nested2")
+        expect(entry.ftype).to eq(:directory)
 
-          entry = zip_file.get_entry("public/nested/nested2/nested.html")
-          expect(zip_file.read(entry)).to eq("Hello nested")
-        end
+        entry = zip_file.get_entry("public/nested/nested2/nested.html")
+        expect(zip_file.read(entry)).to eq("Hello nested")
       end
+    end
 
-      it 'adds a valid symlink' do
-        create_file("public/target.html", "hello")
-        create_link("public/link.html", "./target.html")
+    it 'adds a valid symlink' do
+      create_file("public/target.html", "hello")
+      create_link("public/link.html", "./target.html")
 
-        with_zip_file do |zip_file|
-          entry = zip_file.get_entry("public/link.html")
-          expect(entry.ftype).to eq(:symlink)
-          expect(zip_file.read(entry)).to eq("./target.html")
-        end
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/link.html")
+        expect(entry.ftype).to eq(:symlink)
+        expect(zip_file.read(entry)).to eq("./target.html")
       end
+    end
 
-      it 'ignores the symlink pointing outside of public directory' do
-        create_file("target.html", "hello")
-        create_link("public/link.html", "../target.html")
+    it 'ignores the symlink pointing outside of public directory' do
+      create_file("target.html", "hello")
+      create_link("public/link.html", "../target.html")
 
-        with_zip_file do |zip_file|
-          expect { zip_file.get_entry("public/link.html") }.to raise_error(Errno::ENOENT)
-        end
+      with_zip_file do |zip_file|
+        expect { zip_file.get_entry("public/link.html") }.to raise_error(Errno::ENOENT)
       end
+    end
 
-      it 'ignores the symlink if target is absent' do
-        create_link("public/link.html", "./target.html")
+    it 'ignores the symlink if target is absent' do
+      create_link("public/link.html", "./target.html")
 
-        with_zip_file do |zip_file|
-          expect { zip_file.get_entry("public/link.html") }.to raise_error(Errno::ENOENT)
-        end
+      with_zip_file do |zip_file|
+        expect { zip_file.get_entry("public/link.html") }.to raise_error(Errno::ENOENT)
       end
+    end
 
-      it 'ignores symlink if is absolute and points to outside of directory' do
-        target = File.join(@work_dir, "target")
-        FileUtils.touch(target)
+    it 'ignores symlink if is absolute and points to outside of directory' do
+      target = File.join(@work_dir, "target")
+      FileUtils.touch(target)
 
-        create_link("public/link.html", target)
+      create_link("public/link.html", target)
 
-        with_zip_file do |zip_file|
-          expect { zip_file.get_entry("public/link.html") }.to raise_error(Errno::ENOENT)
-        end
+      with_zip_file do |zip_file|
+        expect { zip_file.get_entry("public/link.html") }.to raise_error(Errno::ENOENT)
       end
+    end
 
-      it "includes raw symlink if it's target is a valid directory" do
-        create_dir("public/target")
-        create_file("public/target/index.html", "hello")
-        create_link("public/link", "./target")
+    it "includes raw symlink if it's target is a valid directory" do
+      create_dir("public/target")
+      create_file("public/target/index.html", "hello")
+      create_link("public/link", "./target")
 
-        with_zip_file do |zip_file|
-          expect(zip_file.entries.count).to eq(4) # /public and 3 created above
+      with_zip_file do |zip_file|
+        expect(zip_file.entries.count).to eq(4) # /public and 3 created above
 
-          entry = zip_file.get_entry("public/link")
-          expect(entry.ftype).to eq(:symlink)
-          expect(zip_file.read(entry)).to eq("./target")
-        end
+        entry = zip_file.get_entry("public/link")
+        expect(entry.ftype).to eq(:symlink)
+        expect(zip_file.read(entry)).to eq("./target")
       end
     end
+  end
 
-    context "validating fixtures pages archives" do
-      using RSpec::Parameterized::TableSyntax
+  context "validating fixtures pages archives" do
+    using RSpec::Parameterized::TableSyntax
 
-      where(:fixture_path) do
-        ["spec/fixtures/pages.zip", "spec/fixtures/pages_non_writeable.zip"]
-      end
+    where(:fixture_path) do
+      ["spec/fixtures/pages.zip", "spec/fixtures/pages_non_writeable.zip"]
+    end
 
-      with_them do
-        let(:full_fixture_path) { Rails.root.join(fixture_path) }
+    with_them do
+      let(:full_fixture_path) { Rails.root.join(fixture_path) }
 
-        it 'a created archives contains exactly the same entries' do
-          SafeZip::Extract.new(full_fixture_path).extract(directories: ['public'], to: @work_dir)
+      it 'a created archives contains exactly the same entries' do
+        SafeZip::Extract.new(full_fixture_path).extract(directories: ['public'], to: @work_dir)
 
-          with_zip_file do |created_archive|
-            Zip::File.open(full_fixture_path) do |original_archive|
-              original_archive.entries do |original_entry|
-                created_entry = created_archive.get_entry(original_entry.name)
+        with_zip_file do |created_archive|
+          Zip::File.open(full_fixture_path) do |original_archive|
+            original_archive.entries do |original_entry|
+              created_entry = created_archive.get_entry(original_entry.name)
 
-                expect(created_entry.name).to eq(original_entry.name)
-                expect(created_entry.ftype).to eq(original_entry.ftype)
-                expect(created_archive.read(created_entry)).to eq(original_archive.read(original_entry))
-              end
+              expect(created_entry.name).to eq(original_entry.name)
+              expect(created_entry.ftype).to eq(original_entry.ftype)
+              expect(created_archive.read(created_entry)).to eq(original_archive.read(original_entry))
             end
           end
         end
       end
     end
+  end
 
-    def create_file(name, content)
-      File.open(File.join(@work_dir, name), "w") do |f|
-        f.write(content)
-      end
+  def create_file(name, content)
+    File.open(File.join(@work_dir, name), "w") do |f|
+      f.write(content)
     end
+  end
 
-    def create_dir(dir)
-      Dir.mkdir(File.join(@work_dir, dir))
-    end
+  def create_dir(dir)
+    Dir.mkdir(File.join(@work_dir, dir))
+  end
 
-    def create_link(new_name, target)
-      File.symlink(target, File.join(@work_dir, new_name))
-    end
+  def create_link(new_name, target)
+    File.symlink(target, File.join(@work_dir, new_name))
+  end
 
-    def with_zip_file
-      Zip::File.open(archive) do |zip_file|
-        yield zip_file
-      end
+  def with_zip_file
+    Zip::File.open(archive) do |zip_file|
+      yield zip_file
     end
   end
 end