Commit b57c1b57 authored by Etienne Baqué's avatar Etienne Baqué

Merge branch '295187-roll-out-pages_migration_mark_as_not_deployed-feature-flag' into 'master'

Replace pages_migration_mark_as_not_deployed feature flag with env variable [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!55862
parents 1e2ae8e5 2178113b
...@@ -2,11 +2,12 @@ ...@@ -2,11 +2,12 @@
module Pages module Pages
class MigrateFromLegacyStorageService class MigrateFromLegacyStorageService
def initialize(logger, migration_threads:, batch_size:, ignore_invalid_entries:) def initialize(logger, migration_threads:, batch_size:, ignore_invalid_entries:, mark_projects_as_not_deployed:)
@logger = logger @logger = logger
@migration_threads = migration_threads @migration_threads = migration_threads
@batch_size = batch_size @batch_size = batch_size
@ignore_invalid_entries = ignore_invalid_entries @ignore_invalid_entries = ignore_invalid_entries
@mark_projects_as_not_deployed = mark_projects_as_not_deployed
@migrated = 0 @migrated = 0
@errored = 0 @errored = 0
...@@ -60,7 +61,9 @@ module Pages ...@@ -60,7 +61,9 @@ module Pages
def migrate_project(project) def migrate_project(project)
result = nil result = nil
time = Benchmark.realtime do time = Benchmark.realtime do
result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project, ignore_invalid_entries: @ignore_invalid_entries).execute result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project,
ignore_invalid_entries: @ignore_invalid_entries,
mark_projects_as_not_deployed: @mark_projects_as_not_deployed).execute
end end
if result[:status] == :success if result[:status] == :success
......
...@@ -9,9 +9,10 @@ module Pages ...@@ -9,9 +9,10 @@ module Pages
attr_reader :project attr_reader :project
def initialize(project, ignore_invalid_entries: false) def initialize(project, ignore_invalid_entries: false, mark_projects_as_not_deployed: false)
@project = project @project = project
@ignore_invalid_entries = ignore_invalid_entries @ignore_invalid_entries = ignore_invalid_entries
@mark_projects_as_not_deployed = mark_projects_as_not_deployed
end end
def execute def execute
...@@ -36,10 +37,12 @@ module Pages ...@@ -36,10 +37,12 @@ module Pages
archive_path = zip_result[:archive_path] archive_path = zip_result[:archive_path]
unless archive_path unless archive_path
return error("Archive not created. Missing public directory in #{@project.pages_path}") unless @mark_projects_as_not_deployed
project.set_first_pages_deployment!(nil) project.set_first_pages_deployment!(nil)
return success( return success(
message: "Archive not created. Missing public directory in #{project.pages_path} ? Marked project as not deployed") message: "Archive not created. Missing public directory in #{project.pages_path}? Marked project as not deployed")
end end
deployment = nil deployment = nil
......
...@@ -18,13 +18,7 @@ module Pages ...@@ -18,13 +18,7 @@ module Pages
end end
def execute def execute
unless resolve_public_dir return success unless resolve_public_dir
if Feature.enabled?(:pages_migration_mark_as_not_deployed)
return success
end
return error("Can not find valid public dir in #{@input_dir}")
end
output_file = File.join(real_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects output_file = File.join(real_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects
......
---
title: Allow users to mark pages projects as not deployed during migration to zip
storage
merge_request: 55862
author:
type: added
---
name: pages_migration_mark_as_not_deployed
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49473
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/295187
milestone: '13.8'
type: development
group: group::release
default_enabled: false
...@@ -11,7 +11,8 @@ namespace :gitlab do ...@@ -11,7 +11,8 @@ namespace :gitlab do
result = ::Pages::MigrateFromLegacyStorageService.new(logger, result = ::Pages::MigrateFromLegacyStorageService.new(logger,
migration_threads: migration_threads, migration_threads: migration_threads,
batch_size: batch_size, batch_size: batch_size,
ignore_invalid_entries: ignore_invalid_entries).execute ignore_invalid_entries: ignore_invalid_entries,
mark_projects_as_not_deployed: mark_projects_as_not_deployed).execute
logger.info("A total of #{result[:migrated] + result[:errored]} projects were processed.") logger.info("A total of #{result[:migrated] + result[:errored]} projects were processed.")
logger.info("- The #{result[:migrated]} projects migrated successfully") logger.info("- The #{result[:migrated]} projects migrated successfully")
...@@ -51,5 +52,11 @@ namespace :gitlab do ...@@ -51,5 +52,11 @@ namespace :gitlab do
ENV.fetch('PAGES_MIGRATION_IGNORE_INVALID_ENTRIES', 'false') ENV.fetch('PAGES_MIGRATION_IGNORE_INVALID_ENTRIES', 'false')
) )
end end
def mark_projects_as_not_deployed
Gitlab::Utils.to_boolean(
ENV.fetch('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'false')
)
end
end end
end end
...@@ -3,7 +3,9 @@ ...@@ -3,7 +3,9 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe Pages::MigrateFromLegacyStorageService do RSpec.describe Pages::MigrateFromLegacyStorageService do
let(:service) { described_class.new(Rails.logger, migration_threads: 3, batch_size: 10, ignore_invalid_entries: false) } let(:batch_size) { 10 }
let(:mark_projects_as_not_deployed) { false }
let(:service) { described_class.new(Rails.logger, migration_threads: 3, batch_size: batch_size, ignore_invalid_entries: false, mark_projects_as_not_deployed: mark_projects_as_not_deployed) }
it 'does not try to migrate pages if pages are not deployed' do it 'does not try to migrate pages if pages are not deployed' do
expect(::Pages::MigrateLegacyStorageToDeploymentService).not_to receive(:new) expect(::Pages::MigrateLegacyStorageToDeploymentService).not_to receive(:new)
...@@ -11,33 +13,35 @@ RSpec.describe Pages::MigrateFromLegacyStorageService do ...@@ -11,33 +13,35 @@ RSpec.describe Pages::MigrateFromLegacyStorageService do
expect(service.execute).to eq(migrated: 0, errored: 0) expect(service.execute).to eq(migrated: 0, errored: 0)
end end
it 'uses multiple threads' do context 'when there is work for multiple threads' do
projects = create_list(:project, 20) let(:batch_size) { 2 } # override to force usage of multiple threads
projects.each do |project|
project.mark_pages_as_deployed it 'uses multiple threads' do
projects = create_list(:project, 20)
projects.each do |project|
project.mark_pages_as_deployed
FileUtils.mkdir_p File.join(project.pages_path, "public") FileUtils.mkdir_p File.join(project.pages_path, "public")
File.open(File.join(project.pages_path, "public/index.html"), "w") do |f| File.open(File.join(project.pages_path, "public/index.html"), "w") do |f|
f.write("Hello!") f.write("Hello!")
end
end end
end
service = described_class.new(Rails.logger, migration_threads: 3, batch_size: 2, ignore_invalid_entries: false) threads = Concurrent::Set.new
threads = Concurrent::Set.new expect(service).to receive(:migrate_project).exactly(20).times.and_wrap_original do |m, *args|
threads.add(Thread.current)
expect(service).to receive(:migrate_project).exactly(20).times.and_wrap_original do |m, *args| # sleep to be 100% certain that once thread can't consume all the queue
threads.add(Thread.current) # it works without it, but I want to avoid making this test flaky
sleep(0.01)
# sleep to be 100% certain that once thread can't consume all the queue m.call(*args)
# it works without it, but I want to avoid making this test flaky end
sleep(0.01)
m.call(*args) expect(service.execute).to eq(migrated: 20, errored: 0)
expect(threads.length).to eq(3)
end end
expect(service.execute).to eq(migrated: 20, errored: 0)
expect(threads.length).to eq(3)
end end
context 'when pages are marked as deployed' do context 'when pages are marked as deployed' do
...@@ -48,12 +52,24 @@ RSpec.describe Pages::MigrateFromLegacyStorageService do ...@@ -48,12 +52,24 @@ RSpec.describe Pages::MigrateFromLegacyStorageService do
end end
context 'when pages directory does not exist' do context 'when pages directory does not exist' do
it 'counts project as migrated' do context 'when mark_projects_as_not_deployed is set' do
expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project, ignore_invalid_entries: false) do |service| let(:mark_projects_as_not_deployed) { true }
it 'counts project as migrated' do
expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project, ignore_invalid_entries: false, mark_projects_as_not_deployed: true) do |service|
expect(service).to receive(:execute).and_call_original
end
expect(service.execute).to eq(migrated: 1, errored: 0)
end
end
it 'counts project as errored' do
expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project, ignore_invalid_entries: false, mark_projects_as_not_deployed: false) do |service|
expect(service).to receive(:execute).and_call_original expect(service).to receive(:execute).and_call_original
end end
expect(service.execute).to eq(migrated: 1, errored: 0) expect(service.execute).to eq(migrated: 0, errored: 1)
end end
end end
...@@ -66,7 +82,7 @@ RSpec.describe Pages::MigrateFromLegacyStorageService do ...@@ -66,7 +82,7 @@ RSpec.describe Pages::MigrateFromLegacyStorageService do
end end
it 'migrates pages projects without deployments' do it 'migrates pages projects without deployments' do
expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project, ignore_invalid_entries: false) do |service| expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project, ignore_invalid_entries: false, mark_projects_as_not_deployed: false) do |service|
expect(service).to receive(:execute).and_call_original expect(service).to receive(:execute).and_call_original
end end
......
...@@ -11,47 +11,47 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do ...@@ -11,47 +11,47 @@ RSpec.describe Pages::MigrateLegacyStorageToDeploymentService do
expect(zip_service).to receive(:execute).and_call_original expect(zip_service).to receive(:execute).and_call_original
end end
expect(described_class.new(project, ignore_invalid_entries: true).execute[:status]).to eq(:success) expect(described_class.new(project, ignore_invalid_entries: true).execute[:status]).to eq(:error)
end end
it 'marks pages as not deployed if public directory is absent' do context 'when mark_projects_as_not_deployed is passed' do
project.mark_pages_as_deployed let(:service) { described_class.new(project, mark_projects_as_not_deployed: true) }
expect(project.pages_metadatum.reload.deployed).to eq(true) it 'marks pages as not deployed if public directory is absent and invalid entries are ignored' do
project.mark_pages_as_deployed
expect(project.pages_metadatum.reload.deployed).to eq(true)
expect(service.execute).to( expect(service.execute).to(
eq(status: :success, eq(status: :success,
message: "Archive not created. Missing public directory in #{project.pages_path} ? Marked project as not deployed") message: "Archive not created. Missing public directory in #{project.pages_path}? Marked project as not deployed")
) )
expect(project.pages_metadatum.reload.deployed).to eq(false) expect(project.pages_metadatum.reload.deployed).to eq(false)
end end
it 'does not mark pages as not deployed if public directory is absent but pages_deployment exists' do
deployment = create(:pages_deployment, project: project)
project.update_pages_deployment!(deployment)
project.mark_pages_as_deployed
expect(project.pages_metadatum.reload.deployed).to eq(true) it 'does not mark pages as not deployed if public directory is absent but pages_deployment exists' do
deployment = create(:pages_deployment, project: project)
project.update_pages_deployment!(deployment)
project.mark_pages_as_deployed
expect(project.pages_metadatum.reload.deployed).to eq(true)
expect(service.execute).to( expect(service.execute).to(
eq(status: :success, eq(status: :success,
message: "Archive not created. Missing public directory in #{project.pages_path} ? Marked project as not deployed") message: "Archive not created. Missing public directory in #{project.pages_path}? Marked project as not deployed")
) )
expect(project.pages_metadatum.reload.deployed).to eq(true) expect(project.pages_metadatum.reload.deployed).to eq(true)
end
end end
it 'does not mark pages as not deployed if public directory is absent but feature is disabled' do it 'does not mark pages as not deployed if public directory is absent but invalid entries are not ignored' do
stub_feature_flags(pages_migration_mark_as_not_deployed: false)
project.mark_pages_as_deployed project.mark_pages_as_deployed
expect(project.pages_metadatum.reload.deployed).to eq(true) expect(project.pages_metadatum.reload.deployed).to eq(true)
expect(service.execute).to( expect(service.execute).to(
eq(status: :error, eq(status: :error,
message: "Can't create zip archive: Can not find valid public dir in #{project.pages_path}") message: "Archive not created. Missing public directory in #{project.pages_path}")
) )
expect(project.pages_metadatum.reload.deployed).to eq(true) expect(project.pages_metadatum.reload.deployed).to eq(true)
......
...@@ -33,15 +33,6 @@ RSpec.describe Pages::ZipDirectoryService do ...@@ -33,15 +33,6 @@ RSpec.describe Pages::ZipDirectoryService do
expect(archive).to be_nil expect(archive).to be_nil
expect(entries_count).to be_nil expect(entries_count).to be_nil
end end
it 'returns error if pages_migration_mark_as_not_deployed is disabled' do
stub_feature_flags(pages_migration_mark_as_not_deployed: false)
expect(status).to eq(:error)
expect(message).to eq("Can not find valid public dir in #{service_directory}")
expect(archive).to be_nil
expect(entries_count).to be_nil
end
end end
context "when work direcotry doesn't exist" do context "when work direcotry doesn't exist" do
......
...@@ -14,7 +14,8 @@ RSpec.describe 'gitlab:pages' do ...@@ -14,7 +14,8 @@ RSpec.describe 'gitlab:pages' do
expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything, expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
migration_threads: 3, migration_threads: 3,
batch_size: 10, batch_size: 10,
ignore_invalid_entries: false) do |service| ignore_invalid_entries: false,
mark_projects_as_not_deployed: false) do |service|
expect(service).to receive(:execute).and_call_original expect(service).to receive(:execute).and_call_original
end end
...@@ -27,7 +28,8 @@ RSpec.describe 'gitlab:pages' do ...@@ -27,7 +28,8 @@ RSpec.describe 'gitlab:pages' do
expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything, expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
migration_threads: 5, migration_threads: 5,
batch_size: 10, batch_size: 10,
ignore_invalid_entries: false) do |service| ignore_invalid_entries: false,
mark_projects_as_not_deployed: false) do |service|
expect(service).to receive(:execute).and_call_original expect(service).to receive(:execute).and_call_original
end end
...@@ -40,7 +42,8 @@ RSpec.describe 'gitlab:pages' do ...@@ -40,7 +42,8 @@ RSpec.describe 'gitlab:pages' do
expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything, expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
migration_threads: 3, migration_threads: 3,
batch_size: 100, batch_size: 100,
ignore_invalid_entries: false) do |service| ignore_invalid_entries: false,
mark_projects_as_not_deployed: false) do |service|
expect(service).to receive(:execute).and_call_original expect(service).to receive(:execute).and_call_original
end end
...@@ -53,7 +56,22 @@ RSpec.describe 'gitlab:pages' do ...@@ -53,7 +56,22 @@ RSpec.describe 'gitlab:pages' do
expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything, expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
migration_threads: 3, migration_threads: 3,
batch_size: 10, batch_size: 10,
ignore_invalid_entries: true) do |service| ignore_invalid_entries: true,
mark_projects_as_not_deployed: false) do |service|
expect(service).to receive(:execute).and_call_original
end
subject
end
it 'uses PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED environment variable' do
stub_env('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'true')
expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
migration_threads: 3,
batch_size: 10,
ignore_invalid_entries: false,
mark_projects_as_not_deployed: true) do |service|
expect(service).to receive(:execute).and_call_original expect(service).to receive(:execute).and_call_original
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment