Commit 05a2ef69 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'tc-clean-pending-delete-projects' into 'master'

Post-deploy to clean `pending_delete` projects

Closes #20984

See merge request !11044
parents 0f576b94 37a79409
# Worker to destroy projects that do not have a namespace
#
# It destroys everything it can without having the info about the namespace it
# used to belong to. Projects in this state should be rare.
# The worker will reject doing anything for projects that *do* have a
# namespace. For those use ProjectDestroyWorker instead.
class NamespacelessProjectDestroyWorker
include Sidekiq::Worker
include DedicatedSidekiqQueue
def self.bulk_perform_async(args_list)
Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list)
end
def perform(project_id)
begin
project = Project.unscoped.find(project_id)
rescue ActiveRecord::RecordNotFound
return
end
return unless project.namespace_id.nil? # Reject doing anything for projects that *do* have a namespace
project.team.truncate
unlink_fork(project) if project.forked?
# Override Project#remove_pages for this instance so it doesn't do anything
def project.remove_pages
end
project.destroy!
end
private
def unlink_fork(project)
merge_requests = project.forked_from_project.merge_requests.opened.from_project(project)
merge_requests.update_all(state: 'closed')
project.forked_project_link.destroy
end
end
---
title: Add post-deploy migration to clean up projects in `pending_delete` state
merge_request: 11044
author:
...@@ -40,6 +40,7 @@ ...@@ -40,6 +40,7 @@
- [expire_build_instance_artifacts, 1] - [expire_build_instance_artifacts, 1]
- [group_destroy, 1] - [group_destroy, 1]
- [irker, 1] - [irker, 1]
- [namespaceless_project_destroy, 1]
- [project_cache, 1] - [project_cache, 1]
- [project_destroy, 1] - [project_destroy, 1]
- [project_export, 1] - [project_export, 1]
......
# This is the counterpart of RequeuePendingDeleteProjects and cleans all
# projects with `pending_delete = true` and that do not have a namespace.
class CleanupNamespacelessPendingDeleteProjects < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
@offset = 0
loop do
ids = pending_delete_batch
break if ids.empty?
args = ids.map { |id| Array(id) }
NamespacelessProjectDestroyWorker.bulk_perform_async(args)
@offset += 1
end
end
def down
# noop
end
private
def pending_delete_batch
connection.exec_query(find_batch).map{ |row| row['id'] }
end
BATCH_SIZE = 5000
def find_batch
projects = Arel::Table.new(:projects)
projects.project(projects[:id]).
where(projects[:pending_delete].eq(true)).
where(projects[:namespace_id].eq(nil)).
skip(@offset * BATCH_SIZE).
take(BATCH_SIZE).
to_sql
end
end
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170502101023_cleanup_namespaceless_pending_delete_projects.rb')
describe CleanupNamespacelessPendingDeleteProjects do
before do
# Stub after_save callbacks that will fail when Project has no namespace
allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(nil)
allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil)
end
describe '#up' do
it 'only cleans up pending delete projects' do
create(:empty_project)
create(:empty_project, pending_delete: true)
project = build(:empty_project, pending_delete: true, namespace_id: nil)
project.save(validate: false)
expect(NamespacelessProjectDestroyWorker).to receive(:bulk_perform_async).with([[project.id.to_s]])
described_class.new.up
end
it 'does nothing when no pending delete projects without namespace found' do
create(:empty_project)
create(:empty_project, pending_delete: true)
expect(NamespacelessProjectDestroyWorker).not_to receive(:bulk_perform_async)
described_class.new.up
end
end
end
require 'spec_helper'
describe NamespacelessProjectDestroyWorker do
subject { described_class.new }
before do
# Stub after_save callbacks that will fail when Project has no namespace
allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(nil)
allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil)
end
describe '#perform' do
context 'project has namespace' do
it 'does not do anything' do
project = create(:empty_project)
subject.perform(project.id)
expect(Project.unscoped.all).to include(project)
end
end
context 'project has no namespace' do
let!(:project) do
project = build(:empty_project, namespace_id: nil)
project.save(validate: false)
project
end
context 'project not a fork of another project' do
it "truncates the project's team" do
expect_any_instance_of(ProjectTeam).to receive(:truncate)
subject.perform(project.id)
end
it 'deletes the project' do
subject.perform(project.id)
expect(Project.unscoped.all).not_to include(project)
end
it 'does not call unlink_fork' do
is_expected.not_to receive(:unlink_fork)
subject.perform(project.id)
end
it 'does not do anything in Project#remove_pages method' do
expect(Gitlab::PagesTransfer).not_to receive(:new)
subject.perform(project.id)
end
end
context 'project forked from another' do
let!(:parent_project) { create(:empty_project) }
before do
create(:forked_project_link, forked_to_project: project, forked_from_project: parent_project)
end
it 'closes open merge requests' do
merge_request = create(:merge_request, source_project: project, target_project: parent_project)
subject.perform(project.id)
expect(merge_request.reload).to be_closed
end
it 'destroys the link' do
subject.perform(project.id)
expect(parent_project.forked_project_links).to be_empty
end
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment