Commit 5a97763e authored by Sean McGivern's avatar Sean McGivern

Merge branch 'ee-29289-project-destroy-clean-up-after-failure' into 'master'

Ee 29289 project destroy clean up after failure

See merge request !2473
parents e61da3ad 7d96f1fd
......@@ -4,10 +4,14 @@ module EE
def execute
raise NotImplementedError unless defined?(super)
super
succeeded = super
mirror_cleanup(project)
log_geo_event(project)
if succeeded
mirror_cleanup(project)
log_geo_event(project)
end
succeeded
end
def mirror_cleanup(project)
......@@ -29,7 +33,7 @@ module EE
# Flush the cache for both repositories. This has to be done _before_
# removing the physical repositories as some expiration code depends on
# Git data (e.g. a list of branch names).
flush_caches(project, wiki_path)
flush_caches(project)
trash_repositories!
remove_tracking_entries!
......
......@@ -16,38 +16,26 @@ module Projects
def execute
return false unless can?(current_user, :remove_project, project)
repo_path = project.path_with_namespace
wiki_path = repo_path + '.wiki'
# Flush the cache for both repositories. This has to be done _before_
# removing the physical repositories as some expiration code depends on
# Git data (e.g. a list of branch names).
flush_caches(project, wiki_path)
flush_caches(project)
Projects::UnlinkForkService.new(project, current_user).execute
Project.transaction do
project.team.truncate
project.destroy!
trash_repositories!
attempt_destroy_transaction(project)
unless remove_legacy_registry_tags
raise_error('Failed to remove some tags in project container registry. Please try again or contact administrator.')
end
unless remove_repository(repo_path)
raise_error('Failed to remove project repository. Please try again or contact administrator.')
end
unless remove_repository(wiki_path)
raise_error('Failed to remove wiki repository. Please try again or contact administrator.')
end
end
log_info("Project \"#{project.path_with_namespace}\" was removed")
system_hook_service.execute_hooks_for(project, :destroy)
log_info("Project \"#{project.full_path}\" was removed")
true
rescue => error
attempt_rollback(project, error.message)
false
rescue Exception => error # rubocop:disable Lint/RescueException
# Project.transaction can raise Exception
attempt_rollback(project, error.message)
raise
end
private
......@@ -91,6 +79,26 @@ module Projects
end
end
def attempt_rollback(project, message)
return unless project
project.update_attributes(delete_error: message, pending_delete: false)
log_error("Deletion failed on #{project.full_path} with the following message: #{message}")
end
def attempt_destroy_transaction(project)
Project.transaction do
unless remove_legacy_registry_tags
raise_error('Failed to remove some tags in project container registry. Please try again or contact administrator.')
end
trash_repositories!
project.team.truncate
project.destroy!
end
end
##
# This method makes sure that we correctly remove registry tags
# for legacy image repository (when repository path equals project path).
......@@ -117,7 +125,7 @@ module Projects
"#{path}+#{project.id}#{DELETED_FLAG}"
end
def flush_caches(project, wiki_path)
def flush_caches(project)
project.repository.before_delete
Repository.new(wiki_path, project).before_delete
......
- project = local_assigns.fetch(:project)
- return unless project.delete_error.present?
.project-deletion-failed-message.alert.alert-warning
This project was scheduled for deletion, but failed with the following message:
= project.delete_error
- project = local_assigns.fetch(:project)
- flash_message_container = show_new_nav? ? :new_global_flash : :flash_message
= content_for flash_message_container do
= render partial: 'deletion_failed', locals: { project: project }
- if current_user && can?(current_user, :download_code, project)
= render 'shared/no_ssh'
= render 'shared/no_password'
= render 'shared/shared_runners_minutes_limit', project: project
- if project.above_size_limit?
= render 'above_size_limit_warning'
- @no_container = true
- flash_message_container = show_new_nav? ? :new_global_flash : :flash_message
= content_for flash_message_container do
- if current_user && can?(current_user, :download_code, @project)
= render 'shared/no_ssh'
= render 'shared/no_password'
= render 'shared/shared_runners_minutes_limit', project: @project
= render partial: 'flash_messages', locals: { project: @project }
= render "projects/head"
= render "home_panel"
......
- @no_container = true
- breadcrumb_title "Project"
- @content_class = "limit-container-width" unless fluid_layout
- flash_message_container = show_new_nav? ? :new_global_flash : :flash_message
= content_for :meta_tags do
= auto_discovery_link_tag(:atom, project_path(@project, rss_url_options), title: "#{@project.name} activity")
= content_for flash_message_container do
- if current_user && can?(current_user, :download_code, @project)
= render 'shared/no_ssh'
= render 'shared/no_password'
= render 'shared/shared_runners_minutes_limit', project: @project
- if @project.above_size_limit?
= render 'above_size_limit_warning'
= render partial: 'flash_messages', locals: { project: @project }
= render "projects/head"
= render "projects/last_push"
......
......@@ -3,14 +3,11 @@ class ProjectDestroyWorker
include DedicatedSidekiqQueue
def perform(project_id, user_id, params)
begin
project = Project.unscoped.find(project_id)
rescue ActiveRecord::RecordNotFound
return
end
project = Project.find(project_id)
user = User.find(user_id)
::Projects::DestroyService.new(project, user, params.symbolize_keys).execute
rescue ActiveRecord::RecordNotFound => error
logger.error("Failed to delete project (#{project_id}): #{error.message}")
end
end
---
title: Handle errors while a project is being deleted asynchronously.
merge_request: 11088
author:
class AddColumnDeleteErrorToProjects < ActiveRecord::Migration
DOWNTIME = false
def change
add_column :projects, :delete_error, :text
end
end
......@@ -1388,6 +1388,7 @@ ActiveRecord::Schema.define(version: 20170719182937) do
t.datetime "last_repository_updated_at"
t.string "ci_config_path"
t.boolean "disable_overriding_approvers_per_merge_request"
t.text "delete_error"
end
add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
......
require 'spec_helper'
describe 'Project show page', feature: true do
context 'when project pending delete' do
let(:project) { create(:project, :empty_repo, pending_delete: true) }
before do
sign_in(project.owner)
end
it 'shows flash error if deletion for project fails' do
project.update_attributes(delete_error: "Something went wrong", pending_delete: false)
visit project_path(project)
expect(page).to have_selector('.project-deletion-failed-message')
expect(page).to have_content("This project was scheduled for deletion, but failed with the following message: #{project.delete_error}")
end
end
end
......@@ -412,6 +412,7 @@ Project:
- service_desk_enabled
- last_repository_updated_at
- ci_config_path
- delete_error
Author:
- name
ProjectFeature:
......
......@@ -40,5 +40,14 @@ describe Projects::DestroyService, services: true do
expect { subject.execute }.to change(Geo::RepositoryDeletedEvent, :count).by(1)
end
end
it 'does not log event to the Geo log if project deletion fails' do
expect_any_instance_of(Project)
.to receive(:destroy!).and_raise(StandardError.new('Other error message'))
Sidekiq::Testing.inline! do
expect { subject.execute }.not_to change(Geo::RepositoryDeletedEvent, :count)
end
end
end
end
......@@ -36,6 +36,27 @@ describe Projects::DestroyService, services: true do
end
end
shared_examples 'handles errors thrown during async destroy' do |error_message|
it 'does not allow the error to bubble up' do
expect do
Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
end.not_to raise_error
end
it 'unmarks the project as "pending deletion"' do
Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
expect(project.reload.pending_delete).to be(false)
end
it 'stores an error message in `projects.delete_error`' do
Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
expect(project.reload.delete_error).to be_present
expect(project.delete_error).to include(error_message)
end
end
context 'Sidekiq inline' do
before do
# Run sidekiq immediatly to check that renamed repository will be removed
......@@ -102,10 +123,51 @@ describe Projects::DestroyService, services: true do
end
it_behaves_like 'deleting the project with pipeline and build'
end
context 'with execute' do
it_behaves_like 'deleting the project with pipeline and build'
context 'errors' do
context 'when `remove_legacy_registry_tags` fails' do
before do
expect_any_instance_of(Projects::DestroyService)
.to receive(:remove_legacy_registry_tags).and_return(false)
end
it_behaves_like 'handles errors thrown during async destroy', "Failed to remove some tags"
end
context 'when `remove_repository` fails' do
before do
expect_any_instance_of(Projects::DestroyService)
.to receive(:remove_repository).and_return(false)
end
it_behaves_like 'handles errors thrown during async destroy', "Failed to remove project repository"
end
context 'when `execute` raises any other error' do
before do
expect_any_instance_of(Project)
.to receive(:destroy!).and_raise(StandardError.new("Other error message"))
end
it_behaves_like 'handles errors thrown during async destroy', "Other error message"
end
context 'when `execute` raises unexpected error' do
before do
expect_any_instance_of(Project)
.to receive(:destroy!).and_raise(Exception.new("Other error message"))
end
it 'allows error to bubble up and rolls back project deletion' do
expect do
Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
end.to raise_error
expect(project.reload.pending_delete).to be(false)
expect(project.delete_error).to include("Other error message")
end
end
end
end
describe 'container registry' do
......@@ -132,8 +194,7 @@ describe Projects::DestroyService, services: true do
expect_any_instance_of(ContainerRepository)
.to receive(:delete_tags!).and_return(false)
expect{ destroy_project(project, user) }
.to raise_error(ActiveRecord::RecordNotDestroyed)
expect(destroy_project(project, user)).to be false
end
end
end
......@@ -158,8 +219,7 @@ describe Projects::DestroyService, services: true do
expect_any_instance_of(ContainerRepository)
.to receive(:delete_tags!).and_return(false)
expect { destroy_project(project, user) }
.to raise_error(Projects::DestroyService::DestroyError)
expect(destroy_project(project, user)).to be false
end
end
end
......
require 'spec_helper'
describe ProjectDestroyWorker do
let(:project) { create(:project, :repository) }
let(:project) { create(:project, :repository, pending_delete: true) }
let(:path) { project.repository.path_to_repo }
subject { described_class.new }
describe "#perform" do
it "deletes the project" do
describe '#perform' do
it 'deletes the project' do
subject.perform(project.id, project.owner.id, {})
expect(Project.all).not_to include(project)
expect(Dir.exist?(path)).to be_falsey
end
it "deletes the project but skips repo deletion" do
it 'deletes the project but skips repo deletion' do
subject.perform(project.id, project.owner.id, { "skip_repo" => true })
expect(Project.all).not_to include(project)
expect(Dir.exist?(path)).to be_truthy
end
it 'does not raise error when project could not be found' do
expect do
subject.perform(-1, project.owner.id, {})
end.not_to raise_error
end
it 'does not raise error when user could not be found' do
expect do
subject.perform(project.id, -1, {})
end.not_to raise_error
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment