Commit 333728a4 authored by Tiago Botelho's avatar Tiago Botelho

Removes hard_failed state out of project state machine. Now when the mirror...

Removes hard_failed state out of project state machine. Now when the mirror reaches the retry count limit it will stay as failed but will not get scheduled anymore, and a new message will appear
parent 344f2d16
......@@ -62,7 +62,7 @@ class Projects::MirrorsController < Projects::ApplicationController
@project.update_remote_mirrors
flash[:notice] = "The remote repository is being updated..."
else
force_import_job
@project.force_import_job!
flash[:notice] = "The repository is being updated..."
end
......@@ -71,14 +71,6 @@ class Projects::MirrorsController < Projects::ApplicationController
private
def force_import_job
if @project.hard_failed?
@project.resume
else
@project.force_import_job!
end
end
def remote_mirror
@remote_mirror = @project.remote_mirrors.first_or_initialize
end
......
module MirrorHelper
def mirror_update_failed?
return true if @project.hard_failed?
def render_mirror_failed_message(raw_message:)
mirror_last_update_at = @project.mirror_last_update_at
message = "The repository failed to update #{time_ago_with_tooltip(mirror_last_update_at)}.".html_safe
@project.mirror_last_update_failed?
end
def render_mirror_failed_message(status:, icon:)
message = get_mirror_failed_message(status)
return message unless icon
return message if raw_message
message_with_icon = "#{icon('warning triangle')} #{message}"
return message_with_icon unless can?(current_user, :admin_project, @project)
message.insert(0, "#{icon('warning triangle')} ")
link_to(project_mirror_path(@project)) { "#{icon('warning triangle')} #{message}" }
if can?(current_user, :admin_project, @project)
link_to message, project_mirror_path(@project)
else
message
end
end
def branch_diverged_tooltip_message
......@@ -30,15 +28,4 @@ module MirrorHelper
def options_for_mirror_user
options_from_collection_for_select(default_mirror_users, :id, :name, @project.mirror_user_id || current_user.id)
end
private
def get_mirror_failed_message(status)
if status == :failed
"The repository failed to update #{time_ago_with_tooltip(@project.last_update_at)}."
else
"The repository failed to update #{time_ago_with_tooltip(last_update_at)}.<br>"\
"Repository mirroring has been paused due to too many failed attempts, and can be resumed by a project admin."
end
end
end
......@@ -7,21 +7,19 @@ class ProjectMirrorData < ActiveRecord::Base
validates :project, presence: true
validates :next_execution_timestamp, presence: true
before_validation on: :create do
self.next_execution_timestamp = Time.now
end
before_validation :set_next_execution_to_now, on: :create
def reset_retry_count!
def reset_retry_count
self.retry_count = 0
end
def increment_retry_count!
def increment_retry_count
self.retry_count += 1
end
# We schedule the next sync time based on the duration of the
# last mirroring period and add it a fixed backoff period with a random jitter
def set_next_execution_timestamp!
def set_next_execution_timestamp
timestamp = Time.now
retry_factor = [1, self.retry_count].max
delay = [base_delay(timestamp), Gitlab::Mirror.min_delay].max
......@@ -30,8 +28,8 @@ class ProjectMirrorData < ActiveRecord::Base
self.next_execution_timestamp = timestamp + delay
end
def set_next_execution_to_now!
self.update_attributes(next_execution_timestamp: Time.now)
def set_next_execution_to_now
self.next_execution_timestamp = Time.now
end
def retry_limit_exceeded?
......
- if mirror_update_failed?
- if @project.mirror_last_update_failed?
.panel.panel-danger
.panel-heading
= render partial: 'shared/projects/mirror_status', show_icon: false
= render partial: 'shared/mirror_status', locals: { raw_message: true }
.panel-body
%pre
:preserve
......
= render partial: 'shared/projects/mirror_status'
- last_successful_update_at = @project.mirror_last_successful_update_at
- raw_message = local_assigns.fetch(:raw_message, false)
- case @project.mirror_last_update_status
- when :success
Updated #{time_ago_with_tooltip(last_successful_update_at)}.
- when :failed
= render_mirror_failed_message(raw_message: raw_message)
- if @project.mirror_hard_failed?
%br
Repository mirroring has been paused due to too many failed attempts, and can be resumed by a project admin.
- if @project.mirror_ever_updated_successfully?
%br
Last successful update #{time_ago_with_tooltip(last_successful_update_at)}.
......@@ -17,6 +17,6 @@
= link_to update_now_project_mirror_path(@project), method: :post, class: 'btn' do
= icon("refresh")
Update Now
- if @project.mirror_last_update_success?
- if @project.mirror_last_update_succeeded?
%p.inline.prepend-left-10
Successfully updated #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}.
- last_successful_update_at = @project.mirror_last_successful_update_at
- mirror_last_update_status = @project.mirror_last_update_status
- show_icon = local_assigns.fetch(:show_icon, true)
- if mirror_last_update_status == :success
Updated #{time_ago_with_tooltip(last_successful_update_at)}.
- else
= render_mirror_failed_message(status: mirror_last_update_status, icon: show_icon)
- if @project.mirror_ever_updated_successfully?
%br
Last successful update #{time_ago_with_tooltip(last_successful_update_at)}.
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddCompositeIndexToProjectMirrorDataNextExecutionTimestampAndRetryCount < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'index_mirror_data_on_next_execution_and_retry_count'
disable_ddl_transaction!
def up
add_concurrent_index(:project_mirror_data, [:next_execution_timestamp, :retry_count], name: INDEX_NAME)
end
def down
if index_exists?(:project_mirror_data, [:next_execution_timestamp, :retry_count], name: INDEX_NAME)
remove_concurrent_index(:project_mirror_data, [:next_execution_timestamp, :retry_count], name: INDEX_NAME)
end
end
end
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20171026082505) do
ActiveRecord::Schema.define(version: 20171107090120) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -1570,6 +1570,7 @@ ActiveRecord::Schema.define(version: 20171026082505) do
t.datetime_with_timezone "updated_at"
end
add_index "project_mirror_data", ["next_execution_timestamp", "retry_count"], name: "index_mirror_data_on_next_execution_and_retry_count", using: :btree
add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", unique: true, using: :btree
create_table "project_statistics", force: :cascade do |t|
......
......@@ -98,7 +98,7 @@ pull mirror settings page.
When a project is hard failed, it will no longer get picked up for mirroring.
A user can resume the project mirroring again by either [forcing an update](#forcing-an-update)
or by changing the import URL field in the repository settings.
or by changing the import URL in repository settings.
### SSH authentication
......
module EE
module ProjectsHelper
def can_force_update_mirror?(project)
return true if project.hard_failed?
return true if project.mirror_hard_failed?
return true unless project.mirror_last_update_at
Time.now - project.mirror_last_update_at >= 5.minutes
......
......@@ -42,8 +42,9 @@ module EE
scope :mirror, -> { where(mirror: true) }
scope :mirrors_to_sync, ->(freeze_at) do
mirror.joins(:mirror_data).without_import_status(:hard_failed, :scheduled, :started)
mirror.joins(:mirror_data).without_import_status(:scheduled, :started)
.where("next_execution_timestamp <= ?", freeze_at)
.where("project_mirror_data.retry_count <= ?", ::Gitlab::Mirror::MAX_RETRY)
end
scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
......@@ -105,7 +106,7 @@ module EE
def scheduled_mirror?
return false unless mirror_with_content?
return false if hard_failed?
return false if mirror_hard_failed?
return true if import_scheduled?
self.mirror_data.next_execution_timestamp <= Time.now
......@@ -118,13 +119,14 @@ module EE
def mirror_last_update_status
return unless mirror_updated?
return :success if self.last_mirror_update_succeeded?
return :failed unless self.mirror_data.retry_limit_exceeded?
:hard_failed
if self.mirror_last_update_at == self.mirror_last_successful_update_at
:success
else
:failed
end
end
def mirror_last_update_success?
def mirror_last_update_succeeded?
mirror_last_update_status == :success
end
......@@ -136,8 +138,8 @@ module EE
mirror_updated? && self.mirror_last_successful_update_at
end
def last_mirror_update_succeeded?
self.mirror_last_update_at == self.mirror_last_successful_update_at
def mirror_hard_failed?
self.mirror_data.retry_limit_exceeded?
end
def has_remote_mirror?
......@@ -213,9 +215,12 @@ module EE
end
def force_import_job!
import_resume if hard_failed?
mirror_data = self.mirror_data
mirror_data.set_next_execution_to_now
mirror_data.reset_retry_count if mirror_data.retry_limit_exceeded?
self.mirror_data.set_next_execution_to_now!
mirror_data.save!
UpdateAllMirrorsWorker.perform_async
end
......
......@@ -5,16 +5,6 @@ module EE
included do
state_machine :import_status, initial: :none do
event :import_hard_fail do
transition failed: :hard_failed
end
event :import_resume do
transition hard_failed: :failed
end
state :hard_failed
before_transition [:none, :finished, :failed] => :scheduled do |project, _|
project.mirror_data&.last_update_scheduled_at = Time.now
end
......@@ -25,9 +15,8 @@ module EE
before_transition scheduled: :failed do |project, _|
if project.mirror?
timestamp = Time.now
project.mirror_last_update_at = timestamp
project.mirror_data.next_execution_timestamp = timestamp
project.mirror_last_update_at = Time.now
project.mirror_data.set_next_execution_to_now
end
end
......@@ -40,10 +29,8 @@ module EE
project.mirror_last_update_at = Time.now
mirror_data = project.mirror_data
mirror_data.increment_retry_count!
mirror_data.set_next_execution_timestamp!
project.run_after_commit { import_hard_fail } if mirror_data.retry_limit_exceeded?
mirror_data.increment_retry_count
mirror_data.set_next_execution_timestamp
end
end
......@@ -54,8 +41,8 @@ module EE
project.mirror_last_successful_update_at = timestamp
mirror_data = project.mirror_data
mirror_data.reset_retry_count!
mirror_data.set_next_execution_timestamp!
mirror_data.reset_retry_count
mirror_data.set_next_execution_timestamp
end
if ::Gitlab::CurrentSettings.current_application_settings.elasticsearch_indexing?
......@@ -66,13 +53,6 @@ module EE
end
end
before_transition hard_failed: :failed do |project, _|
if project.mirror?
project.mirror_data.reset_retry_count!
project.force_import_job!
end
end
after_transition [:finished, :failed] => [:scheduled, :started] do |project, _|
::Gitlab::Mirror.increment_capacity(project.id) if project.mirror?
end
......
......@@ -148,26 +148,12 @@ describe Projects::MirrorsController do
describe 'forcing an update on a pull mirror' do
it 'forces update' do
expect_any_instance_of(EE::Project).to receive(:force_import_job!)
project = create(:project, :mirror)
sign_in(project.owner)
put :update_now, { namespace_id: project.namespace.to_param, project_id: project.to_param }
end
context 'when mirror is hard_failed' do
it 'forces update and changes to failed' do
allow(UpdateAllMirrorsWorker).to receive(:perform_async).and_return(nil)
project = create(:project, :mirror, :import_hard_failed)
sign_in(project.owner)
expect_any_instance_of(EE::Project).to receive(:force_import_job!)
expect_any_instance_of(Project).to receive(:import_resume).and_call_original
expect do
put :update_now, { namespace_id: project.namespace.to_param, project_id: project.to_param }
end.to change { project.reload.import_status }.from('hard_failed').to('failed')
end
put :update_now, { namespace_id: project.namespace.to_param, project_id: project.to_param }
end
end
......
......@@ -19,30 +19,58 @@ describe Project do
it { is_expected.to have_many(:audit_events) }
end
describe 'after update' do
context 'when mirror is hard failed and import url changed' do
it 'resumes the mirroring' do
allow(UpdateAllMirrorsWorker).to receive(:perform_async).and_return(nil)
describe '.mirrors_to_sync' do
let(:timestamp) { Time.now }
project = create(:project, :repository, :mirror, :import_hard_failed)
context 'when mirror is scheduled' do
it 'returns empty' do
create(:project, :mirror, :import_scheduled)
expect do
project.update_attributes(import_url: 'http://foo.com')
end.to change(project, :import_status).from('hard_failed').to('failed')
expect(described_class.mirrors_to_sync(timestamp)).to be_empty
end
end
end
describe 'resuming from a hard failed mirror' do
it 'resets retry count and schedules a mirroring worker' do
timestamp = Time.now
project = create(:project, :mirror, :import_hard_failed)
context 'when mirror is started' do
it 'returns empty' do
create(:project, :mirror, :import_scheduled)
expect(UpdateAllMirrorsWorker).to receive(:perform_async)
expect(described_class.mirrors_to_sync(timestamp)).to be_empty
end
end
Timecop.freeze(timestamp) do
expect { project.import_resume }.to change(project.mirror_data, :retry_count).to(0)
expect(project.mirror_data.next_execution_timestamp).to eq(timestamp)
context 'when mirror is finished' do
let!(:project) { create(:project, :mirror, :import_finished) }
it 'returns project if next_execution_timestamp is not in the future' do
expect(described_class.mirrors_to_sync(timestamp)).to match_array(project)
end
it 'returns empty if next_execution_timestamp is in the future' do
project.mirror_data.update_attributes(next_execution_timestamp: timestamp + 2.minutes)
expect(described_class.mirrors_to_sync(timestamp)).to be_empty
end
end
context 'when project is failed' do
let!(:project) { create(:project, :mirror, :import_failed) }
it 'returns project if next_execution_timestamp is not in the future' do
expect(described_class.mirrors_to_sync(timestamp)).to match_array(project)
end
it 'returns empty if next_execution_timestamp is in the future' do
project.mirror_data.update_attributes(next_execution_timestamp: timestamp + 2.minutes)
expect(described_class.mirrors_to_sync(timestamp)).to be_empty
end
context 'with retry limit exceeded' do
let!(:project) { create(:project, :mirror, :import_hard_failed) }
it 'returns empty' do
expect(described_class.mirrors_to_sync(timestamp)).to be_empty
end
end
end
end
......@@ -232,15 +260,26 @@ describe Project do
it 'sets next execution timestamp to now and schedules UpdateAllMirrorsWorker' do
timestamp = Time.now
project = create(:project, :mirror)
project.mirror_data.update_attributes(next_execution_timestamp: timestamp - 3.minutes)
expect(UpdateAllMirrorsWorker).to receive(:perform_async)
Timecop.freeze(timestamp) do
expect { project.force_import_job! }.to change(project.mirror_data, :next_execution_timestamp).to(timestamp)
end
end
expect(UpdateAllMirrorsWorker).to receive(:perform_async)
context 'when mirror is hard failed' do
it 'resets retry count and schedules a mirroring worker' do
timestamp = Time.now
project = create(:project, :mirror, :import_hard_failed)
project.run_callbacks(:commit)
expect(UpdateAllMirrorsWorker).to receive(:perform_async)
Timecop.freeze(timestamp) do
expect { project.force_import_job! }.to change(project.mirror_data, :retry_count).to(0)
expect(project.mirror_data.next_execution_timestamp).to eq(timestamp)
end
end
end
end
......@@ -348,35 +387,27 @@ describe Project do
end
context 'when mirror has updated' do
let(:timestamp) { Time.now }
before do
project.mirror_last_update_at = Time.now
project.mirror_last_update_at = timestamp
end
context 'when last update time equals the time of the last successful update' do
it 'returns success' do
timestamp = Time.now
project.mirror_last_update_at = timestamp
project.mirror_last_successful_update_at = timestamp
expect(project.mirror_last_update_status).to eq(:success)
end
end
context 'when retry count has not reached the limit' do
context 'when last update time does not equal the time of the last successful update' do
it 'returns failed' do
project.mirror_data.retry_count = Gitlab::Mirror::MAX_RETRY
project.mirror_last_successful_update_at = Time.now - 1.minute
expect(project.mirror_last_update_status).to eq(:failed)
end
end
context 'when retry count has reached the limit' do
it 'returns hard_failed' do
project.mirror_data.retry_count = Gitlab::Mirror::MAX_RETRY + 1
expect(project.mirror_last_update_status).to eq(:hard_failed)
end
end
end
end
......
......@@ -78,15 +78,21 @@ FactoryGirl.define do
end
trait :import_finished do
timestamp = Time.now
import_status :finished
mirror_last_update_at timestamp
mirror_last_successful_update_at timestamp
end
trait :import_failed do
import_status :failed
mirror_last_update_at Time.now
end
trait :import_hard_failed do
import_status :hard_failed
import_status :failed
mirror_last_update_at Time.now - 1.minute
after(:create) do |project|
project.mirror_data&.update_attributes(retry_count: Gitlab::Mirror::MAX_RETRY + 1)
......
......@@ -23,36 +23,25 @@ describe ProjectMirrorData, type: :model do
end
end
describe 'when update' do
context 'when retry limit reached' do
it 'marks mirror as hard failed' do
project = create(:project, :mirror, :import_started)
project.mirror_data.update_attributes(retry_count: Gitlab::Mirror::MAX_RETRY)
expect { project.import_fail }.to change(project, :import_status).from('started').to('hard_failed')
end
end
end
describe '#reset_retry_count!' do
describe '#reset_retry_count' do
let(:mirror_data) { create(:project, :mirror, :import_finished).mirror_data }
it 'resets retry_count to 0' do
mirror_data.retry_count = 3
expect { mirror_data.reset_retry_count! }.to change { mirror_data.retry_count }.from(3).to(0)
expect { mirror_data.reset_retry_count }.to change { mirror_data.retry_count }.from(3).to(0)
end
end
describe '#increment_retry_count!' do
describe '#increment_retry_count' do
let(:mirror_data) { create(:project, :mirror, :import_finished).mirror_data }
it 'increments retry_count' do
expect { mirror_data.increment_retry_count! }.to change { mirror_data.retry_count }.from(0).to(1)
expect { mirror_data.increment_retry_count }.to change { mirror_data.retry_count }.from(0).to(1)
end
end
describe '#set_next_execution_timestamp!' do
describe '#set_next_execution_timestamp' do
let(:mirror_data) { create(:project, :mirror, :import_finished).mirror_data }
let!(:timestamp) { Time.now }
let!(:jitter) { 2.seconds }
......@@ -75,7 +64,7 @@ describe ProjectMirrorData, type: :model do
context 'when incrementing retry count' do
it 'applies transition successfully' do
mirror_data.retry_count = 2
mirror_data.increment_retry_count!
mirror_data.increment_retry_count
expect_next_execution_timestamp(mirror_data, timestamp + 78.minutes)
end
......@@ -103,7 +92,7 @@ describe ProjectMirrorData, type: :model do
context 'when incrementing retry count' do
it 'applies transition successfully' do
mirror_data.retry_count = 3
mirror_data.increment_retry_count!
mirror_data.increment_retry_count
expect_next_execution_timestamp(mirror_data, timestamp + 62.minutes)
end
......@@ -126,7 +115,7 @@ describe ProjectMirrorData, type: :model do
context 'when incrementing retry count' do
it 'applies transition successfully' do
mirror_data.retry_count = 2
mirror_data.increment_retry_count!
mirror_data.increment_retry_count
expect_next_execution_timestamp(mirror_data, max_timestamp + mirror_jitter)
end
......@@ -137,7 +126,7 @@ describe ProjectMirrorData, type: :model do
def expect_next_execution_timestamp(mirror_data, new_timestamp)
Timecop.freeze(timestamp) do
expect do
mirror_data.set_next_execution_timestamp!
mirror_data.set_next_execution_timestamp
end.to change { mirror_data.next_execution_timestamp }.to eq(new_timestamp)
end
end
......
require 'spec_helper'
describe 'shared/_mirror_status.html.haml' do
include ApplicationHelper
context 'when mirror has not updated yet' do
it 'does not render anything' do
@project = create(:project, :mirror)
sign_in(@project.owner)
render 'shared/mirror_status'
expect(rendered).to be_empty
end
end
context 'when mirror successful' do
it 'renders success message' do
@project = create(:project, :mirror, :import_finished)
sign_in(@project.owner)
render 'shared/mirror_status'
expect(rendered).to have_content("Updated")
end
end
context 'when mirror failed' do
before do
@project = create(:project, :mirror, :import_failed)
sign_in(@project.owner)
end
it 'renders failure message' do
render 'shared/mirror_status', raw_message: true
expect(rendered).to have_content("The repository failed to update")
end
context 'with a previous successful update' do
it 'renders failure message' do
@project.mirror_last_successful_update_at = Time.now - 1.minute
render 'shared/mirror_status', raw_message: true
expect(rendered).to have_content("Last successful update")
end
end
context 'with a hard failed mirror' do
it 'renders hard failed message' do
@project.mirror_data.retry_count = Gitlab::Mirror::MAX_RETRY + 1
render 'shared/mirror_status', raw_message: true
expect(rendered).to have_content("Repository mirroring has been paused due to too many failed attempts, and can be resumed by a project admin.")
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment