Commit d4437eee authored by Douwe Maan's avatar Douwe Maan

Merge branch 'mirror-scheduler-fix' into 'master'

Fixes scheduling issue related to all mirrors

See merge request !1183
parents cc12bb81 ea293f2f
...@@ -27,7 +27,7 @@ class UpdateAllMirrorsWorker ...@@ -27,7 +27,7 @@ class UpdateAllMirrorsWorker
private private
def mirrors_to_sync def mirrors_to_sync
Project.mirror.where(sync_time: Gitlab::Mirror.sync_times) Project.mirror.where("mirror_last_successful_update_at + #{Gitlab::Database.minute_interval('sync_time')} <= ? OR sync_time IN (?)", DateTime.now, Gitlab::Mirror.sync_times)
end end
def try_obtain_lease def try_obtain_lease
......
...@@ -17,6 +17,6 @@ class UpdateAllRemoteMirrorsWorker ...@@ -17,6 +17,6 @@ class UpdateAllRemoteMirrorsWorker
private private
def remote_mirrors_to_sync def remote_mirrors_to_sync
RemoteMirror.where(sync_time: Gitlab::Mirror.sync_times) RemoteMirror.where("last_successful_update_at + #{Gitlab::Database.minute_interval('sync_time')} <= ? OR sync_time IN (?)", DateTime.now, Gitlab::Mirror.sync_times)
end end
end end
---
title: Try to update mirrors again after 15 minutes if the previous update failed
merge_request: 1183
author:
class AddIndexToMirrorsLastUpdateAtFields < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def change
add_concurrent_index :projects, :mirror_last_successful_update_at
add_concurrent_index :remote_mirrors, :last_successful_update_at
end
end
...@@ -1127,6 +1127,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do ...@@ -1127,6 +1127,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do
add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree
add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree
add_index "projects", ["mirror_last_successful_update_at"], name: "index_projects_on_mirror_last_successful_update_at", using: :btree
add_index "projects", ["name"], name: "index_projects_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"} add_index "projects", ["name"], name: "index_projects_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
add_index "projects", ["namespace_id"], name: "index_projects_on_namespace_id", using: :btree add_index "projects", ["namespace_id"], name: "index_projects_on_namespace_id", using: :btree
add_index "projects", ["path"], name: "index_projects_on_path", using: :btree add_index "projects", ["path"], name: "index_projects_on_path", using: :btree
...@@ -1216,6 +1217,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do ...@@ -1216,6 +1217,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do
t.integer "sync_time", default: 60, null: false t.integer "sync_time", default: 60, null: false
end end
add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree
add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
add_index "remote_mirrors", ["sync_time"], name: "index_remote_mirrors_on_sync_time", using: :btree add_index "remote_mirrors", ["sync_time"], name: "index_remote_mirrors_on_sync_time", using: :btree
......
...@@ -24,7 +24,7 @@ module Gitlab ...@@ -24,7 +24,7 @@ module Gitlab
def self.nulls_last_order(field, direction = 'ASC') def self.nulls_last_order(field, direction = 'ASC')
order = "#{field} #{direction}" order = "#{field} #{direction}"
if Gitlab::Database.postgresql? if postgresql?
order << ' NULLS LAST' order << ' NULLS LAST'
else else
# `field IS NULL` will be `0` for non-NULL columns and `1` for NULL # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL
...@@ -38,7 +38,7 @@ module Gitlab ...@@ -38,7 +38,7 @@ module Gitlab
def self.nulls_first_order(field, direction = 'ASC') def self.nulls_first_order(field, direction = 'ASC')
order = "#{field} #{direction}" order = "#{field} #{direction}"
if Gitlab::Database.postgresql? if postgresql?
order << ' NULLS FIRST' order << ' NULLS FIRST'
else else
# `field IS NULL` will be `0` for non-NULL columns and `1` for NULL # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL
...@@ -50,7 +50,11 @@ module Gitlab ...@@ -50,7 +50,11 @@ module Gitlab
end end
def self.random def self.random
Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" postgresql? ? "RANDOM()" : "RAND()"
end
def self.minute_interval(value)
postgresql? ? "#{value} * '1 minute'::interval" : "INTERVAL #{value} MINUTE"
end end
def true_value def true_value
......
require 'rails_helper' require 'rails_helper'
describe UpdateAllMirrorsWorker do describe UpdateAllMirrorsWorker do
before do before { allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true) }
allow_any_instance_of(Gitlab::ExclusiveLease)
.to receive(:try_obtain).and_return(true)
end
describe '#perform' do describe '#perform' do
project_count_with_time = { DateTime.now.beginning_of_hour + 15.minutes => 1, let(:worker) { described_class.new }
DateTime.now.beginning_of_hour => 2,
DateTime.now.beginning_of_day => 3
}
let!(:mirror1) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::FIFTEEN) } let!(:mirror1) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::FIFTEEN) }
let!(:mirror2) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::HOURLY) } let!(:mirror2) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::HOURLY) }
let!(:mirror3) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::DAILY) } let!(:mirror3) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::DAILY) }
let(:mirrors) { Project.mirror.where(sync_time: Gitlab::Mirror.sync_times) } let!(:mirror4) { create(:empty_project, :mirror) }
it 'fails stuck mirrors' do it 'fails stuck mirrors' do
worker = described_class.new
expect(worker).to receive(:fail_stuck_mirrors!) expect(worker).to receive(:fail_stuck_mirrors!)
worker.perform worker.perform
end end
project_count_with_time.each do |time, project_count| describe 'sync_time' do
describe "at #{time}" do def expect_worker_to_update_mirrors(mirrors)
before do
allow(DateTime).to receive(:now).and_return(time)
end
it 'enqueues a job on mirrored Projects' do
worker = described_class.new
expect(mirrors.count).to eq(project_count)
mirrors.each do |mirror| mirrors.each do |mirror|
expect(worker).to receive(:rand).with((mirror.sync_time / 2).minutes).and_return(mirror.sync_time / 2) expect(worker).to receive(:rand).with((mirror.sync_time / 2).minutes).and_return(mirror.sync_time / 2)
expect(RepositoryUpdateMirrorDispatchWorker).to receive(:perform_in).with(mirror.sync_time / 2, mirror.id) expect(RepositoryUpdateMirrorDispatchWorker).to receive(:perform_in).with(mirror.sync_time / 2, mirror.id)
end end
end
def setup(time)
Timecop.freeze(time)
mirror4.update_attributes(mirror_last_successful_update_at: time - (Gitlab::Mirror::DAILY + 5).minutes)
end
describe 'fifteen' do
let(:mirrors) { [mirror1, mirror4] }
before { setup(DateTime.now.beginning_of_hour + 15.minutes) }
it 'enqueues a job on mirrored projects' do
expect_worker_to_update_mirrors(mirrors)
worker.perform worker.perform
end end
end end
describe 'hourly' do
let(:mirrors) { [mirror1, mirror2, mirror4] }
before { setup(DateTime.now.beginning_of_hour) }
it 'enqueues a job on mirrored projects' do
expect_worker_to_update_mirrors(mirrors)
worker.perform
end
end
describe 'daily' do
let(:mirrors) { [mirror1, mirror2, mirror3, mirror4] }
before { setup(DateTime.now.beginning_of_day) }
it 'enqueues a job on mirrored projects' do
expect_worker_to_update_mirrors(mirrors)
worker.perform
end
end
after { Timecop.return }
end end
it 'does not execute if cannot get the lease' do it 'does not execute if cannot get the lease' do
allow_any_instance_of(Gitlab::ExclusiveLease) allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(false)
.to receive(:try_obtain).and_return(false)
worker = described_class.new
create(:empty_project, :mirror) create(:empty_project, :mirror)
expect(worker).not_to receive(:fail_stuck_mirrors!) expect(worker).not_to receive(:fail_stuck_mirrors!)
......
require 'rails_helper' require 'rails_helper'
describe UpdateAllRemoteMirrorsWorker do describe UpdateAllRemoteMirrorsWorker do
describe "#perform" do let(:worker) { described_class.new }
project_count_with_time = { DateTime.now.beginning_of_hour + 15.minutes => 1,
DateTime.now.beginning_of_hour => 2,
DateTime.now.beginning_of_day => 3
}
describe "#perform" do
let!(:mirror1) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::FIFTEEN) } let!(:mirror1) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::FIFTEEN) }
let!(:mirror2) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::HOURLY) } let!(:mirror2) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::HOURLY) }
let!(:mirror3) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::DAILY) } let!(:mirror3) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::DAILY) }
let(:mirrors) { RemoteMirror.where(sync_time: Gitlab::Mirror.sync_times) } let!(:mirror4) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::DAILY) }
it 'fails stuck mirrors' do it 'fails stuck mirrors' do
worker = described_class.new
expect(worker).to receive(:fail_stuck_mirrors!) expect(worker).to receive(:fail_stuck_mirrors!)
worker.perform worker.perform
end end
project_count_with_time.each do |time, project_count| describe 'sync time' do
describe "at #{time}" do def expect_worker_to_update_mirrors(mirrors)
before do mirrors.each do |mirror|
allow(DateTime).to receive(:now).and_return(time) expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(mirror.id)
end
end end
it 'enqueues a job on mirrored Projects' do def setup(time)
worker = described_class.new Timecop.freeze(time)
mirror4.remote_mirrors.first.update_attributes(last_successful_update_at: time - (Gitlab::Mirror::DAILY + 5).minutes)
end
expect(mirrors.count).to eq(project_count) describe 'fifteen' do
mirrors.each do |mirror| let(:mirrors) { [mirror1, mirror4] }
expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(mirror.id)
before { setup(DateTime.now.beginning_of_hour + 15.minutes) }
it 'enqueues a job on remote mirrored projects' do
expect_worker_to_update_mirrors(mirrors)
worker.perform
end end
end
describe "hourly" do
let(:mirrors) { [mirror1, mirror2, mirror4] }
before { setup(DateTime.now.beginning_of_hour) }
it 'enqueues a job on remote mirrored projects' do
expect_worker_to_update_mirrors(mirrors)
worker.perform
end
end
describe "daily" do
let(:mirrors) { [mirror1, mirror2, mirror3, mirror4] }
before { setup(DateTime.now.beginning_of_day) }
it 'enqueues a job on remote mirrored projects' do
expect_worker_to_update_mirrors(mirrors)
worker.perform worker.perform
end end
end end
after { Timecop.return }
end end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment