Commit c7616d89 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'global-minimum-mirror-sync-time' into 'master'

adds global minimum mirror sync time option in admin section

See merge request !1217
parents af037c0c b600f20e
......@@ -166,7 +166,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:elasticsearch_search,
:repository_size_limit,
:shared_runners_minutes,
:usage_ping_enabled
:usage_ping_enabled,
:minimum_mirror_sync_time
]
end
end
......@@ -4,4 +4,10 @@ module MirrorHelper
message << "<br>To discard the local changes and overwrite the branch with the upstream version, delete it here and choose 'Update Now' above." if can?(current_user, :push_code, @project)
message
end
def mirror_sync_time_options
Gitlab::Mirror::SYNC_TIME_OPTIONS.select do |key, value|
value >= current_application_settings.minimum_mirror_sync_time
end
end
end
......@@ -128,6 +128,10 @@ class ApplicationSetting < ActiveRecord::Base
presence: true,
numericality: { only_integer: true, greater_than_or_equal_to: 0 }
validates :minimum_mirror_sync_time,
presence: true,
inclusion: { in: Gitlab::Mirror::SYNC_TIME_OPTIONS.values }
validates_each :restricted_visibility_levels do |record, attr, value|
value&.each do |level|
unless Gitlab::VisibilityLevel.options.has_value?(level)
......@@ -155,6 +159,8 @@ class ApplicationSetting < ActiveRecord::Base
before_save :ensure_runners_registration_token
before_save :ensure_health_check_access_token
after_update :update_mirror_cron_jobs, if: :minimum_mirror_sync_time_changed?
after_commit do
Rails.cache.write(CACHE_KEY, self)
end
......@@ -224,7 +230,8 @@ class ApplicationSetting < ActiveRecord::Base
{
elasticsearch_host: ENV['ELASTIC_HOST'] || 'localhost',
elasticsearch_port: ENV['ELASTIC_PORT'] || '9200',
usage_ping_enabled: true
usage_ping_enabled: true,
minimum_mirror_sync_time: Gitlab::Mirror::FIFTEEN
}
end
......@@ -236,6 +243,15 @@ class ApplicationSetting < ActiveRecord::Base
create(defaults)
end
def update_mirror_cron_jobs
Project.mirror.where('sync_time < ?', minimum_mirror_sync_time).
update_all(sync_time: minimum_mirror_sync_time)
RemoteMirror.where('sync_time < ?', minimum_mirror_sync_time).
update_all(sync_time: minimum_mirror_sync_time)
Gitlab::Mirror.configure_cron_jobs!
end
def elasticsearch_host
read_attribute(:elasticsearch_host).split(',').map(&:strip)
end
......
......@@ -218,7 +218,7 @@ class Project < ActiveRecord::Base
validates :sync_time,
presence: true,
inclusion: { in: Gitlab::Mirror.sync_time_options.values }
inclusion: { in: Gitlab::Mirror::SYNC_TIME_OPTIONS.values }
with_options if: :mirror? do |project|
project.validates :import_url, presence: true
......
......@@ -14,7 +14,7 @@ class RemoteMirror < ActiveRecord::Base
validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }
validates :sync_time,
presence: true,
inclusion: { in: Gitlab::Mirror.sync_time_options.values }
inclusion: { in: Gitlab::Mirror::SYNC_TIME_OPTIONS.values }
validate :url_availability, if: -> (mirror) { mirror.url_changed? || mirror.enabled? }
......
......@@ -71,6 +71,10 @@
%span.help-block#repository_size_limit_help_block
Includes LFS objects. It can be overridden per group, or per project. 0 for unlimited.
= link_to icon('question-circle'), help_page_path("user/admin_area/settings/account_and_limit_settings")
.form-group
= f.label :minimum_mirror_sync_time, class: 'control-label col-sm-2'
.col-sm-10
= f.select :minimum_mirror_sync_time, options_for_select(Gitlab::Mirror::SYNC_TIME_OPTIONS, @application_setting.minimum_mirror_sync_time), {}, class: 'form-control'
.form-group
= f.label :session_expire_delay, 'Session duration (minutes)', class: 'control-label col-sm-2'
.col-sm-10
......
......@@ -47,7 +47,7 @@
= render "shared/mirror_trigger_builds_setting", f: f
.form-group
= f.label :sync_time, "Synchronization time", class: "label-light append-bottom-0"
= f.select :sync_time, options_for_select(Gitlab::Mirror.sync_time_options, @project.sync_time), {}, class: 'form-control'
= f.select :sync_time, options_for_select(mirror_sync_time_options, @project.sync_time), {}, class: 'form-control project-mirror-sync-time'
.col-sm-12
%hr
.col-lg-3
......@@ -82,7 +82,7 @@
= render "instructions"
.form-group
= rm_form.label :sync_time, "Synchronization time", class: "label-light append-bottom-0"
= rm_form.select :sync_time, options_for_select(Gitlab::Mirror.sync_time_options, @remote_mirror.sync_time), {}, class: 'form-control'
= rm_form.select :sync_time, options_for_select(mirror_sync_time_options, @remote_mirror.sync_time), {}, class: 'form-control remote-mirror-sync-time'
.col-sm-12.text-center
%hr
= f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
---
title: Add configurable minimum mirror sync time in admin section
merge_request: 1217
author:
......@@ -34,9 +34,7 @@ Sidekiq.configure_server do |config|
end
Sidekiq::Cron::Job.load_from_hash! cron_jobs
# These jobs should not be allowed to be configured in gitlab.yml
Sidekiq::Cron::Job.create(name: 'update_all_remote_mirrors_worker', cron: '*/15 * * * *', class: 'UpdateAllRemoteMirrorsWorker')
Sidekiq::Cron::Job.create(name: 'update_all_mirrors_worker', cron: '*/15 * * * *', class: 'UpdateAllMirrorsWorker')
Gitlab::Mirror.configure_cron_jobs!
# Gitlab Geo: enable bulk notify job only on primary node
Gitlab::Geo.bulk_notify_job.disable! unless Gitlab::Geo.primary?
......
class AddGlobalMinimumMirrorSyncTimeToApplicationSettings < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column_with_default :application_settings,
:minimum_mirror_sync_time,
:integer,
default: 15,
allow_null: false
end
def down
remove_column :application_settings, :minimum_mirror_sync_time
end
end
......@@ -120,6 +120,7 @@ ActiveRecord::Schema.define(version: 20170211073944) do
t.integer "shared_runners_minutes", default: 0, null: false
t.integer "repository_size_limit", limit: 8, default: 0
t.integer "terminal_max_session_time", default: 0, null: false
t.integer "minimum_mirror_sync_time", default: 15, null: false
end
create_table "approvals", force: :cascade do |t|
......
module Gitlab
module Mirror
include Gitlab::CurrentSettings
FIFTEEN = 15
HOURLY = 60
DAILY = 1440
INTERVAL_BEFORE_FIFTEEN = 14.minutes
class << self
def sync_time_options
{
SYNC_TIME_TO_CRON = {
FIFTEEN => "*/15 * * * *",
HOURLY => "0 * * * *",
DAILY => "0 0 * * *"
}.freeze
SYNC_TIME_OPTIONS = {
"Update every 15 minutes" => FIFTEEN,
"Update hourly" => HOURLY,
"Update every day" => DAILY,
}
end
}.freeze
class << self
def sync_times
sync_times = [FIFTEEN]
sync_times << DAILY if at_beginning_of_day?
......@@ -23,6 +29,29 @@ module Gitlab
sync_times
end
def configure_cron_jobs!
minimum_mirror_sync_time = current_application_settings.minimum_mirror_sync_time rescue FIFTEEN
sync_time = SYNC_TIME_TO_CRON[minimum_mirror_sync_time]
update_all_mirrors_worker_job = Sidekiq::Cron::Job.find("update_all_mirrors_worker")
update_all_remote_mirrors_worker_job = Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker")
if update_all_mirrors_worker_job && update_all_remote_mirrors_worker_job
update_all_mirrors_worker_job.destroy
update_all_remote_mirrors_worker_job.destroy
end
Sidekiq::Cron::Job.create(
name: 'update_all_remote_mirrors_worker',
cron: sync_time,
class: 'UpdateAllRemoteMirrorsWorker'
)
Sidekiq::Cron::Job.create(
name: 'update_all_mirrors_worker',
cron: sync_time,
class: 'UpdateAllMirrorsWorker'
)
end
def at_beginning_of_day?
start_at = DateTime.now.at_beginning_of_day
end_at = start_at + INTERVAL_BEFORE_FIFTEEN
......
require 'spec_helper'
describe Projects::MirrorsController do
let(:sync_times) { Gitlab::Mirror.sync_time_options.values }
let(:sync_times) { Gitlab::Mirror::SYNC_TIME_OPTIONS.values }
describe 'setting up a mirror' do
context 'when the current project is a mirror' do
......
......@@ -8,15 +8,54 @@ feature 'Project mirror', feature: true do
before do
project.team << [user, :master]
login_as user
visit namespace_project_mirror_path(project.namespace, project)
end
describe 'pressing "Update now"' do
before { visit namespace_project_mirror_path(project.namespace, project) }
it 'returns with the project updating (job enqueued)' do
Sidekiq::Testing.fake! { click_link('Update Now') }
expect(page).to have_content('Updating')
end
end
describe 'synchronization times' do
describe 'daily minimum mirror sync_time' do
before do
stub_application_setting(minimum_mirror_sync_time: Gitlab::Mirror::DAILY)
visit namespace_project_mirror_path(project.namespace, project)
end
it 'shows the correct selector options' do
expect(page).to have_selector('.project-mirror-sync-time > option', count: 1)
expect(page).to have_selector('.remote-mirror-sync-time > option', count: 1)
end
end
describe 'hourly minimum mirror sync_time' do
before do
stub_application_setting(minimum_mirror_sync_time: Gitlab::Mirror::HOURLY)
visit namespace_project_mirror_path(project.namespace, project)
end
it 'shows the correct selector options' do
expect(page).to have_selector('.project-mirror-sync-time > option', count: 2)
expect(page).to have_selector('.remote-mirror-sync-time > option', count: 2)
end
end
describe 'fifteen minimum mirror sync_time' do
before do
stub_application_setting(minimum_mirror_sync_time: Gitlab::Mirror::FIFTEEN)
visit namespace_project_mirror_path(project.namespace, project)
end
it 'shows the correct selector options' do
expect(page).to have_selector('.project-mirror-sync-time > option', count: 3)
expect(page).to have_selector('.remote-mirror-sync-time > option', count: 3)
end
end
end
end
end
require 'spec_helper'
describe Gitlab::Mirror do
before { Sidekiq::Logging.logger = nil }
describe '#sync_times' do
describe 'at beginning of hour' do
before { Timecop.freeze(DateTime.now.at_beginning_of_hour) }
it 'returns only fifteen and hourly sync_times' do
expect(Gitlab::Mirror.sync_times).to contain_exactly(Gitlab::Mirror::FIFTEEN, Gitlab::Mirror::HOURLY)
end
end
describe 'at beginning of day' do
before { Timecop.freeze(DateTime.now.at_beginning_of_day) }
it 'returns daily hourly and fifteen sync_times' do
expect(Gitlab::Mirror.sync_times).to contain_exactly(Gitlab::Mirror::DAILY, Gitlab::Mirror::HOURLY, Gitlab::Mirror::FIFTEEN)
end
end
describe 'every fifteen minutes' do
before { Timecop.freeze(DateTime.now.at_beginning_of_hour + 15.minutes) }
it 'returns only fifteen minutes' do
expect(Gitlab::Mirror.sync_times).to contain_exactly(Gitlab::Mirror::FIFTEEN)
end
end
after { Timecop.return }
end
describe '#configure_cron_jobs!' do
let(:daily_cron) { Gitlab::Mirror::SYNC_TIME_TO_CRON[Gitlab::Mirror::DAILY] }
let(:hourly_cron) { Gitlab::Mirror::SYNC_TIME_TO_CRON[Gitlab::Mirror::HOURLY] }
let(:fifteen_cron) { Gitlab::Mirror::SYNC_TIME_TO_CRON[Gitlab::Mirror::FIFTEEN] }
describe 'with jobs already running' do
def setup_mirrors_cron_job(current, updated_time)
allow_any_instance_of(ApplicationSetting).to receive(:minimum_mirror_sync_time).and_return(current)
Gitlab::Mirror.configure_cron_jobs!
allow_any_instance_of(ApplicationSetting).to receive(:minimum_mirror_sync_time).and_return(updated_time)
end
describe 'with daily minimum_mirror_sync_time' do
before { setup_mirrors_cron_job(Gitlab::Mirror::HOURLY, Gitlab::Mirror::DAILY) }
it 'changes cron of update_all_mirrors_worker to daily' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_mirrors_worker").cron }.from(hourly_cron).to(daily_cron)
end
it 'changes cron of update_all_remote_mirrors_worker to daily' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker").cron }.from(hourly_cron).to(daily_cron)
end
end
describe 'with hourly minimum_mirror_sync_time' do
before { setup_mirrors_cron_job(Gitlab::Mirror::DAILY, Gitlab::Mirror::HOURLY) }
it 'changes cron of update_all_mirrors_worker to daily' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_mirrors_worker").cron }.from(daily_cron).to(hourly_cron)
end
it 'changes cron of update_all_remote_mirrors_worker to daily' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker").cron }.from(daily_cron).to(hourly_cron)
end
end
describe 'with fifteen minimum_mirror_sync_time' do
before { setup_mirrors_cron_job(Gitlab::Mirror::DAILY, Gitlab::Mirror::FIFTEEN) }
it 'changes cron of update_all_mirrors_worker to fifteen' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_mirrors_worker").cron }.from(daily_cron).to(fifteen_cron)
end
it 'changes cron of update_all_remote_mirrors_worker to fifteen' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker").cron }.from(daily_cron).to(fifteen_cron)
end
end
end
describe 'without jobs already running' do
before do
Sidekiq::Cron::Job.find("update_all_mirrors_worker").destroy
Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker").destroy
end
describe 'with daily minimum_mirror_sync_time' do
before { allow_any_instance_of(ApplicationSetting).to receive(:minimum_mirror_sync_time).and_return(Gitlab::Mirror::DAILY) }
it 'creates update_all_mirrors_worker with cron of daily sync_time' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_mirrors_worker") }.from(nil).to(Sidekiq::Cron::Job)
expect(Sidekiq::Cron::Job.find("update_all_mirrors_worker").cron).to eq(daily_cron)
end
it 'creates update_all_remote_mirrors_worker with cron of daily sync_time' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker") }.from(nil).to(Sidekiq::Cron::Job)
expect(Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker").cron).to eq(daily_cron)
end
end
describe 'with hourly minimum_mirror_sync_time' do
before { allow_any_instance_of(ApplicationSetting).to receive(:minimum_mirror_sync_time).and_return(Gitlab::Mirror::HOURLY) }
it 'creates update_all_mirrors_worker with cron of hourly sync_time' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_mirrors_worker") }.from(nil).to(Sidekiq::Cron::Job)
expect(Sidekiq::Cron::Job.find("update_all_mirrors_worker").cron).to eq(hourly_cron)
end
it 'creates update_all_remote_mirrors_worker with cron of hourly sync_time' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker") }.from(nil).to(Sidekiq::Cron::Job)
expect(Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker").cron).to eq(hourly_cron)
end
end
describe 'with fifteen minimum_mirror_sync_time' do
it 'creates update_all_mirrors_worker with cron of fifteen sync_time' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_mirrors_worker") }.from(nil).to(Sidekiq::Cron::Job)
expect(Sidekiq::Cron::Job.find("update_all_mirrors_worker").cron).to eq(fifteen_cron)
end
it 'creates update_all_remote_mirrors_worker with cron of fifteen sync_time' do
expect { Gitlab::Mirror.configure_cron_jobs! }.to change { Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker") }.from(nil).to(Sidekiq::Cron::Job)
expect(Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker").cron).to eq(fifteen_cron)
end
end
end
end
describe '#at_beginning_of_day?' do
it 'returns true if at beginning_of_day' do
Timecop.freeze(DateTime.now.at_beginning_of_day)
expect(Gitlab::Mirror.at_beginning_of_day?).to be true
end
it 'returns false if at beginning of hour' do
Timecop.freeze(DateTime.now.at_beginning_of_hour)
expect(Gitlab::Mirror.at_beginning_of_day?).to be false
end
it 'returns false in every 15 minute mark' do
Timecop.freeze(DateTime.now.at_beginning_of_hour + 15.minutes)
expect(Gitlab::Mirror.at_beginning_of_day?).to be false
end
after { Timecop.return }
end
describe '#at_beginning_of_hour?' do
it 'returns true if at beginning of day' do
Timecop.freeze(DateTime.now.at_beginning_of_day)
expect(Gitlab::Mirror.at_beginning_of_day?).to be true
end
it 'returns true if at beginning of hour' do
Timecop.freeze(DateTime.now.at_beginning_of_hour)
expect(Gitlab::Mirror.at_beginning_of_hour?).to be true
end
it 'returns false in every 15 minute mark' do
Timecop.freeze(DateTime.now.at_beginning_of_hour + 15.minutes)
expect(Gitlab::Mirror.at_beginning_of_hour?).to be false
end
after { Timecop.return }
end
end
......@@ -20,6 +20,12 @@ describe ApplicationSetting, models: true do
it { is_expected.to allow_value(https).for(:after_sign_out_path) }
it { is_expected.not_to allow_value(ftp).for(:after_sign_out_path) }
it { is_expected.to allow_value(Gitlab::Mirror::FIFTEEN).for(:minimum_mirror_sync_time) }
it { is_expected.to allow_value(Gitlab::Mirror::HOURLY).for(:minimum_mirror_sync_time) }
it { is_expected.to allow_value(Gitlab::Mirror::DAILY).for(:minimum_mirror_sync_time) }
it { is_expected.not_to allow_value(nil).for(:minimum_mirror_sync_time) }
it { is_expected.not_to allow_value(61).for(:minimum_mirror_sync_time) }
describe 'disabled_oauth_sign_in_sources validations' do
before do
allow(Devise).to receive(:omniauth_providers).and_return([:github])
......@@ -41,6 +47,74 @@ describe ApplicationSetting, models: true do
subject { setting }
end
context "update minimum_mirror_sync_time" do
before do
Sidekiq::Logging.logger = nil
Gitlab::Mirror::SYNC_TIME_TO_CRON.keys.each do |sync_time|
create(:project, :mirror, sync_time: sync_time)
create(:project, :remote_mirror, sync_time: sync_time)
end
end
context 'with daily sync_time' do
let(:sync_time) { Gitlab::Mirror::DAILY }
it 'updates minimum_mirror_sync_time to daily and updates cron jobs' do
expect_any_instance_of(ApplicationSetting).to receive(:update_mirror_cron_jobs).and_call_original
expect(Gitlab::Mirror).to receive(:configure_cron_jobs!)
setting.update_attributes(minimum_mirror_sync_time: sync_time)
end
it 'updates every mirror to the current minimum_mirror_sync_time' do
expect { setting.update_attributes(minimum_mirror_sync_time: sync_time) }.to change { Project.mirror.where('sync_time < ?', sync_time).count }.from(2).to(0)
end
it 'updates every remote mirror to the current minimum_mirror_sync_time' do
expect { setting.update_attributes(minimum_mirror_sync_time: sync_time) }.to change { RemoteMirror.where('sync_time < ?', sync_time).count }.from(2).to(0)
end
end
context 'with hourly sync time' do
let(:sync_time) { Gitlab::Mirror::HOURLY }
it 'updates minimum_mirror_sync_time to daily and updates cron jobs' do
expect_any_instance_of(ApplicationSetting).to receive(:update_mirror_cron_jobs).and_call_original
expect(Gitlab::Mirror).to receive(:configure_cron_jobs!)
setting.update_attributes(minimum_mirror_sync_time: sync_time)
end
it 'updates every mirror to the current minimum_mirror_sync_time' do
expect { setting.update_attributes(minimum_mirror_sync_time: sync_time) }.to change { Project.mirror.where('sync_time < ?', sync_time).count }.from(1).to(0)
end
it 'updates every remote mirror to the current minimum_mirror_sync_time' do
expect { setting.update_attributes(minimum_mirror_sync_time: sync_time) }.to change { RemoteMirror.where('sync_time < ?', sync_time).count }.from(1).to(0)
end
end
context 'with default fifteen sync time' do
let(:sync_time) { Gitlab::Mirror::FIFTEEN }
it 'does not update minimum_mirror_sync_time' do
expect_any_instance_of(ApplicationSetting).not_to receive(:update_mirror_cron_jobs)
expect(Gitlab::Mirror).not_to receive(:configure_cron_jobs!)
expect(setting.minimum_mirror_sync_time).to eq(Gitlab::Mirror::FIFTEEN)
setting.update_attributes(minimum_mirror_sync_time: sync_time)
end
it 'updates every mirror to the current minimum_mirror_sync_time' do
expect { setting.update_attributes(minimum_mirror_sync_time: sync_time) }.not_to change { Project.mirror.where('sync_time < ?', sync_time).count }
end
it 'updates every remote mirror to the current minimum_mirror_sync_time' do
expect { setting.update_attributes(minimum_mirror_sync_time: sync_time) }.not_to change { RemoteMirror.where('sync_time < ?', sync_time).count }
end
end
end
# Upgraded databases will have this sort of content
context 'repository_storages is a String, not an Array' do
before { setting.__send__(:raw_write_attribute, :repository_storages, 'default') }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment