fixes scheduling issue related to all mirrors

app/workers/update_all_mirrors_worker.rb

@@ -27,7 +27,7 @@ class UpdateAllMirrorsWorker
   private
 
   def mirrors_to_sync
-    Project.mirror.where(sync_time: Gitlab::Mirror.sync_times)
+    Project.mirror.where("mirror_last_successful_update_at + #{Gitlab::Database.minute_interval('sync_time')} <= ? OR sync_time IN (?)", DateTime.now, Gitlab::Mirror.sync_times)
   end
 
   def try_obtain_lease

app/workers/update_all_remote_mirrors_worker.rb

@@ -17,6 +17,6 @@ class UpdateAllRemoteMirrorsWorker
   private
 
   def remote_mirrors_to_sync
-    RemoteMirror.where(sync_time: Gitlab::Mirror.sync_times)
+    RemoteMirror.where("last_successful_update_at + #{Gitlab::Database.minute_interval('sync_time')} <= ? OR sync_time IN (?)", DateTime.now, Gitlab::Mirror.sync_times)
   end
 end

db/migrate/20170208144550_add_index_to_mirrors_last_update_at_fields.rb

+class AddIndexToMirrorsLastUpdateAtFields < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def change
+    add_concurrent_index :projects, :mirror_last_successful_update_at
+    add_concurrent_index :remote_mirrors, :last_successful_update_at
+  end
+end

db/schema.rb

@@ -1127,6 +1127,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do
   add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
   add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree
   add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree
+  add_index "projects", ["mirror_last_successful_update_at"], name: "index_projects_on_mirror_last_successful_update_at", using: :btree
   add_index "projects", ["name"], name: "index_projects_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
   add_index "projects", ["namespace_id"], name: "index_projects_on_namespace_id", using: :btree
   add_index "projects", ["path"], name: "index_projects_on_path", using: :btree
@@ -1216,6 +1217,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do
     t.integer "sync_time", default: 60, null: false
   end
 
+  add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree
   add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
   add_index "remote_mirrors", ["sync_time"], name: "index_remote_mirrors_on_sync_time", using: :btree
 

lib/gitlab/database.rb

@@ -53,6 +53,10 @@ module Gitlab
       Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()"
     end
 
+    def self.minute_interval(value)
+      Gitlab::Database.postgresql? ? "#{value} * '1 minute'::interval" : "INTERVAL #{value} MINUTE"
+    end
+
     def true_value
       if Gitlab::Database.postgresql?
         "'t'"

spec/workers/update_all_mirrors_worker_spec.rb

@@ -7,15 +7,14 @@ describe UpdateAllMirrorsWorker do
   end
 
   describe '#perform' do
-    project_count_with_time = { DateTime.now.beginning_of_hour + 15.minutes => 1,
-                                DateTime.now.beginning_of_hour => 2,
-                                DateTime.now.beginning_of_day => 3
+    project_count_with_time = { DateTime.now.beginning_of_hour + 15.minutes => 2,
+                                DateTime.now.beginning_of_hour => 3,
+                                DateTime.now.beginning_of_day => 4
                               }
 
     let!(:mirror1) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::FIFTEEN) }
     let!(:mirror2) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::HOURLY) }
     let!(:mirror3) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::DAILY) }
-    let(:mirrors) { Project.mirror.where(sync_time: Gitlab::Mirror.sync_times) }
 
     it 'fails stuck mirrors' do
       worker = described_class.new
@@ -27,8 +26,11 @@ describe UpdateAllMirrorsWorker do
 
     project_count_with_time.each do |time, project_count|
       describe "at #{time}" do
+        let!(:mirror4) { create(:empty_project, :mirror, sync_time: Gitlab::Mirror::DAILY, mirror_last_successful_update_at: time - (Gitlab::Mirror::DAILY + 5).minutes }
+        let(:mirrors) { Project.mirror.where("mirror_last_successful_update_at + #{Gitlab::Database.minute_interval('sync_time')} <= ? OR sync_time IN (?)", time, Gitlab::Mirror.sync_times) }
+
         before do
-          allow(DateTime).to receive(:now).and_return(time)
+          Timecop.freeze(time)
         end
 
         it 'enqueues a job on mirrored Projects' do

spec/workers/update_all_remote_mirrors_worker.rb

@@ -2,15 +2,14 @@ require 'rails_helper'
 
 describe UpdateAllRemoteMirrorsWorker do
   describe "#perform" do
-    project_count_with_time = { DateTime.now.beginning_of_hour + 15.minutes => 1,
-                                DateTime.now.beginning_of_hour => 2,
-                                DateTime.now.beginning_of_day => 3
+    project_count_with_time = { DateTime.now.beginning_of_hour + 15.minutes => 2,
+                                DateTime.now.beginning_of_hour => 3,
+                                DateTime.now.beginning_of_day => 4
                               }
 
     let!(:mirror1) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::FIFTEEN) }
     let!(:mirror2) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::HOURLY) }
     let!(:mirror3) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::DAILY) }
-    let(:mirrors) { RemoteMirror.where(sync_time: Gitlab::Mirror.sync_times) }
 
     it 'fails stuck mirrors' do
       worker = described_class.new
@@ -22,8 +21,12 @@ describe UpdateAllRemoteMirrorsWorker do
 
     project_count_with_time.each do |time, project_count|
       describe "at #{time}" do
+        let!(:mirror4) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::DAILY) }
+        let(:mirrors) { RemoteMirror.where("last_successful_update_at + #{Gitlab::Database.minute_interval('sync_time')} <= ? OR sync_time IN (?)", time, Gitlab::Mirror.sync_times) }
+
         before do
-          allow(DateTime).to receive(:now).and_return(time)
+          Timecop.freeze(time)
+          mirror4.remote_mirrors.first.update_attributes(last_successful_update_at: time - (Gitlab::Mirror::DAILY + 5).minutes)
         end
 
         it 'enqueues a job on mirrored Projects' do