Merge branch 'feature/sidekiq-cron' into 'master'

Migrate from Sidetiq to Sidekiq-cron EE specific jobs

This is a followup from gitlab-org/gitlab-ce#2355 with specific EE jobs migrated to use Sidekiq-cron instead of Sidetiq

See merge request !69

Gemfile

@@ -125,8 +125,8 @@ gem 'acts-as-taggable-on', '~> 3.4'
 
 # Background jobs
 gem 'sinatra', '~> 1.4.4', require: nil
-gem 'sidekiq', '3.3.0'
-gem 'sidetiq', '~> 0.6.3'
+gem 'sidekiq', '~> 3.5.0'
+gem 'sidekiq-cron', '~> 0.3.0'
 
 # HTTP requests
 gem "httparty", '~> 0.13.3'

Gemfile.lock

@@ -116,8 +116,23 @@ GEM
       activemodel (>= 3.2.0)
       activesupport (>= 3.2.0)
       json (>= 1.7)
-    celluloid (0.16.0)
-      timers (~> 4.0.0)
+    celluloid (0.17.2)
+      celluloid-essentials
+      celluloid-extras
+      celluloid-fsm
+      celluloid-pool
+      celluloid-supervision
+      timers (>= 4.1.1)
+    celluloid-essentials (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-extras (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-fsm (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-pool (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-supervision (0.20.5)
+      timers (>= 4.1.1)
     charlock_holmes (0.7.3)
     chunky_png (1.3.5)
     cliver (0.3.2)
@@ -374,7 +389,6 @@ GEM
       multi_xml (>= 0.5.2)
     httpclient (2.7.0.1)
     i18n (0.7.0)
-    ice_cube (0.11.1)
     ice_nine (0.11.1)
     inflecto (0.0.2)
     ipaddress (0.8.0)
@@ -645,6 +659,7 @@ GEM
       sexp_processor (~> 4.1)
     rubyntlm (0.5.2)
     rubypants (0.2.0)
+    rufus-scheduler (3.1.10)
     rugged (0.23.3)
     safe_yaml (1.0.4)
     sanitize (2.1.0)
@@ -672,16 +687,15 @@ GEM
       rack
     shoulda-matchers (2.8.0)
       activesupport (>= 3.0.0)
-    sidekiq (3.3.0)
-      celluloid (>= 0.16.0)
-      connection_pool (>= 2.0.0)
-      json
-      redis (>= 3.0.6)
-      redis-namespace (>= 1.3.1)
-    sidetiq (0.6.3)
-      celluloid (>= 0.14.1)
-      ice_cube (= 0.11.1)
-      sidekiq (>= 3.0.0)
+    sidekiq (3.5.3)
+      celluloid (~> 0.17.2)
+      connection_pool (~> 2.2, >= 2.2.0)
+      json (~> 1.0)
+      redis (~> 3.2, >= 3.2.1)
+      redis-namespace (~> 1.5, >= 1.5.2)
+    sidekiq-cron (0.3.1)
+      rufus-scheduler (>= 2.0.24)
+      sidekiq (>= 2.17.3)
     simple_oauth (0.1.9)
     simplecov (0.10.0)
       docile (~> 1.1.0)
@@ -747,7 +761,7 @@ GEM
     thor (0.19.1)
     thread_safe (0.3.5)
     tilt (1.4.1)
-    timers (4.0.4)
+    timers (4.1.1)
       hitimes
     timfel-krb5-auth (0.8.3)
     tinder (1.10.1)
@@ -945,8 +959,8 @@ DEPENDENCIES
   settingslogic (~> 2.0.9)
   sham_rack
   shoulda-matchers (~> 2.8.0)
-  sidekiq (= 3.3.0)
-  sidetiq (~> 0.6.3)
+  sidekiq (~> 3.5.0)
+  sidekiq-cron (~> 0.3.0)
   simplecov (~> 0.10.0)
   sinatra (~> 1.4.4)
   six (~> 0.2.0)

app/workers/historical_data_worker.rb

 class HistoricalDataWorker
   include Sidekiq::Worker
-  include Sidetiq::Schedulable
-
-  recurrence { daily.hour_of_day(12) }
 
   def perform
     HistoricalData.track!

app/workers/ldap_sync_worker.rb

 class LdapSyncWorker
   include Sidekiq::Worker
-  include Sidetiq::Schedulable
 
   sidekiq_options retry: false
 
-  if Gitlab.config.ldap.enabled
-    DAILY = Gitlab.config.ldap.schedule_sync_daily
-    HOUR = Gitlab.config.ldap.schedule_sync_hour
-    MINUTE = Gitlab.config.ldap.schedule_sync_minute
-
-    recurrence { daily(DAILY).hour_of_day(HOUR).minute_of_hour(MINUTE) }
-  end
-
   def perform
     Rails.logger.info "Performing daily LDAP sync task."
     User.ldap.find_each(batch_size: 100).each do |ldap_user|

app/workers/stuck_ci_builds_worker.rb

 class StuckCiBuildsWorker
   include Sidekiq::Worker
-  include Sidetiq::Schedulable
 
   BUILD_STUCK_TIMEOUT = 1.day
 
-  recurrence { daily }
-
   def perform
     Rails.logger.info 'Cleaning stuck builds'
 

app/workers/update_all_mirrors_worker.rb

 class UpdateAllMirrorsWorker
   include Sidekiq::Worker
-  include Sidetiq::Schedulable
-
-  recurrence { hourly }
 
   def perform
     Project.mirror.each(&:update_mirror)

config/gitlab.yml.example

@@ -170,13 +170,10 @@ production: &base
   ldap:
     enabled: false
 
-    # GitLab EE only.
-    # In addition to refreshing users when they log in,
-    # enabling this setting will refresh LDAP user membership once a day.
-    # Default time of the day when this will happen is at 1:30am server time.
-    schedule_sync_daily: 1 # Run once every X days. Default: 1, meaning 'daily'
-    schedule_sync_hour: 1 # Hour of the day. Value from 0-23.
-    schedule_sync_minute: 30 # Minute of the hour. Value from 0-59.
+    #
+    # Gitlab EE only:
+    # Since Gitlab 8.3, LDAP refresh job execution can be configured in schedule.yml
+    #
 
     servers:
       ##########################################################################

config/initializers/sidekiq.rb

@@ -17,6 +17,12 @@ Sidekiq.configure_server do |config|
     chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
     chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS']
   end
+
+  # Sidekiq-cron: load recurring jobs from schedule.yml
+  schedule_file = 'config/schedule.yml'
+  if File.exists?(schedule_file)
+    Sidekiq::Cron::Job.load_from_hash YAML.load(ERB.new(File.read(schedule_file)).result)
+  end
 end
 
 Sidekiq.configure_client do |config|

config/routes.rb

 require 'sidekiq/web'
+require 'sidekiq/cron/web'
 require 'api/api'
 
 Rails.application.routes.draw do

config/schedule.yml

+# Here is a list of jobs that are scheduled to run periodically.
+# We use a UNIX cron notation to specify execution schedule.
+#
+# Please read here for more information:
+# https://github.com/ondrejbartas/sidekiq-cron#adding-cron-job
+
+stuck_ci_builds_worker:
+  cron: "0 0 * * *"
+  class: "StuckCiBuildsWorker"
+  queue: "default"
+
+#
+# GitLab EE only jobs:
+#
+
+historical_data_worker:
+  cron: "0 12 * * *"
+  class: "HistoricalDataWorker"
+  queue: "default"
+
+update_all_mirrors_worker:
+  cron: "0 * * * *"
+  class: "UpdateAllMirrorsWorker"
+  queue: "default"
+
+<% if Gitlab.config.ldap.enabled %>
+# In addition to refreshing users when they log in,
+# enabling this setting will refresh LDAP user membership once a day.
+# Default time of the day when this will happen is at 1:30am server time.
+ldap_sync_worker:
+  cron: "30 1 * * *"
+  class: "LdapSyncWorker"
+  queue: "default"
+<% end %>