Added additional service and workers to key change notification

app/services/geo/base_notify.rb

+module Geo
+  class BaseNotify
+    include HTTParty
+
+    # HTTParty timeout
+    default_timeout Gitlab.config.gitlab.webhook_timeout
+
+    def notify(notify_url, content)
+      response = self.class.post(notify_url,
+                                 body: content,
+                                 headers: {
+                                   'Content-Type' => 'application/json',
+                                   'PRIVATE-TOKEN' => private_token
+                                 })
+
+      [(response.code >= 200 && response.code < 300), ActionView::Base.full_sanitizer.sanitize(response.to_s)]
+    rescue HTTParty::Error, Errno::ECONNREFUSED => e
+      [false, ActionView::Base.full_sanitizer.sanitize(e.message)]
+    end
+
+    private
+
+    def private_token
+      # TODO: should we ask admin user to be defined as part of configuration?
+      @private_token ||= User.find_by(admin: true).authentication_token
+    end
+  end
+end

app/services/geo/notify_key_change_service.rb

+module Geo
+  class NotifyKeyChangeService < BaseNotify
+
+    def initialize(key_id, change)
+      @id = key_id
+      @action = change
+    end
+
+    def execute
+      key_change = { 'id' => @id, 'action' => @action }
+      content = { key_change: key_change }.to_json
+
+      ::Gitlab::Geo.secondary_nodes.each do |node|
+        notify_url = node.notify_key_url
+        success, message = notify(notify_url, content)
+
+        unless success
+          error_message = "GitLab failed to notify #{node.url} to #{notify_url} : #{message}"
+          Rails.logger.error(error_message)
+          fail error_message # we must throw exception here to re-schedule job execution.
+        end
+      end
+    end
+  end
+end

app/services/geo/notify_nodes_service.rb

 module Geo
-  class NotifyNodesService
-    include HTTParty
-
-    # HTTParty timeout
-    default_timeout Gitlab.config.gitlab.webhook_timeout
+  class NotifyNodesService < BaseNotify
 
     def initialize
       @proj_queue = Gitlab::Geo::UpdateQueue.new('updated_projects')
@@ -19,34 +15,18 @@ module Geo
 
     def process(queue, notify_url_method)
       return if queue.empty?
+
       projects = queue.fetch_batched_data
+      content = { projects: projects }.to_json
 
       ::Gitlab::Geo.secondary_nodes.each do |node|
         notify_url = node.send(notify_url_method.to_sym)
-        success, message = notify(notify_url, projects)
+        success, message = notify(notify_url, content)
         unless success
           Rails.logger.error("GitLab failed to notify #{node.url} to #{notify_url} : #{message}")
           queue.store_batched_data(projects)
         end
       end
     end
-
-    def notify(notify_url, projects)
-      response = self.class.post(notify_url,
-                                 body: { projects: projects }.to_json,
-                                 headers: {
-                                   'Content-Type' => 'application/json',
-                                   'PRIVATE-TOKEN' => private_token
-                                 })
-
-      [(response.code >= 200 && response.code < 300), ActionView::Base.full_sanitizer.sanitize(response.to_s)]
-    rescue HTTParty::Error, Errno::ECONNREFUSED => e
-      [false, ActionView::Base.full_sanitizer.sanitize(e.message)]
-    end
-
-    def private_token
-      # TODO: should we ask admin user to be defined as part of configuration?
-      @private_token ||= User.find_by(admin: true).authentication_token
-    end
   end
 end

app/workers/geo_key_change_notify_worker.rb

+class GeoKeyChangeNotifyWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :default, retry: 55
+
+  sidekiq_retry_in do |count|
+    count <= 30 ? linear_backoff_strategy(count) : geometric_backoff_strategy(count)
+  end
+
+  def perform(key_id, change)
+    Geo::NotifyKeyChangeService.new(key_id, change).execute
+  end
+
+  private
+
+  def linear_backoff_strategy(count)
+    rand(1..20) + count
+  end
+
+  def geometric_backoff_strategy(count)
+    # This strategy is based on the original one from sidekiq
+    count = count-30 # we must start counting after 30
+    (count ** 4) + 15 + (rand(30)*(count+1))
+  end
+end

lib/gitlab/geo.rb

@@ -43,7 +43,7 @@ module Gitlab
     end
 
     def self.notify_ssh_key_change(key_id, change)
-      ::Geo::ScheduleKeyChangeService.new(key_id, change.to_sym).execute
+      GeoKeyChangeNotifyWorker.perform_async(key_id, change)
     end
 
     def self.bulk_notify_job