Merge branch 'geo-repository-backfill-service' into 'master'

Added Backfill service for Geo

Geo nodes now can be synced without rsync. The sync will be triggered using our own transport method instead
and it will be triggered from the primary node at the moment a secondary node is created.

![](https://gitlab.com/gitlab-org/gitlab-ee/uploads/5a0eae3c9bf6620121e171bc07ca0a61/GeoNode_backfill.png)

Fixes #1190

cc @brodock @regisF

See merge request !861

app/controllers/admin/geo_nodes_controller.rb

 class Admin::GeoNodesController < Admin::ApplicationController
   before_action :check_license
+  before_action :load_node, only: [:destroy, :repair, :backfill_repositories]
 
   def index
     @nodes = GeoNode.all
@@ -18,15 +19,12 @@ class Admin::GeoNodesController < Admin::ApplicationController
   end
 
   def destroy
-    @node = GeoNode.find(params[:id])
     @node.destroy
 
     redirect_to admin_geo_nodes_path, notice: 'Node was successfully removed.'
   end
 
   def repair
-    @node = GeoNode.find(params[:id])
-
     if @node.primary? || !@node.missing_oauth_application?
       flash[:notice] = "This node doesn't need to be repaired."
     elsif @node.save
@@ -38,6 +36,16 @@ class Admin::GeoNodesController < Admin::ApplicationController
     redirect_to admin_geo_nodes_path
   end
 
+  def backfill_repositories
+    if @node.primary?
+      redirect_to admin_geo_nodes_path, notice: 'This is the primary node. Please run this action with a secondary node.'
+    else
+      @node.backfill_repositories
+
+      redirect_to admin_geo_nodes_path, notice: 'Backfill scheduled successfully.'
+    end
+  end
+
   private
 
   def geo_node_params
@@ -50,4 +58,8 @@ class Admin::GeoNodesController < Admin::ApplicationController
       redirect_to admin_license_path
     end
   end
+
+  def load_node
+    @node = GeoNode.find(params[:id])
+  end
 end

app/models/geo_node.rb

@@ -65,6 +65,12 @@ class GeoNode < ActiveRecord::Base
     self.primary? ? false : !oauth_application.present?
   end
 
+  def backfill_repositories
+    if Gitlab::Geo.enabled? && !primary?
+      GeoScheduleBackfillWorker.perform_async(id)
+    end
+  end
+
   private
 
   def url_helper_args

app/services/geo/repository_backfill_service.rb

+module Geo
+  class RepositoryBackfillService
+    attr_reader :project, :geo_node
+
+    def initialize(project, geo_node)
+      @project = project
+      @geo_node = geo_node
+    end
+
+    def execute
+      geo_node.system_hook.execute(hook_data, 'system_hooks')
+    end
+
+    private
+
+    def hook_data
+      {
+        event_name: 'push',
+        project_id: project.id,
+        project: project.hook_attrs
+      }
+    end
+  end
+end

app/services/geo/schedule_backfill_service.rb

+module Geo
+  class ScheduleBackfillService
+    attr_accessor :geo_node_id
+
+    def initialize(geo_node_id)
+      @geo_node_id = geo_node_id
+    end
+
+    def execute
+      return if geo_node_id.nil?
+
+      Project.find_each(batch_size: 100) do |project|
+        GeoRepositoryBackfillWorker.perform_async(geo_node_id, project.id) if project.valid_repo?
+      end
+    end
+  end
+end

app/views/admin/geo_nodes/index.html.haml

@@ -53,7 +53,13 @@
 
           .pull-right
             - if node.missing_oauth_application?
-              = link_to repair_admin_geo_node_path(node), method: :post, title: 'OAuth application is missing', class: 'btn btn-default, btn-sm' do
+              = link_to repair_admin_geo_node_path(node), method: :post, title: 'OAuth application is missing', class: 'btn btn-default btn-sm prepend-left-10' do
                 = icon('exclamation-triangle fw')
                 Repair authentication
-            = link_to 'Remove', admin_geo_node_path(node), data: { confirm: 'Are you sure?' }, method: :delete, class: 'btn btn-remove btn-sm'
+            - unless node.primary?
+              = link_to backfill_repositories_admin_geo_node_path(node), method: :post, class: 'btn btn-primary btn-sm prepend-left-10' do
+                = icon 'map-signs'
+                Backfill all repositories
+            = link_to admin_geo_node_path(node), data: { confirm: 'Are you sure?' }, method: :delete, class: 'btn btn-remove btn-sm prepend-left-10' do
+              = icon 'trash'
+              Remove

app/workers/geo_repository_backfill_worker.rb

+class GeoRepositoryBackfillWorker
+  include Sidekiq::Worker
+  include ::GeoDynamicBackoff
+  include GeoQueue
+
+  def perform(geo_node_id, project_id)
+    project = Project.find(project_id)
+    geo_node = GeoNode.find(geo_node_id)
+
+    Geo::RepositoryBackfillService.new(project, geo_node).execute
+  end
+end

app/workers/geo_repository_update_worker.rb

 class GeoRepositoryUpdateWorker
   include Sidekiq::Worker
   include Gitlab::ShellAdapter
-  include GeoQueue
+  include DedicatedSidekiqQueue
 
   attr_accessor :project
 

app/workers/geo_schedule_backfill_worker.rb

+class GeoScheduleBackfillWorker
+  include Sidekiq::Worker
+  include ::GeoDynamicBackoff
+  include GeoQueue
+
+  def perform(geo_node_id)
+    Geo::ScheduleBackfillService.new(geo_node_id).execute
+  end
+end

changelogs/unreleased-ee/geo-repository-backfill-service.yml

+---
+title: Added Backfill service for Geo
+merge_request: 861
+author:

config/routes/admin.rb

@@ -105,6 +105,7 @@ namespace :admin do
   resources :geo_nodes, only: [:index, :create, :destroy] do
     member do
       post :repair
+      post :backfill_repositories
     end
   end
   ## EE-specific

config/sidekiq_queues.yml

@@ -53,3 +53,4 @@
   - [geo, 1]
   - [project_update_repository_storage, 1]
   - [admin_emails, 1]
+  - [geo_repository_update, 1]

doc/gitlab-geo/README.md

@@ -69,8 +69,10 @@ there are a few things to consider:
          git remote set-url --push origin git@primary.gitlab.example.com:user/repo.git
          ```
 
-> **Important**: The initialization of a new Geo secondary node requires data
-to be copied from the primary, as there is no backfill feature bundled with it.
+>**Important**:
+The initialization of a new Geo secondary node on versions older than 8.14 
+requires data to be copied from the primary, as there is no backfill 
+feature bundled with those versions.
 See more details in the [Configure GitLab](configuration.md) step.
 
 ## Current limitations

doc/gitlab-geo/configuration.md

@@ -214,8 +214,12 @@ The two most obvious issues that replication can have here are:
 ### Step 5. Replicating the repositories data
 
 Getting a new secondary Geo node up and running, will also require the
-repositories directory to be synced from the primary node. You can use `rsync`
-for that.
+repositories directory to be synced from the primary node.
+
+With GitLab **8.14** you can start the syncing process by clicking the
+"Backfill all repositories" button on `Admin > Geo Nodes` screen. 
+
+On previous versions, you can use `rsync` for that:
 
 Make sure `rsync` is installed in both primary and secondary servers and root
 SSH access with a password is enabled. Otherwise, you can set up an SSH key-based

spec/models/geo_node_spec.rb

@@ -225,4 +225,28 @@ describe GeoNode, type: :model do
       expect(node).to be_missing_oauth_application
     end
   end
+
+  describe '#backfill_repositories' do
+    before do
+      Sidekiq::Worker.clear_all
+    end
+
+    it 'schedules the scheduler worker' do
+      Sidekiq::Testing.fake! do
+        expect { node.backfill_repositories }.to change(GeoScheduleBackfillWorker.jobs, :size).by(1)
+      end
+    end
+
+    it 'schedules the correct worker for the number of projects' do
+      Sidekiq::Testing.fake! do
+        2.times do
+          create(:project)
+        end
+
+        node.backfill_repositories
+
+        expect { GeoScheduleBackfillWorker.drain }.to change(GeoRepositoryBackfillWorker.jobs, :size).by(2)
+      end
+    end
+  end
 end

spec/requests/api/geo_spec.rb

@@ -78,6 +78,14 @@ describe API::API, api: true do
       post api('/geo/receive_events'), push_payload, geo_token_header
       expect(response.status).to eq 201
     end
+
+    it 'can start a refresh process from the backfill service' do
+      project = create(:project)
+      backfill = Geo::RepositoryBackfillService.new(project, geo_node)
+      post api('/geo/receive_events'), backfill.send(:hook_data), geo_token_header
+
+      expect(response.status).to eq 201
+    end
   end
 
   describe 'POST /geo/receive_events push_tag events' do

spec/services/geo/repository_backfill_service_spec.rb

+require 'spec_helper'
+
+describe Geo::RepositoryBackfillService, services: true do
+  SYSTEM_HOOKS_HEADER = { 'Content-Type' => 'application/json', 'X-Gitlab-Event' => 'System Hook' }
+
+  let(:project) { create(:project) }
+  let(:geo_node) { create(:geo_node) }
+
+  subject { Geo::RepositoryBackfillService.new(project, geo_node) }
+
+  describe '#execute' do
+    it 'calls upon the system hook of the Geo Node' do
+      WebMock.stub_request(:post, geo_node.geo_events_url)
+
+      subject.execute
+
+      expect(WebMock).to have_requested(:post, geo_node.geo_events_url).with(
+        headers: SYSTEM_HOOKS_HEADER,
+        body: {
+          event_name: 'push',
+          project_id: project.id,
+          project: project.hook_attrs
+        }
+      ).once
+    end
+  end
+end

spec/services/geo/schedule_backfill_service_spec.rb

+require 'spec_helper'
+
+describe Geo::ScheduleBackfillService, services: true do
+  subject { Geo::ScheduleBackfillService.new(geo_node.id) }
+  let(:geo_node) { create(:geo_node) }
+
+  describe '#execute' do
+    it 'schedules the backfill service' do
+      Sidekiq::Worker.clear_all
+
+      Sidekiq::Testing.fake! do
+        2.times do
+          create(:project)
+        end
+
+        expect{ subject.execute }.to change(GeoRepositoryBackfillWorker.jobs, :size).by(2)
+      end
+    end
+  end
+end