Merge branch 'geo/backfilling' into 'master'

Basic support to track the backfilling of repositories

See merge request !1197

.gitignore

@@ -21,6 +21,7 @@ eslint-report.html
 /backups/*
 /config/aws.yml
 /config/database.yml
+/config/database_geo.yml
 /config/gitlab.yml
 /config/gitlab_ci.yml
 /config/initializers/rack_attack.rb

.gitlab-ci.yml

@@ -27,6 +27,7 @@ before_script:
   - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) $FLAGS'
   - retry gem install knapsack
   - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
+  - '[ "$SETUP_DB" != "true" ] || bundle exec rake geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate'
 
 stages:
 - prepare

.rubocop.yml

@@ -20,6 +20,7 @@ AllCops:
     - 'node_modules/**/*'
     - 'db/*'
     - 'db/fixtures/**/*'
+    - 'db/geo/*'
     - 'tmp/**/*'
     - 'bin/**/*'
     - 'generator_templates/**/*'

app/assets/stylesheets/pages/geo_nodes.scss

+.geo-node-icon-healthy {
+  color: $gl-success;
+}
+
+.geo-node-icon-unhealthy {
+  color: $gl-danger;
+}
+
+.geo-node-icon-disabled {
+  color: $gray-darkest;
+}

app/controllers/admin/application_settings_controller.rb

@@ -171,7 +171,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :repository_size_limit,
       :shared_runners_minutes,
       :usage_ping_enabled,
-      :minimum_mirror_sync_time
+      :minimum_mirror_sync_time,
+      :geo_status_timeout
     ]
   end
 end

app/controllers/admin/geo_nodes_controller.rb

 class Admin::GeoNodesController < Admin::ApplicationController
   before_action :check_license, except: [:index, :destroy]
-  before_action :load_node, only: [:destroy, :repair, :backfill_repositories]
+  before_action :load_node, only: [:destroy, :repair, :toggle]
 
   def index
-    @nodes = GeoNode.all
+    # Ensure all nodes are using their Presenter
+    @nodes = GeoNode.all.map(&:present)
     @node = GeoNode.new
 
     unless Gitlab::Geo.license_allows?
@@ -40,14 +41,20 @@ class Admin::GeoNodesController < Admin::ApplicationController
     redirect_to admin_geo_nodes_path
   end
 
-  def backfill_repositories
+  def toggle
     if @node.primary?
-      redirect_to admin_geo_nodes_path, notice: 'This is the primary node. Please run this action with a secondary node.'
+      flash[:alert] = "Primary node can't be disabled."
     else
-      @node.backfill_repositories
-
-      redirect_to admin_geo_nodes_path, notice: 'Backfill scheduled successfully.'
+      if @node.toggle!(:enabled)
+        new_status = @node.enabled? ? 'enabled' : 'disabled'
+        flash[:notice] = "Node #{@node.url} was successfully #{new_status}."
+      else
+        action = @node.enabled? ? 'disabling' : 'enabling'
+        flash[:alert] = "There was a problem #{action} node #{@node.url}."
+      end
     end
+
+    redirect_to admin_geo_nodes_path
   end
 
   private

app/controllers/admin/health_check_controller.rb

 class Admin::HealthCheckController < Admin::ApplicationController
   def show
-    @errors = HealthCheck::Utils.process_checks(['standard'])
+    checks = ['standard']
+    checks << 'geo' if Gitlab::Geo.secondary?
+
+    @errors = HealthCheck::Utils.process_checks(checks)
   end
 end

app/helpers/ee/geo_helper.rb

+module EE
+  module GeoHelper
+    def node_status_icon(node)
+      if node.primary?
+        icon 'star fw', class: 'has-tooltip', title: 'Primary node'
+      else
+        status =
+          if node.enabled?
+            node.healthy? ? 'healthy' : 'unhealthy'
+          else
+            'disabled'
+          end
+
+        icon 'globe fw',
+             class: "geo-node-icon-#{status} has-tooltip",
+             title: status.capitalize
+      end
+    end
+
+    def toggle_node_button(node)
+      btn_class, title, data =
+        if node.enabled?
+          ['warning', 'Disable node', { confirm: 'Disabling a node stops the repositories backfilling process. Are you sure?' }]
+        else
+          ['success', 'Enable node']
+        end
+
+      link_to icon('power-off fw', text: title),
+              toggle_admin_geo_node_path(node),
+              method: :post,
+              class: "btn btn-sm btn-#{btn_class} prepend-left-10 has-tooltip",
+              title: title,
+              data: data
+    end
+  end
+end

app/models/ee/repository.rb

+module EE
+  # Repository EE mixin
+  #
+  # This module is intended to encapsulate EE-specific model logic
+  # and be prepended in the `Repository` model
+  module Repository
+    extend ActiveSupport::Concern
+
+    # Runs code after a repository has been synced.
+    def after_sync
+      expire_all_method_caches
+      expire_branch_cache
+      expire_content_cache
+    end
+  end
+end

app/models/geo/base_registry.rb

+class Geo::BaseRegistry < ActiveRecord::Base
+  self.abstract_class = true
+
+  if Gitlab::Geo.secondary? || Rails.env.test?
+    establish_connection Rails.configuration.geo_database
+  end
+end

app/models/geo/project_registry.rb

+class Geo::ProjectRegistry < Geo::BaseRegistry
+  belongs_to :project
+
+  validates :project, presence: true
+
+  scope :failed, -> { where.not(last_repository_synced_at: nil).where(last_repository_successful_sync_at: nil) }
+  scope :synced, -> { where.not(last_repository_synced_at: nil, last_repository_successful_sync_at: nil) }
+end

app/models/geo_node.rb

 class GeoNode < ActiveRecord::Base
+  include Presentable
+
   belongs_to :geo_node_key, dependent: :destroy
   belongs_to :oauth_application, class_name: 'Doorkeeper::Application', dependent: :destroy
   belongs_to :system_hook, dependent: :destroy
@@ -31,6 +33,10 @@ class GeoNode < ActiveRecord::Base
                  mode: :per_attribute_iv,
                  encode: true
 
+  def secondary?
+    !primary
+  end
+
   def uri
     if relative_url_root
       relative_url = relative_url_root.starts_with?('/') ? relative_url_root : "/#{relative_url_root}"
@@ -67,6 +73,10 @@ class GeoNode < ActiveRecord::Base
     geo_api_url("transfers/#{file_type}/#{file_id}")
   end
 
+  def status_url
+    geo_api_url('status')
+  end
+
   def oauth_callback_url
     Gitlab::Routing.url_helpers.oauth_geo_callback_url(url_helper_args)
   end
@@ -79,12 +89,6 @@ class GeoNode < ActiveRecord::Base
     self.primary? ? false : !oauth_application.present?
   end
 
-  def backfill_repositories
-    if Gitlab::Geo.enabled? && !primary?
-      GeoScheduleBackfillWorker.perform_async(id)
-    end
-  end
-
   private
 
   def geo_api_url(suffix)

app/models/geo_node_status.rb

+class GeoNodeStatus
+  include ActiveModel::Model
+
+  attr_writer :health
+
+  def health
+    @health ||= HealthCheck::Utils.process_checks(['geo'])
+  end
+
+  def healthy?
+    health.blank?
+  end
+
+  def repositories_count
+    @repositories_count ||= Project.count
+  end
+
+  def repositories_count=(value)
+    @repositories_count = value.to_i
+  end
+
+  def repositories_synced_count
+    @repositories_synced_count ||= Geo::ProjectRegistry.synced.count
+  end
+
+  def repositories_synced_count=(value)
+    @repositories_synced_count = value.to_i
+  end
+
+  def repositories_synced_in_percentage
+    return 0 if repositories_count.zero?
+
+    (repositories_synced_count.to_f / repositories_count.to_f) * 100.0
+  end
+
+  def repositories_failed_count
+    @repositories_failed_count ||= Geo::ProjectRegistry.failed.count
+  end
+
+  def repositories_failed_count=(value)
+    @repositories_failed_count = value.to_i
+  end
+end

app/models/repository.rb

@@ -6,6 +6,7 @@ class Repository
   include Gitlab::ShellAdapter
   include Elastic::RepositoriesSearch
   include RepositoryMirroring
+  prepend EE::Repository
 
   attr_accessor :path_with_namespace, :project
 

app/presenters/geo_node_presenter.rb

+class GeoNodePresenter < Gitlab::View::Presenter::Delegated
+  presents :geo_node
+
+  delegate :healthy?, :health, :repositories_count, :repositories_synced_count,
+           :repositories_synced_in_percentage, :repositories_failed_count,
+           to: :status
+
+  private
+
+  def status
+    @status ||= Geo::NodeStatusService.new.call(geo_node.status_url)
+  end
+end

app/services/geo/node_status_service.rb

+module Geo
+  class NodeStatusService
+    include Gitlab::CurrentSettings
+    include HTTParty
+
+    KEYS = %w(health repositories_count repositories_synced_count repositories_failed_count).freeze
+
+    # HTTParty timeout
+    default_timeout current_application_settings.geo_status_timeout
+
+    def call(status_url)
+      values =
+        begin
+          response = self.class.get(status_url, headers: headers)
+
+          if response.success?
+            response.parsed_response.values_at(*KEYS)
+          else
+            ["Could not connect to Geo node - HTTP Status Code: #{response.code}"]
+          end
+        rescue HTTParty::Error, Timeout::Error, SocketError, Errno::ECONNRESET, Errno::ECONNREFUSED => e
+          [e.message]
+        end
+
+      GeoNodeStatus.new(KEYS.zip(values).to_h)
+    end
+
+    private
+
+    def headers
+      Gitlab::Geo::BaseRequest.new.headers
+    end
+  end
+end

app/services/geo/notify_nodes_service.rb

@@ -17,8 +17,11 @@ module Geo
       content = { projects: projects }.to_json
 
       ::Gitlab::Geo.secondary_nodes.each do |node|
+        next unless node.enabled?
+
         notify_url = node.send(notify_url_method.to_sym)
         success, message = notify(notify_url, content)
+
         unless success
           Rails.logger.error("GitLab failed to notify #{node.url} to #{notify_url} : #{message}")
           queue.store_batched_data(projects)

app/services/geo/repository_backfill_service.rb

 module Geo
   class RepositoryBackfillService
-    attr_reader :project, :geo_node
+    attr_reader :project_id
 
-    def initialize(project, geo_node)
-      @project = project
-      @geo_node = geo_node
+    LEASE_TIMEOUT    = 8.hours.freeze
+    LEASE_KEY_PREFIX = 'repository_backfill_service'.freeze
+
+    def initialize(project_id)
+      @project_id = project_id
     end
 
     def execute
-      geo_node.system_hook.execute(hook_data, 'system_hooks')
+      try_obtain_lease do
+        log('Started repository sync')
+        started_at, finished_at = fetch_repositories
+        update_registry(started_at, finished_at)
+        log('Finished repository sync')
+      end
+    rescue ActiveRecord::RecordNotFound
+      logger.error("Couldn't find project with ID=#{project_id}, skipping syncing")
     end
 
     private
 
-    def hook_data
-      {
-        event_name: 'repository_update',
-        project_id: project.id,
-        project: project.hook_attrs,
-        remote_url: project.ssh_url_to_repo
-      }
+    def project
+      @project ||= Project.find(project_id)
+    end
+
+    def fetch_repositories
+      started_at  = DateTime.now
+      finished_at = nil
+
+      begin
+        fetch_project_repository
+        fetch_wiki_repository
+        expire_repository_caches
+
+        finished_at = DateTime.now
+      rescue Gitlab::Shell::Error => e
+        Rails.logger.error "Error syncing repository for project #{project.path_with_namespace}: #{e}"
+      end
+
+      [started_at, finished_at]
+    end
+
+    def fetch_project_repository
+      log('Fetching project repository')
+      project.create_repository unless project.repository_exists?
+      project.repository.fetch_geo_mirror(ssh_url_to_repo)
+    end
+
+    def fetch_wiki_repository
+      # Second .wiki call returns a Gollum::Wiki, and it will always create the physical repository when not found
+      if project.wiki.wiki.exist?
+        log('Fetching wiki repository')
+        project.wiki.repository.fetch_geo_mirror(ssh_url_to_wiki)
+      end
+    end
+
+    def expire_repository_caches
+      log('Expiring caches')
+      project.repository.after_sync
+    end
+
+    def try_obtain_lease
+      log('Trying to obtain lease to sync repository')
+
+      repository_lease = Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT).try_obtain
+
+      if repository_lease.nil?
+        log('Could not obtain lease to sync repository')
+        return
+      end
+
+      yield
+
+      # We should release the lease for a repository, only if we have obtained
+      # it. If something went wrong when syncing the repository, we should wait
+      # for the lease timeout to try again.
+      log('Releasing leases to sync repository')
+      Gitlab::ExclusiveLease.cancel(lease_key, repository_lease)
+    end
+
+    def update_registry(started_at, finished_at)
+      log('Updating registry information')
+      registry = Geo::ProjectRegistry.find_or_initialize_by(project_id: project.id)
+      registry.last_repository_synced_at = started_at
+      registry.last_repository_successful_sync_at = finished_at if finished_at
+      registry.save
+    end
+
+    def lease_key
+      @lease_key ||= "#{LEASE_KEY_PREFIX}:#{project.id}"
+    end
+
+    def primary_ssh_path_prefix
+      Gitlab::Geo.primary_ssh_path_prefix
+    end
+
+    def ssh_url_to_repo
+      "#{primary_ssh_path_prefix}#{project.path_with_namespace}.git"
+    end
+
+    def ssh_url_to_wiki
+      "#{primary_ssh_path_prefix}#{project.path_with_namespace}.wiki.git"
+    end
+
+    def log(message)
+      Rails.logger.info "#{self.class.name}: #{message} for project #{project.path_with_namespace} (#{project.id})"
     end
   end
 end

app/services/geo/schedule_backfill_service.rb

-module Geo
-  class ScheduleBackfillService
-    attr_accessor :geo_node_id
-
-    def initialize(geo_node_id)
-      @geo_node_id = geo_node_id
-    end
-
-    def execute
-      return if geo_node_id.nil?
-
-      Project.find_each(batch_size: 100) do |project|
-        GeoRepositoryBackfillWorker.perform_async(geo_node_id, project.id) if project.valid_repo?
-      end
-    end
-  end
-end

app/services/projects/destroy_service.rb

@@ -52,6 +52,7 @@ module Projects
       flush_caches(project, wiki_path)
 
       trash_repositories!
+      remove_tracking_entries!
       log_info("Project \"#{project.name}\" was removed")
     end
 
@@ -98,6 +99,12 @@ module Projects
       project.container_registry_repository.delete_tags
     end
 
+    def remove_tracking_entries!
+      return unless Gitlab::Geo.secondary?
+
+      Geo::ProjectRegistry.where(project_id: project.id).delete_all
+    end
+
     def raise_error(message)
       raise DestroyError.new(message)
     end

app/views/admin/application_settings/_form.html.haml

@@ -619,5 +619,18 @@
           Maximum time for web terminal websocket connection (in seconds).
           0 for unlimited.
 
+  - if Gitlab::Geo.license_allows?
+    %fieldset
+      %legend GitLab Geo
+      %p
+        These settings will only take effect if Geo is enabled and require a restart to take effect.
+      .form-group
+        = f.label :geo_status_timeout, 'Connection timeout', class: 'control-label col-sm-2'
+        .col-sm-10
+          = f.number_field :geo_status_timeout, class: 'form-control'
+          .help-block
+            The amount of seconds after which a request to get a secondary node
+            status will time out.
+
   .form-actions
     = f.submit 'Save', class: 'btn btn-save'

app/views/admin/geo_nodes/_form.html.haml

@@ -21,7 +21,6 @@
         %p.help-block
           Paste a machine public key here for the GitLab user this node runs on. Read more about how to generate it
           = link_to "here", help_page_path("ssh/README")
-
   .form-actions
     = f.submit 'Add Node', class: 'btn btn-create'
 %hr

app/views/admin/geo_nodes/index.html.haml

@@ -8,7 +8,7 @@
 
 %hr
 
-= render :partial => 'form', locals: {geo_node: @node} if Gitlab::Geo.license_allows?
+= render partial: 'form', locals: {geo_node: @node} if Gitlab::Geo.license_allows?
 
 - if @nodes.any?
   .panel.panel-default
@@ -19,10 +19,19 @@
         %li
           .list-item-name
             %span
-              = node.primary ? icon('star fw') : icon('globe fw')
+              = node_status_icon(node)
             %strong= node.url
-            %p
-              %span.help-block= node.primary ? 'Primary node' : 'Secondary node'
+            - if node.primary?
+              %span.help-block Primary node
+            - else
+              %p
+                %span.help-block
+                  Repositories synced: #{node.repositories_synced_count}/#{node.repositories_count} (#{number_to_percentage(node.repositories_synced_in_percentage, precision: 2)})
+              %p
+                %span.help-block
+                  Repositories failed: #{node.repositories_failed_count}
+              %p
+                %span.help-block= node.healthy? ? 'No Health Problems Detected' : node.health
 
           .pull-right
             - if Gitlab::Geo.license_allows?
@@ -30,10 +39,8 @@
                 = link_to repair_admin_geo_node_path(node), method: :post, title: 'OAuth application is missing', class: 'btn btn-default btn-sm prepend-left-10' do
                   = icon('exclamation-triangle fw')
                   Repair authentication
-              - unless node.primary?
-                = link_to backfill_repositories_admin_geo_node_path(node), method: :post, class: 'btn btn-primary btn-sm prepend-left-10' do
-                  = icon 'map-signs'
-                  Backfill all repositories
+              - if node.secondary?
+                = toggle_node_button(node)
             = link_to admin_geo_node_path(node), data: { confirm: 'Are you sure?' }, method: :delete, class: 'btn btn-remove btn-sm prepend-left-10' do
               = icon 'trash'
               Remove

app/views/admin/health_check/show.html.haml

@@ -34,6 +34,9 @@
         %code= health_check_url(token: current_application_settings.health_check_access_token, checks: :database)
       %li
         %code= health_check_url(token: current_application_settings.health_check_access_token, checks: :migrations)
+      - if Gitlab::Geo.secondary?
+        %li
+          %code= health_check_url(token: current_application_settings.health_check_access_token, checks: :geo)
 
   %hr
   .panel.panel-default

app/workers/geo_backfill_worker.rb

+class GeoBackfillWorker
+  include Sidekiq::Worker
+  include CronjobQueue
+
+  RUN_TIME = 5.minutes.to_i.freeze
+  BATCH_SIZE = 100.freeze
+
+  def perform
+    return unless Gitlab::Geo.primary_node.present?
+
+    start_time  = Time.now
+    project_ids = find_project_ids
+
+    logger.info "Started Geo backfilling for #{project_ids.length} project(s)"
+
+    project_ids.each do |project_id|
+      begin
+        break if over_time?(start_time)
+        break unless node_enabled?
+
+        project = Project.find(project_id)
+        next if synced?(project)
+
+        # We try to obtain a lease here for the entire backfilling process
+        # because backfill the repositories continuously at a controlled rate
+        # instead of hammering the primary node. Initially, we are backfilling
+        # one repo at a time. If we don't obtain the lease here, every 5
+        # minutes all of 100 projects will be synced.
+        try_obtain_lease do |lease|
+          Geo::RepositoryBackfillService.new(project_id).execute
+        end
+      rescue ActiveRecord::RecordNotFound
+        logger.error("Couldn't find project with ID=#{project_id}, skipping syncing")
+        next
+      end
+    end
+
+    logger.info "Finished Geo backfilling for #{project_ids.length} project(s)"
+  end
+
+  private
+
+  def find_project_ids
+    Project.where.not(id: Geo::ProjectRegistry.pluck(:project_id))
+           .limit(BATCH_SIZE)
+           .pluck(:id)
+  end
+
+  def over_time?(start_time)
+    Time.now - start_time >= RUN_TIME
+  end
+
+  def synced?(project)
+    project.repository_exists? || registry_exists?(project)
+  end
+
+  def registry_exists?(project)
+    Geo::ProjectRegistry.where(project_id: project.id)
+                        .where.not(last_repository_synced_at: nil)
+                        .any?
+  end
+
+  def try_obtain_lease
+    lease = Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout).try_obtain
+
+    return unless lease
+
+    begin
+      yield lease
+    ensure
+      Gitlab::ExclusiveLease.cancel(lease_key, lease)
+    end
+  end
+
+  def lease_key
+    Geo::RepositoryBackfillService::LEASE_KEY_PREFIX
+  end
+
+  def lease_timeout
+    Geo::RepositoryBackfillService::LEASE_TIMEOUT
+  end
+
+  def node_enabled?
+    # No caching of the enabled! If we cache it and an admin disables
+    # this node, an active GeoBackfillWorker would keep going for up
+    # to max run time after the node was disabled.
+    Gitlab::Geo.current_node.reload.enabled?
+  end
+end

app/workers/geo_repository_backfill_worker.rb

-class GeoRepositoryBackfillWorker
-  include Sidekiq::Worker
-  include ::GeoDynamicBackoff
-  include GeoQueue
-
-  def perform(geo_node_id, project_id)
-    project = Project.find(project_id)
-    geo_node = GeoNode.find(geo_node_id)
-
-    Geo::RepositoryBackfillService.new(project, geo_node).execute
-  end
-end

app/workers/geo_schedule_backfill_worker.rb

-class GeoScheduleBackfillWorker
-  include Sidekiq::Worker
-  include ::GeoDynamicBackoff
-  include GeoQueue
-
-  def perform(geo_node_id)
-    Geo::ScheduleBackfillService.new(geo_node_id).execute
-  end
-end

config/database_geo.yml.mysql

+# Warning: We don't support MySQL replication for GitLab Geo,
+# this file is needed to run tests on GitLab CI.
+
+#
+# PRODUCTION
+#
+production:
+  adapter: mysql2
+  encoding: utf8
+  collation: utf8_general_ci
+  reconnect: false
+  database: gitlabhq_geo_production
+  pool: 10
+  username: git
+  password: "secure password"
+  # host: localhost
+  # socket: /tmp/mysql.sock
+
+#
+# Development specific
+#
+development:
+  adapter: mysql2
+  encoding: utf8
+  collation: utf8_general_ci
+  reconnect: false
+  database: gitlabhq_geo_development
+  pool: 5
+  username: root
+  password: "secure password"
+  # socket: /tmp/mysql.sock
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test: &test
+  adapter: mysql2
+  encoding: utf8mb4
+  collation: utf8mb4_general_ci
+  reconnect: false
+  database: gitlabhq_geo_test
+  pool: 5
+  username: root
+  password:
+  # socket: /tmp/mysql.sock
+

config/database_geo.yml.postgresql

+#
+# PRODUCTION
+#
+production:
+  adapter: postgresql
+  encoding: unicode
+  database: gitlabhq_geo_production
+  pool: 10
+  # username: git
+  # password:
+  # host: localhost
+  # port: 5432
+
+#
+# Development specific
+#
+development:
+  adapter: postgresql
+  encoding: unicode
+  database: gitlabhq_geo_development
+  pool: 5
+  username: postgres
+  password:
+
+#
+# Staging specific
+#
+staging:
+  adapter: postgresql
+  encoding: unicode
+  database: gitlabhq_geo_staging
+  pool: 5
+  username: postgres
+  password:
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test: &test
+  adapter: postgresql
+  encoding: unicode
+  database: gitlabhq_geo_test
+  pool: 5
+  username: postgres
+  password:

config/gitlab.yml.example

@@ -208,11 +208,16 @@ production: &base
     ldap_sync_worker:
       cron: "30 1 * * *"
 
-    # Gitlab Geo nodes notification worker
+    # GitLab Geo nodes notification worker
     # NOTE: This will only take effect if Geo is enabled
     geo_bulk_notify_worker:
       cron: "*/10 * * * * *"
 
+    # GitLab Geo backfill worker
+    # NOTE: This will only take effect if Geo is enabled
+    geo_backfill_worker:
+      cron: "*/5 * * * *"
+
   registry:
     # enabled: true
     # host: registry.example.com

config/initializers/1_settings.rb

@@ -109,7 +109,7 @@ class Settings < Settingslogic
 
     def base_url(config)
       custom_port = on_standard_port?(config) ? nil : ":#{config.port}"
-      
+
       [
         config.protocol,
         "://",
@@ -330,6 +330,11 @@ Settings.pages['url']             ||= Settings.send(:build_pages_url)
 Settings.pages['external_http']   ||= false if Settings.pages['external_http'].nil?
 Settings.pages['external_https']  ||= false if Settings.pages['external_https'].nil?
 
+#
+# Geo
+#
+Settings.gitlab['geo_status_timeout'] ||= 10
+
 #
 # Git LFS
 #
@@ -384,6 +389,9 @@ Settings.cron_jobs['ldap_group_sync_worker']['job_class'] = 'LdapGroupSyncWorker
 Settings.cron_jobs['geo_bulk_notify_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['geo_bulk_notify_worker']['cron'] ||= '*/10 * * * * *'
 Settings.cron_jobs['geo_bulk_notify_worker']['job_class'] ||= 'GeoBulkNotifyWorker'
+Settings.cron_jobs['geo_backfill_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['geo_backfill_worker']['cron'] ||= '*/5 * * * *'
+Settings.cron_jobs['geo_backfill_worker']['job_class'] ||= 'GeoBackfillWorker'
 Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.send(:cron_random_weekly_time)
 Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'

config/initializers/geo.rb

+if File.exist?(Rails.root.join('config/database_geo.yml'))
+  Rails.application.configure do
+    config.geo_database = config_for(:database_geo)
+  end
+end

config/initializers/health_check.rb

 HealthCheck.setup do |config|
   config.standard_checks = %w(database migrations cache)
   config.full_checks = %w(database migrations cache)
+
+  config.add_custom_check('geo') do
+    Gitlab::Geo::HealthCheck.perform_checks
+  end
 end

config/initializers/inflections.rb

@@ -10,6 +10,6 @@
 # end
 #
 ActiveSupport::Inflector.inflections do |inflect|
-  inflect.uncountable %w(award_emoji project_statistics)
+  inflect.uncountable %w(award_emoji project_statistics project_registry)
   inflect.acronym 'EE'
 end

config/initializers/sidekiq.rb

@@ -39,6 +39,9 @@ Sidekiq.configure_server do |config|
   # Gitlab Geo: enable bulk notify job only on primary node
   Gitlab::Geo.bulk_notify_job.disable! unless Gitlab::Geo.primary?
 
+  # GitLab Geo: enable backfill job only on secondary nodes
+  Gitlab::Geo.backfill_job.disable! unless Gitlab::Geo.secondary?
+
   Gitlab::SidekiqThrottler.execute!
 
   config = ActiveRecord::Base.configurations[Rails.env] ||

config/routes/admin.rb

@@ -113,7 +113,7 @@ namespace :admin do
   resources :geo_nodes, only: [:index, :create, :destroy] do
     member do
       post :repair
-      post :backfill_repositories
+      post :toggle
     end
   end
   ## EE-specific

db/geo/migrate/20170206203234_create_project_registry.rb

+class CreateProjectRegistry < ActiveRecord::Migration
+  def change
+    create_table :project_registry do |t|
+      t.integer  :project_id, null: false
+      t.datetime :last_repository_synced_at
+      t.datetime :last_repository_successful_sync_at
+
+      t.datetime :created_at, null: false
+    end
+  end
+end

db/geo/migrate/20170302005747_add_index_to_project_id_on_project_registry.rb

+class AddIndexToProjectIdOnProjectRegistry < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :project_registry, :project_id
+  end
+
+  def down
+    remove_index :project_registry, :project_id
+  end
+end

db/geo/schema.rb

+# encoding: UTF-8
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema.define(version: 20170302005747) do
+
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "plpgsql"
+
+  create_table "project_registry", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.datetime "last_repository_synced_at"
+    t.datetime "last_repository_successful_sync_at"
+    t.datetime "created_at", null: false
+  end
+
+  add_index "project_registry", ["project_id"], name: "index_project_registry_on_project_id", using: :btree
+
+end

db/migrate/20170216000450_add_enabled_to_geo_nodes.rb

+class AddEnabledToGeoNodes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :geo_nodes, :enabled, :boolean, default: true
+  end
+
+  def down
+    remove_column :geo_nodes, :enabled
+  end
+end

db/migrate/20170308015651_add_geo_status_timout_to_application_settings.rb

+class AddGeoStatusTimoutToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :geo_status_timeout, :integer, default: 10
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170306180725) do
+ActiveRecord::Schema.define(version: 20170308015651) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -125,6 +125,7 @@ ActiveRecord::Schema.define(version: 20170306180725) do
     t.string "elasticsearch_aws_region", default: "us-east-1"
     t.string "elasticsearch_aws_access_key"
     t.string "elasticsearch_aws_secret_access_key"
+    t.integer "geo_status_timeout", default: 10
   end
 
   create_table "approvals", force: :cascade do |t|
@@ -529,6 +530,7 @@ ActiveRecord::Schema.define(version: 20170306180725) do
     t.string "access_key"
     t.string "encrypted_secret_access_key"
     t.string "encrypted_secret_access_key_iv"
+    t.boolean "enabled", default: true, null: false
   end
 
   add_index "geo_nodes", ["access_key"], name: "index_geo_nodes_on_access_key", using: :btree

lib/api/entities.rb

@@ -768,5 +768,12 @@ module API
       expose :id, :message, :starts_at, :ends_at, :color, :font
       expose :active?, as: :active
     end
+
+    class GeoNodeStatus < Grape::Entity
+      expose :health
+      expose :repositories_count
+      expose :repositories_synced_count
+      expose :repositories_failed_count
+    end
   end
 end

lib/api/geo.rb

@@ -28,6 +28,17 @@ module API
         end
       end
 
+      #  Get node information (e.g. health, repos synced, repos failed, etc.)
+      #
+      # Example request:
+      #   GET /geo/status
+      get 'status' do
+        authenticate_by_gitlab_geo_node_token!
+        require_node_to_be_secondary!
+
+        present GeoNodeStatus.new, with: Entities::GeoNodeStatus
+      end
+
       # Enqueue a batch of IDs of wiki's projects to have their
       # wiki repositories updated
       #
@@ -35,6 +46,7 @@ module API
       #   POST /geo/refresh_wikis
       post 'refresh_wikis' do
         authenticated_as_admin!
+        require_node_to_be_enabled!
         required_attributes! [:projects]
         ::Geo::ScheduleWikiRepoUpdateService.new(params[:projects]).execute
       end
@@ -45,6 +57,7 @@ module API
       #   POST /geo/receive_events
       post 'receive_events' do
         authenticate_by_gitlab_geo_token!
+        require_node_to_be_enabled!
         required_attributes! %w(event_name)
 
         case params['event_name']
@@ -75,5 +88,23 @@ module API
         end
       end
     end
+
+    helpers do
+      def authenticate_by_gitlab_geo_node_token!
+        auth_header = headers['Authorization']
+
+        unless auth_header && Gitlab::Geo::JwtRequestDecoder.new(auth_header).decode
+          unauthorized!
+        end
+      end
+
+      def require_node_to_be_enabled!
+        forbidden! 'Geo node is disabled.' unless Gitlab::Geo.current_node&.enabled?
+      end
+
+      def require_node_to_be_secondary!
+        forbidden! 'Geo node is not secondary node.' unless Gitlab::Geo.current_node&.secondary?
+      end
+    end
   end
 end

lib/generators/rails/geo_migration_generator.rb

+require 'rails/generators'
+require 'rails/generators/active_record'
+require 'rails/generators/active_record/migration/migration_generator'
+
+# TODO: this can be removed once we verify that this works as intended if we're
+# on Rails 5 since we use private Rails APIs.
+raise "Vendored ActiveRecord 5 code! Delete #{__FILE__}!" if ActiveRecord::VERSION::MAJOR >= 5
+
+class GeoMigrationGenerator < ActiveRecord::Generators::MigrationGenerator
+  source_root File.join(File.dirname(ActiveRecord::Generators::MigrationGenerator.instance_method(:create_migration_file).source_location.first), 'templates')
+
+  def create_migration_file
+    set_local_assigns!
+    validate_file_name!
+    migration_template @migration_template, "db/geo/migrate/#{file_name}.rb"
+  end
+end

lib/gitlab/geo.rb

@@ -31,7 +31,11 @@ module Gitlab
     end
 
     def self.secondary?
-      self.cache_value(:geo_node_secondary) { self.enabled? && self.current_node && !self.current_node.primary? }
+      self.cache_value(:geo_node_secondary) { self.enabled? && self.current_node && self.current_node.secondary? }
+    end
+
+    def self.primary_ssh_path_prefix
+      self.cache_value(:geo_primary_ssh_path_prefix) { self.enabled? && self.primary_node && build_primary_ssh_path_prefix }
     end
 
     def self.geo_node?(host:, port:)
@@ -46,6 +50,10 @@ module Gitlab
       Sidekiq::Cron::Job.find('geo_bulk_notify_worker')
     end
 
+    def self.backfill_job
+      Sidekiq::Cron::Job.find('geo_backfill_worker')
+    end
+
     def self.oauth_authentication
       return false unless Gitlab::Geo.secondary?
 
@@ -72,5 +80,19 @@ module Gitlab
       # urlsafe_base64 may return a string of size * 4/3
       SecureRandom.urlsafe_base64(size)[0, size]
     end
+
+    def self.build_primary_ssh_path_prefix
+      primary_host = "#{Gitlab.config.gitlab_shell.ssh_user}@#{self.primary_node.host}"
+
+      if Gitlab.config.gitlab_shell.ssh_port != 22
+        "ssh://#{primary_host}:#{Gitlab.config.gitlab_shell.ssh_port}/"
+      else
+        if self.primary_node.host.include? ':'
+          "[#{primary_host}]:"
+        else
+          "#{primary_host}:"
+        end
+      end
+    end
   end
 end

lib/gitlab/geo/base_request.rb

+module Gitlab
+  module Geo
+    class BaseRequest
+      GITLAB_GEO_AUTH_TOKEN_TYPE = 'GL-Geo'.freeze
+
+      attr_reader :request_data
+
+      def initialize(request_data = {})
+        @request_data = request_data
+      end
+
+      def headers
+        {
+          'Authorization' => geo_auth_token(request_data)
+        }
+      end
+
+      private
+
+      def geo_auth_token(message)
+        geo_node = requesting_node
+        return unless geo_node
+
+        payload = { data: message.to_json, iat: Time.now.to_i }
+        token = JWT.encode(payload, geo_node.secret_access_key, 'HS256')
+
+        "#{GITLAB_GEO_AUTH_TOKEN_TYPE} #{geo_node.access_key}:#{token}"
+      end
+
+      def requesting_node
+        Gitlab::Geo.current_node
+      end
+    end
+  end
+end

lib/gitlab/geo/health_check.rb

+module Gitlab
+  module Geo
+    class HealthCheck
+      def self.perform_checks
+        return '' unless Gitlab::Geo.secondary?
+
+        database_version  = self.get_database_version.to_i
+        migration_version = self.get_migration_version.to_i
+
+        if database_version != migration_version
+          "Current Geo database version (#{database_version}) does not match latest migration (#{migration_version})."
+        else
+          ''
+        end
+      rescue => e
+        e.message
+      end
+
+      def self.db_migrate_path
+        # Lazy initialisation so Rails.root will be defined
+        @db_migrate_path ||= File.join(Rails.root, 'db', 'geo', 'migrate')
+      end
+
+      def self.get_database_version
+        if defined?(ActiveRecord)
+          connection = ::Geo::BaseRegistry.connection
+          schema_migrations_table_name = ActiveRecord::Base.schema_migrations_table_name
+
+          if connection.table_exists?(schema_migrations_table_name)
+            connection.execute("SELECT MAX(version) AS version FROM #{schema_migrations_table_name}")
+                      .first
+                      .fetch('version')
+          end
+        end
+      end
+
+      def self.get_migration_version
+        latest_migration = nil
+
+        Dir[File.join(self.db_migrate_path, "[0-9]*_*.rb")].each do |f|
+          timestamp = f.scan(/0*([0-9]+)_[_.a-zA-Z0-9]*.rb/).first.first rescue -1
+
+          if latest_migration.nil? || timestamp.to_i > latest_migration.to_i
+            latest_migration = timestamp
+          end
+        end
+
+        latest_migration
+      end
+    end
+  end
+end

lib/gitlab/geo/jwt_request_decoder.rb

@@ -16,7 +16,7 @@ module Gitlab
       private
 
       def decode_geo_request
-        # A Geo transfer request has an Authorization header:
+        # A Geo request has an Authorization header:
         # Authorization: GL-Geo: <Geo Access Key>:<JWT payload>
         #
         # For example:
@@ -41,7 +41,7 @@ module Gitlab
           data&.deep_symbolize_keys!
           data
         rescue JWT::DecodeError => e
-          Rails.logger.error("Error decoding Geo transfer request: #{e}")
+          Rails.logger.error("Error decoding Geo request: #{e}")
         end
       end
 
@@ -58,7 +58,7 @@ module Gitlab
         tokens = auth_header.split(' ')
 
         return unless tokens.count == 2
-        return unless tokens[0] == Gitlab::Geo::TransferRequest::GITLAB_GEO_AUTH_TOKEN_TYPE
+        return unless tokens[0] == Gitlab::Geo::BaseRequest::GITLAB_GEO_AUTH_TOKEN_TYPE
 
         # Split at the first occurence of a colon
         geo_tokens = tokens[1].split(':', 2)

lib/gitlab/geo/transfer.rb

@@ -20,11 +20,11 @@ module Gitlab
         return unless primary
 
         url = primary.geo_transfers_url(file_type, file_id.to_s)
-        req_header = TransferRequest.new(request_data).header
+        req_headers = TransferRequest.new(request_data).headers
 
         return unless ensure_path_exists
 
-        download_file(url, req_header)
+        download_file(url, req_headers)
       end
 
       private
@@ -56,12 +56,12 @@ module Gitlab
 
       # Use HTTParty for now but switch to curb if performance becomes
       # an issue
-      def download_file(url, req_header)
+      def download_file(url, req_headers)
         file_size = -1
 
         begin
           File.open(filename, "wb") do |file|
-            response = HTTParty.get(url, headers: req_header, stream_body: true) do |fragment|
+            response = HTTParty.get(url, headers: req_headers, stream_body: true) do |fragment|
               file.write(fragment)
             end
 

lib/gitlab/geo/transfer_request.rb

 module Gitlab
   module Geo
-    class TransferRequest
-      GITLAB_GEO_AUTH_TOKEN_TYPE = 'GL-Geo'.freeze
-
-      attr_reader :request_data
-
-      def initialize(request_data)
-        @request_data = request_data
-      end
-
-      def header
-        {
-          "Authorization" => geo_transfer_auth(request_data.to_json),
-          "X-Sendfile-Type" => "X-Sendfile"
-        }
-      end
-
-      private
-
-      def geo_transfer_auth(message)
-        geo_node = requesting_node
-
-        return unless geo_node
-
-        payload = { data: message, iat: Time.now.to_i }
-        token = JWT.encode(payload, geo_node.secret_access_key, 'HS256')
-
-        "#{GITLAB_GEO_AUTH_TOKEN_TYPE} #{geo_node.access_key}:#{token}"
-      end
-
-      def requesting_node
-        Gitlab::Geo.current_node
+    class TransferRequest < BaseRequest
+      def headers
+        super.merge({ 'X-Sendfile-Type' => 'X-Sendfile' })
       end
     end
   end

lib/tasks/geo.rake

+task spec: ['geo:db:test:prepare']
+
+namespace :geo do
+  namespace :db do |ns|
+    %i(drop create setup migrate rollback seed version reset).each do |task_name|
+      task task_name do
+        Rake::Task["db:#{task_name}"].invoke
+      end
+    end
+
+    namespace :schema do
+      %i(load dump).each do |task_name|
+        task task_name do
+          Rake::Task["db:schema:#{task_name}"].invoke
+        end
+      end
+    end
+
+    namespace :migrate do
+      %i(up down redo).each do |task_name|
+        task task_name do
+          Rake::Task["db:migrate:#{task_name}"].invoke
+        end
+      end
+    end
+
+    namespace :test do
+      task :prepare do
+        Rake::Task['db:test:prepare'].invoke
+      end
+    end
+
+    # append and prepend proper tasks to all the tasks defined above
+    ns.tasks.each do |task|
+      task.enhance ['geo:config:check', 'geo:config:set'] do
+        Rake::Task['geo:config:restore'].invoke
+      end
+    end
+  end
+
+  namespace :config do
+    task :check do
+      unless File.exist?(Rails.root.join('config/database_geo.yml'))
+        abort('You should run these tasks only when GitLab Geo is enabled.')
+      end
+    end
+
+    task :set do
+      # save current configuration
+      @previous_config = {
+        config: Rails.application.config.dup,
+        schema: ENV['SCHEMA'],
+        skip_post_deployment_migrations: ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS']
+      }
+
+      # set config variables for geo database
+      ENV['SCHEMA'] = 'db/geo/schema.rb'
+      ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] = 'true'
+      Rails.application.config.paths['db'] = ['db/geo']
+      Rails.application.config.paths['db/migrate'] = ['db/geo/migrate']
+      Rails.application.config.paths['db/seeds.rb'] = ['db/geo/seeds.rb']
+      Rails.application.config.paths['config/database'] = ['config/database_geo.yml']
+    end
+
+    task :restore do
+      # restore config variables to previous values
+      ENV['SCHEMA'] = @previous_config[:schema]
+      ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] = @previous_config[:skip_post_deployment_migrations]
+      Rails.application.config = @previous_config[:config]
+    end
+  end
+end

scripts/prepare_build.sh

@@ -21,6 +21,11 @@ if [ -f /.dockerenv ] || [ -f ./dockerinit ]; then
     sed -i 's/password:.*/password:/g' config/database.yml
     sed -i 's/# socket:.*/host: mysql/g' config/database.yml
 
+    cp config/database_geo.yml.mysql config/database_geo.yml
+    sed -i 's/username:.*/username: root/g' config/database_geo.yml
+    sed -i 's/password:.*/password:/g' config/database_geo.yml
+    sed -i 's/# socket:.*/host: mysql/g' config/database_geo.yml
+
     cp config/resque.yml.example config/resque.yml
     sed -i 's/localhost/redis/g' config/resque.yml
 
@@ -32,4 +37,9 @@ else
     sed "s/username\:.*$/username\: runner/" -i config/database.yml
     sed "s/password\:.*$/password\: 'password'/" -i config/database.yml
     sed "s/gitlabhq_test/gitlabhq_test_$rnd/" -i config/database.yml
+
+    cp config/database_geo.yml.mysql config/database_geo.yml
+    sed "s/username\:.*$/username\: runner/" -i config/database_geo.yml
+    sed "s/password\:.*$/password\: 'password'/" -i config/database_geo.yml
+    sed "s/gitlabhq_test/gitlabhq_test_$rnd/" -i config/database_geo.yml
 fi

spec/controllers/admin/geo_nodes_controller_spec.rb

@@ -116,15 +116,83 @@ describe Admin::GeoNodesController do
     it_behaves_like 'unlicensed geo action'
   end
 
-  describe '#backfill_repositories' do
-    let(:geo_node) { create(:geo_node) }
-    subject { post :backfill_repositories, id: geo_node }
+  describe '#toggle' do
+    context 'without add-on license' do
+      let(:geo_node) { create(:geo_node) }
 
-    before do
-      allow(Gitlab::Geo).to receive(:license_allows?) { false }
-      subject
+      before do
+        allow(Gitlab::Geo).to receive(:license_allows?).and_return(false)
+        post :toggle, id: geo_node
+      end
+
+      it_behaves_like 'unlicensed geo action'
     end
 
-    it_behaves_like 'unlicensed geo action'
+    context 'with add-on license' do
+      before do
+        allow(Gitlab::Geo).to receive(:license_allows?).and_return(true)
+      end
+
+      context 'with a primary node' do
+        before do
+          post :toggle, id: geo_node
+        end
+
+        let(:geo_node) { create(:geo_node, :primary, enabled: true) }
+
+        it 'does not disable the node' do
+          expect(geo_node.reload).to be_enabled
+        end
+
+        it 'displays a flash message' do
+          expect(controller).to set_flash.now[:alert].to("Primary node can't be disabled.")
+        end
+
+        it 'redirects to the geo nodes page' do
+          expect(response).to redirect_to(admin_geo_nodes_path)
+        end
+      end
+
+      context 'with a secondary node' do
+        let(:geo_node) { create(:geo_node, host: 'example.com', port: 80, enabled: true) }
+
+        context 'when succeed' do
+          before do
+            post :toggle, id: geo_node
+          end
+
+          it 'disables the node' do
+            expect(geo_node.reload).not_to be_enabled
+          end
+
+          it 'displays a flash message' do
+            expect(controller).to set_flash.now[:notice].to('Node http://example.com/ was successfully disabled.')
+          end
+
+          it 'redirects to the geo nodes page' do
+            expect(response).to redirect_to(admin_geo_nodes_path)
+          end
+        end
+
+        context 'when fail' do
+          before do
+            allow_any_instance_of(GeoNode).to receive(:toggle!).and_return(false)
+            post :toggle, id: geo_node
+          end
+
+          it 'does not disable the node' do
+            expect(geo_node.reload).to be_enabled
+          end
+
+          it 'displays a flash message' do
+            expect(controller).to set_flash.now[:alert].to('There was a problem disabling node http://example.com/.')
+          end
+
+          it 'redirects to the geo nodes page' do
+            expect(response).to redirect_to(admin_geo_nodes_path)
+          end
+        end
+      end
+    end
   end
 end

spec/lib/gitlab/geo/jwt_request_decoder_spec.rb

@@ -5,7 +5,7 @@ describe Gitlab::Geo::JwtRequestDecoder do
   let(:data) { { input: 123 } }
   let(:request) { Gitlab::Geo::TransferRequest.new(data) }
 
-  subject { described_class.new(request.header['Authorization']) }
+  subject { described_class.new(request.headers['Authorization']) }
 
   describe '#decode' do
     it 'decodes correct data' do
@@ -13,7 +13,7 @@ describe Gitlab::Geo::JwtRequestDecoder do
     end
 
     it 'fails to decode with wrong key' do
-      data = request.header['Authorization']
+      data = request.headers['Authorization']
 
       primary_node.secret_access_key = ''
       primary_node.save

spec/models/geo/project_registry_spec.rb

+require 'spec_helper'
+
+describe Geo::ProjectRegistry, models: true do
+  describe 'relationships' do
+    it { is_expected.to belong_to(:project) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:project) }
+  end
+end

spec/models/geo_node_spec.rb

@@ -190,6 +190,14 @@ describe GeoNode, type: :model do
     end
   end
 
+  describe '#geo_status_url' do
+    let(:status_url) { "https://localhost:3000/gitlab/api/#{api_version}/geo/status" }
+
+    it 'returns api url based on node uri' do
+      expect(new_node.status_url).to eq(status_url)
+    end
+  end
+
   describe '#oauth_callback_url' do
     let(:oauth_callback_url) { 'https://localhost:3000/gitlab/oauth/geo/callback' }
 
@@ -234,28 +242,4 @@ describe GeoNode, type: :model do
       expect(node).to be_missing_oauth_application
     end
   end
-
-  describe '#backfill_repositories' do
-    before do
-      Sidekiq::Worker.clear_all
-    end
-
-    it 'schedules the scheduler worker' do
-      Sidekiq::Testing.fake! do
-        expect { node.backfill_repositories }.to change(GeoScheduleBackfillWorker.jobs, :size).by(1)
-      end
-    end
-
-    it 'schedules the correct worker for the number of projects' do
-      Sidekiq::Testing.fake! do
-        2.times do
-          create(:project)
-        end
-
-        node.backfill_repositories
-
-        expect { GeoScheduleBackfillWorker.drain }.to change(GeoRepositoryBackfillWorker.jobs, :size).by(2)
-      end
-    end
-  end
 end

spec/requests/api/geo_spec.rb

@@ -2,11 +2,16 @@ require 'spec_helper'
 
 describe API::Geo, api: true do
   include ApiHelpers
+
   let(:admin) { create(:admin) }
   let(:user) { create(:user) }
-  let!(:geo_node) { create(:geo_node, :primary) }
+  let!(:primary_node) { create(:geo_node, :primary) }
   let(:geo_token_header) do
-    { 'X-Gitlab-Token' => geo_node.system_hook.token }
+    { 'X-Gitlab-Token' => primary_node.system_hook.token }
+  end
+
+  before(:each) do
+    allow(Gitlab::Geo).to receive(:current_node) { primary_node }
   end
 
   describe 'POST /geo/receive_events authentication' do
@@ -21,6 +26,26 @@ describe API::Geo, api: true do
     end
   end
 
+  describe 'POST /geo/refresh_wikis disabled node' do
+    it 'responds with forbidden' do
+      primary_node.enabled = false
+
+      post api('/geo/refresh_wikis', admin), nil
+
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'POST /geo/receive_events disabled node' do
+    it 'responds with forbidden' do
+      primary_node.enabled = false
+
+      post api('/geo/receive_events'), nil, geo_token_header
+
+      expect(response).to have_http_status(403)
+    end
+  end
+
   describe 'POST /geo/receive_events key events' do
     before(:each) { allow_any_instance_of(::Geo::ScheduleKeyChangeService).to receive(:execute) }
 
@@ -75,14 +100,6 @@ describe API::Geo, api: true do
       post api('/geo/receive_events'), push_payload, geo_token_header
       expect(response.status).to eq 201
     end
-
-    it 'can start a refresh process from the backfill service' do
-      project = create(:project)
-      backfill = Geo::RepositoryBackfillService.new(project, geo_node)
-      post api('/geo/receive_events'), backfill.send(:hook_data), geo_token_header
-
-      expect(response.status).to eq 201
-    end
   end
 
   describe 'POST /geo/receive_events push_tag events' do
@@ -109,7 +126,7 @@ describe API::Geo, api: true do
     let(:lfs_object) { create(:lfs_object, :with_file) }
     let(:req_header) do
       transfer = Gitlab::Geo::LfsTransfer.new(lfs_object)
-      Gitlab::Geo::TransferRequest.new(transfer.request_data).header
+      Gitlab::Geo::TransferRequest.new(transfer.request_data).headers
     end
 
     before do
@@ -140,4 +157,42 @@ describe API::Geo, api: true do
       end
     end
   end
+
+  describe 'GET /geo/status' do
+    let!(:secondary_node) { create(:geo_node) }
+    let(:request) { Gitlab::Geo::BaseRequest.new }
+
+    it 'responds with 401 with invalid auth header' do
+      get api('/geo/status'), nil, Authorization: 'Test'
+
+      expect(response.status).to eq 401
+    end
+
+    context 'when requesting secondary node with valid auth header' do
+      before(:each) do
+        allow(Gitlab::Geo).to receive(:current_node) { secondary_node }
+        allow(request).to receive(:requesting_node) { primary_node }
+      end
+
+      it 'responds with 200' do
+        get api('/geo/status'), nil, request.headers
+
+        expect(response.status).to eq 200
+        expect(response.headers['Content-Type']).to eq('application/json')
+      end
+    end
+
+    context 'when requesting primary node with valid auth header' do
+      before(:each) do
+        allow(Gitlab::Geo).to receive(:current_node) { primary_node }
+        allow(request).to receive(:requesting_node) { secondary_node }
+      end
+
+      it 'responds with 403' do
+        get api('/geo/status'), nil, request.headers
+
+        expect(response).to have_http_status(403)
+      end
+    end
+  end
 end

spec/services/geo/file_upload_service_spec.rb

@@ -5,7 +5,7 @@ describe Geo::FileUploadService, services: true do
   let(:params) { { id: lfs_object.id, type: 'lfs' } }
   let(:lfs_transfer) { Gitlab::Geo::LfsTransfer.new(lfs_object) }
   let(:transfer_request) { Gitlab::Geo::TransferRequest.new(lfs_transfer.request_data) }
-  let(:req_header) { transfer_request.header['Authorization'] }
+  let(:req_header) { transfer_request.headers['Authorization'] }
 
   before do
     create(:geo_node, :current)

spec/services/geo/repository_backfill_service_spec.rb

 require 'spec_helper'
 
 describe Geo::RepositoryBackfillService, services: true do
-  SYSTEM_HOOKS_HEADER = { 'Content-Type' => 'application/json', 'X-Gitlab-Event' => 'System Hook' }.freeze
+  let!(:primary) { create(:geo_node, :primary, host: 'primary-geo-node') }
+  let(:project) { create(:empty_project) }
 
-  let(:project) { create(:project) }
-  let(:geo_node) { create(:geo_node) }
-
-  subject { Geo::RepositoryBackfillService.new(project, geo_node) }
+  subject { described_class.new(project.id) }
 
   describe '#execute' do
-    it 'calls upon the system hook of the Geo Node' do
-      WebMock.stub_request(:post, geo_node.geo_events_url)
+    it 'fetches project repositories' do
+      fetch_count = 0
+
+      allow_any_instance_of(Repository).to receive(:fetch_geo_mirror) do
+        fetch_count += 1
+      end
 
       subject.execute
 
-      expect(WebMock).to have_requested(:post, geo_node.geo_events_url).with(
-        headers: SYSTEM_HOOKS_HEADER,
-        body: {
-          event_name: 'repository_update',
-          project_id: project.id,
-          project: project.hook_attrs,
-          remote_url: project.ssh_url_to_repo
-        }
-      ).once
+      expect(fetch_count).to eq 2
+    end
+
+    it 'expires repository caches' do
+      allow_any_instance_of(Repository).to receive(:fetch_geo_mirror) { true }
+
+      expect_any_instance_of(Repository).to receive(:expire_all_method_caches).once
+      expect_any_instance_of(Repository).to receive(:expire_branch_cache).once
+      expect_any_instance_of(Repository).to receive(:expire_content_cache).once
+
+      subject.execute
+    end
+
+    it 'releases lease' do
+      expect(Gitlab::ExclusiveLease).to receive(:cancel).once.and_call_original
+
+      subject.execute
+    end
+
+    context 'tracking database' do
+      it 'tracks repository sync' do
+        expect { subject.execute }.to change(Geo::ProjectRegistry, :count).by(1)
+      end
+
+      it 'stores last_repository_successful_sync_at when succeed' do
+        allow_any_instance_of(Repository).to receive(:fetch_geo_mirror) { true }
+
+        subject.execute
+
+        registry = Geo::ProjectRegistry.find_by(project_id: project.id)
+
+        expect(registry.last_repository_successful_sync_at).not_to be_nil
+      end
+
+      it 'reset last_repository_successful_sync_at when fail' do
+        allow_any_instance_of(Repository).to receive(:fetch_geo_mirror) { raise Gitlab::Shell::Error }
+
+        subject.execute
+
+        registry = Geo::ProjectRegistry.find_by(project_id: project.id)
+
+        expect(registry.last_repository_successful_sync_at).to be_nil
+      end
     end
   end
 end

spec/services/geo/schedule_backfill_service_spec.rb

-require 'spec_helper'
-
-describe Geo::ScheduleBackfillService, services: true do
-  subject { Geo::ScheduleBackfillService.new(geo_node.id) }
-  let(:geo_node) { create(:geo_node) }
-
-  describe '#execute' do
-    it 'schedules the backfill service' do
-      Sidekiq::Worker.clear_all
-
-      Sidekiq::Testing.fake! do
-        2.times do
-          create(:project)
-        end
-
-        expect{ subject.execute }.to change(GeoRepositoryBackfillWorker.jobs, :size).by(2)
-      end
-    end
-  end
-end

spec/workers/geo_backfill_worker_spec.rb

+require 'spec_helper'
+
+describe Geo::GeoBackfillWorker, services: true do
+  let!(:primary)   { create(:geo_node, :primary, host: 'primary-geo-node') }
+  let!(:secondary) { create(:geo_node, :current) }
+  let!(:projects)  { create_list(:empty_project, 2) }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    before do
+      allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { true }
+    end
+
+    it 'performs Geo::RepositoryBackfillService for each project' do
+      expect(Geo::RepositoryBackfillService).to receive(:new).twice.and_return(spy)
+
+      subject.perform
+    end
+
+    it 'does not perform Geo::RepositoryBackfillService when node is disabled' do
+      allow_any_instance_of(GeoNode).to receive(:enabled?) { false }
+
+      expect(Geo::RepositoryBackfillService).not_to receive(:new)
+
+      subject.perform
+    end
+
+    it 'does not perform Geo::RepositoryBackfillService for projects that repository exists' do
+      create_list(:project, 2)
+
+      expect(Geo::RepositoryBackfillService).to receive(:new).twice.and_return(spy)
+
+      subject.perform
+    end
+
+    it 'does not perform Geo::RepositoryBackfillService when can not obtain a lease' do
+      allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { false }
+
+      expect(Geo::RepositoryBackfillService).not_to receive(:new)
+
+      subject.perform
+    end
+  end
+end