Geo: When a repository or Wiki has failed, mark resync flag as true

This is to prevent inconsistent state that prevents a resync
from happening as seen in https://gitlab.com/gitlab-com/migration/issues/408.

ee/app/services/geo/base_sync_service.rb

@@ -176,6 +176,7 @@ module Geo
     def fail_registry!(message, error, attrs = {})
       log_error(message, error)
 
+      attrs["resync_#{type}"] = true
       attrs["last_#{type}_sync_failure"] = "#{message}: #{error.message}"
       attrs["#{type}_retry_count"] = retry_count + 1
 

ee/changelogs/unreleased/sh-mark-resync-when-failed.yml

+---
+title: 'Geo: When a repository or Wiki sync has failed, mark resync flag as true'
+merge_request:
+author:
+type: fixed

ee/db/geo/migrate/20180510223634_set_resync_flag_for_retried_projects.rb

+class SetResyncFlagForRetriedProjects < ActiveRecord::Migration
+  def up
+   execute <<-SQL
+      UPDATE project_registry SET resync_repository = 't' WHERE repository_retry_count > 0 AND resync_repository = 'f';
+      UPDATE project_registry SET resync_wiki = 't' WHERE wiki_retry_count > 0 AND resync_wiki = 'f';
+    SQL
+  end
+end

ee/db/geo/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180427114641) do
+ActiveRecord::Schema.define(version: 20180510223634) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"

ee/spec/migrations/geo/set_resync_flag_for_retried_projects_spec.rb

+require 'spec_helper'
+require Rails.root.join('ee', 'db', 'geo', 'migrate', '20180510223634_set_resync_flag_for_retried_projects.rb')
+
+describe SetResyncFlagForRetriedProjects, :geo, :migration do
+  let(:registry) { table(:project_registry) }
+
+  before do
+    registry.create!(project_id: 1, resync_repository: false, resync_wiki: false, repository_retry_count: 0, wiki_retry_count: 0)
+    registry.create!(project_id: 2, resync_repository: false, resync_wiki: false, repository_retry_count: 1)
+    registry.create!(project_id: 3, resync_repository: false, resync_wiki: false, wiki_retry_count: 1)
+  end
+
+  describe '#up' do
+    it 'sets resync_repository to true' do
+      expect(registry.where(resync_repository: true).count).to eq(0)
+
+      migrate!
+
+      dirty_projects = registry.where(resync_repository: true)
+      expect(dirty_projects.count).to eq(1)
+      expect(dirty_projects.first.project_id).to eq(2)
+    end
+
+    it 'sets resync_wiki to true' do
+      expect(registry.where(resync_wiki: true).count). to eq(0)
+
+      migrate!
+
+      dirty_wikis = registry.where(resync_wiki: true)
+      expect(dirty_wikis.count).to eq(1)
+      expect(dirty_wikis.first.project_id).to eq(3)
+    end
+  end
+end

ee/spec/services/geo/repository_sync_service_spec.rb

@@ -111,6 +111,7 @@ describe Geo::RepositorySyncService do
 
       subject.execute
 
+      expect(Geo::ProjectRegistry.last.resync_repository).to be true
       expect(Geo::ProjectRegistry.last.repository_retry_count).to eq(1)
     end
 
@@ -127,6 +128,18 @@ describe Geo::RepositorySyncService do
       expect(registry.reload.last_repository_successful_sync_at).not_to be nil
     end
 
+    it 'marks resync as true after a failure' do
+      subject.execute
+
+      allow(repository).to receive(:fetch_as_mirror)
+        .with(url_to_repo, remote_name: 'geo', forced: true)
+        .and_raise(Gitlab::Git::Repository::NoRepository)
+
+      subject.execute
+
+      expect(Geo::ProjectRegistry.last.resync_repository).to be true
+    end
+
     context 'tracking database' do
       context 'temporary repositories' do
         include_examples 'cleans temporary repositories' do

ee/spec/services/geo/wiki_sync_service_spec.rb

@@ -90,6 +90,7 @@ RSpec.describe Geo::WikiSyncService do
 
       subject.execute
 
+      expect(Geo::ProjectRegistry.last.resync_wiki).to be true
       expect(Geo::ProjectRegistry.last.wiki_retry_count).to eq(1)
     end
 
@@ -106,6 +107,18 @@ RSpec.describe Geo::WikiSyncService do
       expect(registry.last_wiki_successful_sync_at).not_to be nil
     end
 
+    it 'marks resync as true after a failure' do
+      subject.execute
+
+      allow(repository).to receive(:fetch_as_mirror)
+        .with(url_to_repo, remote_name: 'geo', forced: true)
+        .and_raise(Gitlab::Git::Repository::NoRepository)
+
+      subject.execute
+
+      expect(Geo::ProjectRegistry.last.resync_wiki).to be true
+    end
+
     context 'tracking database' do
       context 'temporary repositories' do
         include_examples 'cleans temporary repositories' do