Merge branch 'geo-fetch-repo-once' into 'master'

Only fetch repo once on secondary after push

See merge request !1015

app/models/hooks/system_hook.rb

@@ -2,4 +2,8 @@ class SystemHook < WebHook
   def async_execute(data, hook_name)
     Sidekiq::Client.enqueue(SystemHookWorker, id, data, hook_name)
   end
+
+  def self.repository_update_hooks
+    GeoNode.where(primary: false).map(&:system_hook)
+  end
 end

app/services/geo/schedule_repo_fetch_service.rb

+module Geo
+  class ScheduleRepoFetchService
+    def initialize(params)
+      @project_id = params[:project_id]
+      @remote_url = params[:remote_url]
+    end
+
+    def execute
+      GeoRepositoryFetchWorker.perform_async(@project_id, @remote_url)
+    end
+  end
+end

app/workers/geo_repository_fetch_worker.rb

+class GeoRepositoryFetchWorker
+  include Sidekiq::Worker
+  include Gitlab::ShellAdapter
+  sidekiq_options queue: 'geo_repository_update'
+
+  def perform(project_id, clone_url)
+    project = Project.find(project_id)
+
+    project.create_repository unless project.repository_exists?
+    project.repository.after_create if project.empty_repo?
+    project.repository.fetch_geo_mirror(clone_url)
+    project.repository.expire_all_method_caches
+    project.repository.expire_branch_cache
+    project.repository.expire_content_cache
+  end
+end

app/workers/geo_repository_update_worker.rb

@@ -5,11 +5,10 @@ class GeoRepositoryUpdateWorker
 
   attr_accessor :project
 
-  def perform(project_id, clone_url, push_data = nil)
+  def perform(project_id, _clone_url, push_data = nil)
     @project = Project.find(project_id)
     @push_data = push_data
 
-    fetch_repository(clone_url)
     process_hooks if push_data # we should be compatible with old unprocessed data
   end
 

app/workers/post_receive.rb

@@ -27,6 +27,18 @@ class PostReceive
       # Triggers repository update on secondary nodes when Geo is enabled
       Gitlab::Geo.notify_wiki_update(post_received.project) if Gitlab::Geo.enabled?
     elsif post_received.regular_project?
+      # TODO: gitlab-org/gitlab-ce#26325. Remove this.
+      if Gitlab::Geo.enabled?
+        hook_data = {
+          event_name: 'repository_update',
+          project_id: post_received.project.id,
+          project: post_received.project.hook_attrs,
+          remote_url: post_received.project.ssh_url_to_repo
+        }
+
+        SystemHooksService.new.execute_hooks(hook_data, :repository_update_hooks)
+      end
+
       process_project_changes(post_received)
     else
       log("Triggered hook for unidentifiable repository type with full path \"#{repo_path}\"")

changelogs/unreleased-ee/geo-fetch-repo-once.yml

+---
+title: Perform only one fetch per push on Geo secondary nodes
+merge_request:
+author:

lib/api/geo.rb

@@ -24,6 +24,9 @@ module API
         when 'key_create', 'key_destroy'
           required_attributes! %w(key id)
           ::Geo::ScheduleKeyChangeService.new(params).execute
+        when 'repository_update'
+          required_attributes! %w(event_name project_id remote_url)
+          ::Geo::ScheduleRepoFetchService.new(params).execute
         when 'push'
           required_attributes! %w(event_name project_id project)
           ::Geo::ScheduleRepoUpdateService.new(params).execute

spec/workers/geo_repository_fetch_worker_spec.rb

+require 'spec_helper'
+
+describe GeoRepositoryFetchWorker do
+  describe '#perform' do
+    let(:project) { create(:empty_project) }
+
+    before do
+      allow_any_instance_of(Gitlab::Geo).to receive_messages(secondary?: true)
+      allow_any_instance_of(Repository).to receive(:fetch_geo_mirror).and_return(true)
+      allow_any_instance_of(Project).to receive(:repository_exists?) { false }
+      allow_any_instance_of(Project).to receive(:empty_repo?) { true }
+
+      allow_any_instance_of(Repository).to receive(:expire_all_method_caches)
+      allow_any_instance_of(Repository).to receive(:expire_branch_cache)
+      allow_any_instance_of(Repository).to receive(:expire_content_cache)
+    end
+
+    it 'creates a new repository' do
+      expect_any_instance_of(Project).to receive(:create_repository)
+
+      perform
+    end
+
+    it 'executes after_create hook' do
+      expect_any_instance_of(Repository).to receive(:after_create).at_least(:once)
+
+      perform
+    end
+
+    it 'fetches the Geo mirror' do
+      expect_any_instance_of(Repository).to receive(:fetch_geo_mirror)
+
+      perform
+    end
+
+    it 'expires repository caches' do
+      expect_any_instance_of(Repository).to receive(:expire_all_method_caches)
+      expect_any_instance_of(Repository).to receive(:expire_branch_cache)
+      expect_any_instance_of(Repository).to receive(:expire_content_cache)
+
+      perform
+    end
+  end
+
+  def perform
+    subject.perform(project.id, project.ssh_url_to_repo)
+  end
+end

spec/workers/geo_repository_update_worker_spec.rb

@@ -30,25 +30,6 @@ describe GeoRepositoryUpdateWorker do
     expect(Project).to receive(:find).at_least(:once).with(project.id) { project }
   end
 
-  context 'when no repository' do
-    before do
-      allow(project.repository).to receive(:fetch_geo_mirror)
-      allow(project).to receive(:repository_exists?) { false }
-    end
-
-    it 'creates a new repository' do
-      expect(project).to receive(:create_repository)
-
-      performed
-    end
-
-    it 'executes after_create hook' do
-      expect(project.repository).to receive(:after_create)
-
-      performed
-    end
-  end
-
   context 'when empty repository' do
     before do
       allow(project.repository).to receive(:fetch_geo_mirror)