Merge branch...

Merge branch '237891-improve-ci-for-external-repo-with-configurable-maximum-mirroring-frequency-on-self-hosted' into 'master'

Improve CI for external repo with configurable maximum mirroring frequency on self-hosted

See merge request gitlab-org/gitlab!48955

changelogs/unreleased/237891-improve-ci-for-external-repo-with-configurable-maximum-mirroring-f.yml

+---
+title: Improve CI for external repo with configurable maximum mirroring frequency
+  on self-hosted
+merge_request: 48955
+author:
+type: changed

db/migrate/20201204215353_add_pull_mirror_interval_to_plan_limits.rb

+# frozen_string_literal: true
+
+class AddPullMirrorIntervalToPlanLimits < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    add_column :plan_limits, :pull_mirror_interval_seconds, :integer, default: 300, null: false
+  end
+end

db/schema_migrations/20201204215353

+a3dd8cfe4a5d83ca370cac90acf127facf40c0fd63ae8d1d3f99418295bae148
\ No newline at end of file

db/structure.sql

@@ -14987,7 +14987,8 @@ CREATE TABLE plan_limits (
     debian_max_file_size bigint DEFAULT '3221225472'::bigint NOT NULL,
     project_feature_flags integer DEFAULT 200 NOT NULL,
     ci_max_artifact_size_api_fuzzing integer DEFAULT 0 NOT NULL,
-    ci_pipeline_deployments integer DEFAULT 500 NOT NULL
+    ci_pipeline_deployments integer DEFAULT 500 NOT NULL,
+    pull_mirror_interval_seconds integer DEFAULT 300 NOT NULL
 );
 
 CREATE SEQUENCE plan_limits_id_seq

doc/administration/instance_limits.md

@@ -181,6 +181,23 @@ Plan.default.actual_limits.update!(group_hooks: 100)
 
 Set the limit to `0` to disable it.
 
+## Pull Mirroring Interval
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/237891) in GitLab 13.7.
+
+The [minimum time between pull refreshes](../user/project/repository/repository_mirroring.md)
+defaults to 300 seconds (5 minutes).
+
+To change this limit on a self-managed installation, run the following in the
+[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session):
+
+```ruby
+# If limits don't exist for the default plan, you can create one with:
+# Plan.default.create_limits!
+
+Plan.default.actual_limits.update!(pull_mirror_interval_seconds: 200)
+```
+
 ## Incoming emails from auto-responders
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30327) in GitLab 12.4.

doc/user/project/repository/repository_mirroring.md

@@ -11,8 +11,7 @@ Repository mirroring allows for mirroring of repositories to and from external s
 used to mirror branches, tags, and commits between repositories.
 
 A repository mirror at GitLab will be updated automatically. You can also manually trigger an update
-at most once every 5 minutes. Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/237891)
-for discussions on how to potentially reduce the delay.
+at most once every 5 minutes on GitLab.com with [the limit set by the administrator on self-managed instances](../../../administration/instance_limits.md#pull-mirroring-interval).
 
 ## Overview
 
@@ -30,7 +29,7 @@ Users with at least [Developer access](../../permissions.md) to the project can
 immediate update, unless:
 
 - The mirror is already being updated.
-- 5 minutes haven't elapsed since its last update.
+- The [limit for pull mirroring interval seconds](../../../administration/instance_limits.md#pull-mirroring-interval) has not elapsed since its last update.
 
 For security reasons, in [GitLab 12.10 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27166),
 the URL to the original repository is only displayed to users with

ee/app/services/start_pull_mirroring_service.rb

 # frozen_string_literal: true
 
 class StartPullMirroringService < BaseService
-  INTERVAL = 5.minutes
-
   def execute
     import_state = project.import_state
 
@@ -12,7 +10,7 @@ class StartPullMirroringService < BaseService
       import_state.force_import_job!
     else
       import_state.reset_retry_count if import_state.hard_failed?
-      import_state.update(next_execution_timestamp: INTERVAL.since(import_state.last_update_at))
+      import_state.update(next_execution_timestamp: interval.since(import_state.last_update_at))
     end
 
     success
@@ -20,9 +18,13 @@ class StartPullMirroringService < BaseService
 
   private
 
+  def interval
+    @interval ||= project.actual_limits.pull_mirror_interval_seconds.seconds
+  end
+
   def update_now?(import_state)
     import_state.last_successful_update_at.nil? ||
       import_state.last_update_at.nil? ||
-      import_state.last_update_at < INTERVAL.ago
+      import_state.last_update_at < interval.ago
   end
 end

ee/spec/services/start_pull_mirroring_service_spec.rb

@@ -29,33 +29,47 @@ RSpec.describe StartPullMirroringService do
 
     it_behaves_like 'force mirror update'
 
-    context 'when project mirror has been updated in the last 5 minutes' do
-      it 'schedules next execution' do
-        travel_to(Time.current) do
-          import_state.update(last_update_at: 3.minutes.ago, last_successful_update_at: 10.minutes.ago)
+    shared_examples 'mirrors using interval' do |interval_minutes:|
+      context 'when project mirror has been updated in the interval' do
+        it 'schedules next execution' do
+          freeze_time do
+            import_state.update(last_update_at: (interval_minutes - 1).minutes.ago, last_successful_update_at: 10.minutes.ago)
+
+            expect { execute }
+              .to change { import_state.next_execution_timestamp }
+              .to(interval_minutes.minutes.since(import_state.last_update_at))
+              .and not_change { UpdateAllMirrorsWorker.jobs.size }
+          end
+        end
+      end
 
-          expect { execute }
-            .to change { import_state.next_execution_timestamp }
-            .to(2.minutes.from_now)
-            .and not_change { UpdateAllMirrorsWorker.jobs.size }
+      context 'when project mirror has been updated outside of the interval' do
+        before do
+          import_state.update(last_update_at: (interval_minutes + 1).minutes.ago, last_successful_update_at: 10.minutes.ago)
         end
+
+        it_behaves_like 'force mirror update'
       end
-    end
 
-    context 'when project mirror has been updated more than 5 minutes ago' do
-      before do
-        import_state.update(last_update_at: 6.minutes.ago, last_successful_update_at: 10.minutes.ago)
+      context 'when project mirror has been updated in interval but has never been successfully updated' do
+        before do
+          import_state.update(last_update_at: (interval_minutes - 1).minutes.ago, last_successful_update_at: nil)
+        end
+
+        it_behaves_like 'force mirror update'
       end
+    end
 
-      it_behaves_like 'force mirror update'
+    context 'with default interval' do
+      it_behaves_like 'mirrors using interval', interval_minutes: 5
     end
 
-    context 'when project mirror has been updated in the last 5 minutes but has never been successfully updated' do
+    context 'with a custom interval' do
       before do
-        import_state.update(last_update_at: 3.minutes.ago, last_successful_update_at: nil)
+        Plan.default.actual_limits.update!(pull_mirror_interval_seconds: 2.minutes)
       end
 
-      it_behaves_like 'force mirror update'
+      it_behaves_like 'mirrors using interval', interval_minutes: 2
     end
   end