Merge branch '356794_distribute_github_import_workload' into 'master'

Distribute GitHub import workload

See merge request gitlab-org/gitlab!83616

config/feature_flags/development/distribute_github_parallel_import.yml

+---
+name: distribute_github_parallel_import
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83616
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/356800
+milestone: '14.10'
+type: development
+group: group::source code
+default_enabled: false

lib/gitlab/github_import/parallel_scheduling.rb

@@ -209,7 +209,11 @@ module Gitlab
       # Default batch settings for parallel import (can be redefined in Importer classes)
       # Example: { size: 100, delay: 1.minute }
       def parallel_import_batch
-        {}
+        if Feature.enabled?(:distribute_github_parallel_import, default_enabled: :yaml)
+          { size: 1000, delay: 1.minute }
+        else
+          {}
+        end
       end
 
       def abort_on_failure

spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb

@@ -98,9 +98,9 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNotesImporter do
         .to receive(:each_object_to_import)
         .and_yield(github_comment)
 
-      expect(Gitlab::GithubImport::ImportDiffNoteWorker)
-        .to receive(:perform_async)
-        .with(project.id, an_instance_of(Hash), an_instance_of(String))
+      expect(Gitlab::GithubImport::ImportDiffNoteWorker).to receive(:bulk_perform_in).with(1.second, [
+          [project.id, an_instance_of(Hash), an_instance_of(String)]
+        ], batch_size: 1000, batch_delay: 1.minute)
 
       waiter = importer.parallel_import
 

spec/lib/gitlab/github_import/importer/issues_importer_spec.rb

@@ -91,9 +91,9 @@ RSpec.describe Gitlab::GithubImport::Importer::IssuesImporter do
         .to receive(:each_object_to_import)
         .and_yield(github_issue)
 
-      expect(Gitlab::GithubImport::ImportIssueWorker)
-        .to receive(:perform_async)
-        .with(project.id, an_instance_of(Hash), an_instance_of(String))
+      expect(Gitlab::GithubImport::ImportIssueWorker).to receive(:bulk_perform_in).with(1.second, [
+          [project.id, an_instance_of(Hash), an_instance_of(String)]
+        ], batch_size: 1000, batch_delay: 1.minute)
 
       waiter = importer.parallel_import
 

spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb

@@ -118,9 +118,9 @@ RSpec.describe Gitlab::GithubImport::Importer::LfsObjectsImporter do
         expect(service).to receive(:execute).and_return([lfs_download_object])
       end
 
-      expect(Gitlab::GithubImport::ImportLfsObjectWorker)
-        .to receive(:perform_async)
-        .with(project.id, an_instance_of(Hash), an_instance_of(String))
+      expect(Gitlab::GithubImport::ImportLfsObjectWorker).to receive(:bulk_perform_in).with(1.second, [
+          [project.id, an_instance_of(Hash), an_instance_of(String)]
+        ], batch_size: 1000, batch_delay: 1.minute)
 
       waiter = importer.parallel_import
 

spec/lib/gitlab/github_import/importer/notes_importer_spec.rb

@@ -84,9 +84,9 @@ RSpec.describe Gitlab::GithubImport::Importer::NotesImporter do
         .to receive(:each_object_to_import)
         .and_yield(github_comment)
 
-      expect(Gitlab::GithubImport::ImportNoteWorker)
-        .to receive(:perform_async)
-        .with(project.id, an_instance_of(Hash), an_instance_of(String))
+      expect(Gitlab::GithubImport::ImportNoteWorker).to receive(:bulk_perform_in).with(1.second, [
+          [project.id, an_instance_of(Hash), an_instance_of(String)]
+        ], batch_size: 1000, batch_delay: 1.minute)
 
       waiter = importer.parallel_import
 

spec/lib/gitlab/github_import/parallel_scheduling_spec.rb

@@ -22,10 +22,6 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do
       def collection_method
         :issues
       end
-
-      def parallel_import_batch
-        { size: 10, delay: 1.minute }
-      end
     end
   end
 
@@ -261,7 +257,7 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do
     let(:repr_class) { double(:representation) }
     let(:worker_class) { double(:worker) }
     let(:object) { double(:object) }
-    let(:batch_size) { 200 }
+    let(:batch_size) { 1000 }
     let(:batch_delay) { 1.minute }
 
     before do
@@ -281,7 +277,6 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do
 
     context 'with multiple objects' do
       before do
-        allow(importer).to receive(:parallel_import_batch) { { size: batch_size, delay: batch_delay } }
         expect(importer).to receive(:each_object_to_import).and_yield(object).and_yield(object).and_yield(object)
       end
 
@@ -295,6 +290,25 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do
         importer.parallel_import
       end
     end
+
+    context 'when distribute_github_parallel_import feature flag is disabled' do
+      before do
+        stub_feature_flags(distribute_github_parallel_import: false)
+      end
+
+      it 'imports data in parallel' do
+        expect(importer)
+          .to receive(:each_object_to_import)
+          .and_yield(object)
+
+        expect(worker_class)
+          .to receive(:perform_async)
+          .with(project.id, { title: 'Foo' }, an_instance_of(String))
+
+        expect(importer.parallel_import)
+          .to be_an_instance_of(Gitlab::JobWaiter)
+      end
+    end
   end
 
   describe '#each_object_to_import' do