Merge branch...

Merge branch '197310-geo-improve-performance-of-registry-backfill-service-with-bulk-insert' into 'master'

Geo - Improve performance of RegistryBackfillService with bulk insert

Closes #197310

See merge request gitlab-org/gitlab!27720

ee/app/models/geo/base_registry.rb

 # frozen_string_literal: true
 
 class Geo::BaseRegistry < Geo::TrackingBase
+  include BulkInsertSafe
+
   self.abstract_class = true
 
   def self.pluck_model_ids_in_range(range)
@@ -15,12 +17,11 @@ class Geo::BaseRegistry < Geo::TrackingBase
     where.not(self::MODEL_FOREIGN_KEY => ids)
   end
 
-  # TODO: Investigate replacing this with bulk insert (there was an obstacle).
-  #       https://gitlab.com/gitlab-org/gitlab/issues/197310
   def self.insert_for_model_ids(ids)
-    ids.map do |id|
-      registry = create(self::MODEL_FOREIGN_KEY => id)
-      registry.id
-    end.compact
+    records = ids.map do |id|
+      new(self::MODEL_FOREIGN_KEY => id, created_at: Time.zone.now)
+    end
+
+    bulk_insert!(records, returns: :ids)
   end
 end

ee/app/models/geo/upload_registry.rb

@@ -28,13 +28,12 @@ class Geo::UploadRegistry < Geo::BaseRegistry
     false
   end
 
-  # TODO: Investigate replacing this with bulk insert (there was an obstacle).
-  #       https://gitlab.com/gitlab-org/gitlab/issues/197310
   def self.insert_for_model_ids(attrs)
-    attrs.map do |file_id, file_type|
-      registry = create(file_id: file_id, file_type: file_type)
-      registry.id
-    end.compact
+    records = attrs.map do |file_id, file_type|
+      new(file_id: file_id, file_type: file_type, created_at: Time.zone.now)
+    end
+
+    bulk_insert!(records, returns: :ids)
   end
 
   def self.with_search(query)

ee/app/services/geo/registry_consistency_service.rb

@@ -99,8 +99,8 @@ module Geo
           registry_class: registry_class.name,
           start: range.first,
           finish: range.last,
-          created: created.size,
-          failed_to_create: untracked.size - created.size
+          created: created.length,
+          failed_to_create: untracked.length - created.length
         }
       )
     end

ee/changelogs/unreleased/197310-geo-improve-performance-of-registry-backfill-service-with-bulk-.yml

+---
+title: Geo - Use bulk insert to improve performance of RegistryBackfillService
+merge_request: 27720
+author:
+type: performance

ee/spec/models/geo/container_repository_registry_spec.rb

@@ -3,6 +3,11 @@
 require 'spec_helper'
 
 describe Geo::ContainerRepositoryRegistry, :geo do
+  it_behaves_like 'a BulkInsertSafe model', Geo::ContainerRepositoryRegistry do
+    let(:valid_items_for_bulk_insertion) { build_list(:container_repository_registry, 10, created_at: Time.zone.now) }
+    let(:invalid_items_for_bulk_insertion) { [] } # class does not have any validations defined
+  end
+
   let_it_be(:registry) { create(:container_repository_registry) }
 
   describe 'relationships' do

ee/spec/models/geo/design_registry_spec.rb

@@ -3,6 +3,11 @@
 require 'spec_helper'
 
 describe Geo::DesignRegistry, :geo do
+  it_behaves_like 'a BulkInsertSafe model', Geo::DesignRegistry do
+    let(:valid_items_for_bulk_insertion) { build_list(:geo_design_registry, 10, created_at: Time.zone.now) }
+    let(:invalid_items_for_bulk_insertion) { [] } # class does not have any validations defined
+  end
+
   let!(:design_registry) { create(:geo_design_registry) }
 
   describe 'relationships' do

ee/spec/models/geo/job_artifact_registry_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Geo::JobArtifactRegistry, :geo do
+  it_behaves_like 'a BulkInsertSafe model', Geo::JobArtifactRegistry do
+    let(:valid_items_for_bulk_insertion) { build_list(:geo_job_artifact_registry, 10) }
+    let(:invalid_items_for_bulk_insertion) { [] } # class does not have any validations defined
+  end
+end

ee/spec/models/geo/lfs_object_registry_spec.rb

@@ -6,4 +6,9 @@ describe Geo::LfsObjectRegistry, :geo do
   describe 'relationships' do
     it { is_expected.to belong_to(:lfs_object).class_name('LfsObject') }
   end
+
+  it_behaves_like 'a BulkInsertSafe model', Geo::LfsObjectRegistry do
+    let(:valid_items_for_bulk_insertion) { build_list(:geo_lfs_object_registry, 10) }
+    let(:invalid_items_for_bulk_insertion) { [] } # class does not have any validations defined
+  end
 end

ee/spec/models/geo/package_file_registry_spec.rb

@@ -3,9 +3,8 @@
 require 'spec_helper'
 
 describe Geo::PackageFileRegistry, :geo, type: :model do
-  let_it_be(:registry) { create(:package_file_registry) }
-
-  specify 'factory is valid' do
-    expect(registry).to be_valid
+  it_behaves_like 'a BulkInsertSafe model', Geo::PackageFileRegistry do
+    let(:valid_items_for_bulk_insertion) { build_list(:package_file_registry, 10, created_at: Time.zone.now) }
+    let(:invalid_items_for_bulk_insertion) { [] } # class does not have any validations defined
   end
 end

ee/spec/models/geo/project_registry_spec.rb

@@ -11,6 +11,11 @@ describe Geo::ProjectRegistry do
 
   subject { registry }
 
+  it_behaves_like 'a BulkInsertSafe model', Geo::ProjectRegistry do
+    let(:valid_items_for_bulk_insertion) { build_list(:geo_project_registry, 10, created_at: Time.zone.now) }
+    let(:invalid_items_for_bulk_insertion) { [] } # class does not have any validations defined
+  end
+
   describe 'relationships' do
     it { is_expected.to belong_to(:project) }
   end

ee/spec/models/geo/upload_registry_spec.rb

@@ -6,6 +6,11 @@ describe Geo::UploadRegistry, :geo, :geo_fdw do
   let!(:failed) { create(:geo_upload_registry, :failed) }
   let!(:synced) { create(:geo_upload_registry) }
 
+  it_behaves_like 'a BulkInsertSafe model', Geo::UploadRegistry do
+    let(:valid_items_for_bulk_insertion) { build_list(:geo_upload_registry, 10, created_at: Time.zone.now) }
+    let(:invalid_items_for_bulk_insertion) { [] } # class does not have any validations defined
+  end
+
   it 'finds associated Upload record' do
     registry = create(:geo_upload_registry, :attachment, :with_file)