Merge branch '13309_01-expose-sync-enabled-in-api' into 'master'

Expose Enabled Replication to API

See merge request gitlab-org/gitlab!28478

ee/app/models/geo/base_registry.rb

@@ -17,6 +17,10 @@ class Geo::BaseRegistry < Geo::TrackingBase
     where.not(self::MODEL_FOREIGN_KEY => ids)
   end
 
+  def self.replication_enabled?
+    true
+  end
+
   def self.insert_for_model_ids(ids)
     records = ids.map do |id|
       new(self::MODEL_FOREIGN_KEY => id, created_at: Time.zone.now)

ee/app/models/geo/container_repository_registry.rb

@@ -38,6 +38,10 @@ class Geo::ContainerRepositoryRegistry < Geo::BaseRegistry
     where(nil).pluck(:container_repository_id)
   end
 
+  def self.replication_enabled?
+    Gitlab.config.geo.registry_replication.enabled
+  end
+
   def fail_sync!(message, error)
     new_retry_count = retry_count + 1
 

ee/app/models/geo/job_artifact_registry.rb

@@ -30,4 +30,8 @@ class Geo::JobArtifactRegistry < Geo::BaseRegistry
   def self.has_create_events?
     false
   end
+
+  def self.replication_enabled?
+    JobArtifactUploader.object_store_enabled? ? Gitlab::Geo.current_node.sync_object_storage? : true
+  end
 end

ee/app/models/geo/upload_registry.rb

@@ -51,6 +51,10 @@ class Geo::UploadRegistry < Geo::BaseRegistry
     end
   end
 
+  def self.replication_enabled?
+    FileUploader.object_store_enabled? ? Gitlab::Geo.current_node.sync_object_storage? : true
+  end
+
   def file
     upload&.path || s_('Removed %{type} with id %{id}') % { type: file_type, id: file_id }
   end

ee/app/models/geo_node_status.rb

@@ -11,7 +11,9 @@ class GeoNodeStatus < ApplicationRecord
 
   attr_accessor :storage_shards
 
-  attr_accessor :repository_verification_enabled
+  attr_accessor :attachments_replication_enabled, :lfs_objects_replication_enabled, :job_artifacts_replication_enabled,
+                :container_repositories_replication_enabled, :design_repositories_replication_enabled, :repositories_replication_enabled,
+                :repository_verification_enabled
 
   # Prometheus metrics, no need to store them in the database
   attr_accessor :event_log_max_id, :repository_created_max_id, :repository_updated_max_id,
@@ -35,6 +37,7 @@ class GeoNodeStatus < ApplicationRecord
   # Be sure to keep this consistent with Prometheus naming conventions
   PROMETHEUS_METRICS = {
     db_replication_lag_seconds: 'Database replication lag (seconds)',
+    repositories_replication_enabled: 'Boolean denoting if replication is enabled for Repositories',
     repositories_count: 'Total number of repositories available on primary',
     repositories_synced_count: 'Number of repositories synced on secondary',
     repositories_failed_count: 'Number of repositories failed to sync on secondary',
@@ -50,16 +53,19 @@ class GeoNodeStatus < ApplicationRecord
     wikis_verified_count: 'Number of wikis verified on secondary',
     wikis_verification_failed_count: 'Number of wikis failed to verify on secondary',
     wikis_checksum_mismatch_count: 'Number of wikis that checksum mismatch on secondary',
+    lfs_objects_replication_enabled: 'Boolean denoting if replication is enabled for LFS Objects',
     lfs_objects_count: 'Total number of syncable LFS objects available on primary',
     lfs_objects_synced_count: 'Number of syncable LFS objects synced on secondary',
     lfs_objects_failed_count: 'Number of syncable LFS objects failed to sync on secondary',
     lfs_objects_registry_count: 'Number of LFS objects in the registry',
     lfs_objects_synced_missing_on_primary_count: 'Number of LFS objects marked as synced due to the file missing on the primary',
+    job_artifacts_replication_enabled: 'Boolean denoting if replication is enabled for Job Artifacts',
     job_artifacts_count: 'Total number of syncable job artifacts available on primary',
     job_artifacts_synced_count: 'Number of syncable job artifacts synced on secondary',
     job_artifacts_failed_count: 'Number of syncable job artifacts failed to sync on secondary',
     job_artifacts_registry_count: 'Number of job artifacts in the registry',
     job_artifacts_synced_missing_on_primary_count: 'Number of job artifacts marked as synced due to the file missing on the primary',
+    attachments_replication_enabled: 'Boolean denoting if replication is enabled for Attachments',
     attachments_count: 'Total number of syncable file attachments available on primary',
     attachments_synced_count: 'Number of syncable file attachments synced on secondary',
     attachments_failed_count: 'Number of syncable file attachments failed to sync on secondary',
@@ -88,10 +94,12 @@ class GeoNodeStatus < ApplicationRecord
     repositories_checked_failed_count: 'Number of failed repositories checked',
     repositories_retrying_verification_count: 'Number of repositories verification failures that Geo is actively trying to correct on secondary',
     wikis_retrying_verification_count: 'Number of wikis verification failures that Geo is actively trying to correct on secondary',
+    container_repositories_replication_enabled: 'Boolean denoting if replication is enabled for Container Repositories',
     container_repositories_count: 'Total number of syncable container repositories available on primary',
     container_repositories_synced_count: 'Number of syncable container repositories synced on secondary',
     container_repositories_failed_count: 'Number of syncable container repositories failed to sync on secondary',
     container_repositories_registry_count: 'Number of container repositories in the registry',
+    design_repositories_replication_enabled: 'Boolean denoting if replication is enabled for Design Repositories',
     design_repositories_count: 'Total number of syncable design repositories available on primary',
     design_repositories_synced_count: 'Number of syncable design repositories synced on secondary',
     design_repositories_failed_count: 'Number of syncable design repositories failed to sync on secondary',
@@ -152,6 +160,12 @@ class GeoNodeStatus < ApplicationRecord
   end
 
   def initialize_feature_flags
+    self.attachments_replication_enabled = Geo::UploadRegistry.replication_enabled?
+    self.container_repositories_replication_enabled = Geo::ContainerRepositoryRegistry.replication_enabled?
+    self.design_repositories_replication_enabled = Geo::DesignRegistry.replication_enabled?
+    self.job_artifacts_replication_enabled = Geo::JobArtifactRegistry.replication_enabled?
+    self.lfs_objects_replication_enabled = Geo::LfsObjectRegistry.replication_enabled?
+    self.repositories_replication_enabled = Geo::ProjectRegistry.replication_enabled?
     self.repository_verification_enabled = Gitlab::Geo.repository_verification_enabled?
   end
 
@@ -312,6 +326,8 @@ class GeoNodeStatus < ApplicationRecord
   end
 
   def load_lfs_objects_data
+    return unless lfs_objects_replication_enabled
+
     self.lfs_objects_count = lfs_objects_finder.count_syncable
     self.lfs_objects_synced_count = lfs_objects_finder.count_synced
     self.lfs_objects_failed_count = lfs_objects_finder.count_failed
@@ -320,6 +336,8 @@ class GeoNodeStatus < ApplicationRecord
   end
 
   def load_job_artifacts_data
+    return unless job_artifacts_replication_enabled
+
     self.job_artifacts_count = job_artifacts_finder.count_syncable
     self.job_artifacts_synced_count = job_artifacts_finder.count_synced
     self.job_artifacts_failed_count = job_artifacts_finder.count_failed
@@ -328,6 +346,8 @@ class GeoNodeStatus < ApplicationRecord
   end
 
   def load_attachments_data
+    return unless attachments_replication_enabled
+
     self.attachments_count = attachments_finder.count_syncable
     self.attachments_synced_count = attachments_finder.count_synced
     self.attachments_failed_count = attachments_finder.count_failed
@@ -336,6 +356,8 @@ class GeoNodeStatus < ApplicationRecord
   end
 
   def load_container_registry_data
+    return unless container_repositories_replication_enabled
+
     self.container_repositories_count = container_registry_finder.count_syncable
     self.container_repositories_synced_count = container_registry_finder.count_synced
     self.container_repositories_failed_count = container_registry_finder.count_failed
@@ -343,6 +365,8 @@ class GeoNodeStatus < ApplicationRecord
   end
 
   def load_designs_data
+    return unless design_repositories_replication_enabled
+
     self.design_repositories_count = design_registry_finder.count_syncable
     self.design_repositories_synced_count = design_registry_finder.count_synced
     self.design_repositories_failed_count = design_registry_finder.count_failed

ee/lib/ee/api/entities/geo_node_status.rb

@@ -16,6 +16,7 @@ module EE
         expose :health_status
         expose :missing_oauth_application
 
+        expose :attachments_replication_enabled
         expose :attachments_count
         expose :attachments_synced_count
         expose :attachments_failed_count
@@ -26,6 +27,7 @@ module EE
 
         expose :db_replication_lag_seconds
 
+        expose :lfs_objects_replication_enabled
         expose :lfs_objects_count
         expose :lfs_objects_synced_count
         expose :lfs_objects_failed_count
@@ -34,6 +36,7 @@ module EE
           number_to_percentage(node.lfs_objects_synced_in_percentage, precision: 2)
         end
 
+        expose :job_artifacts_replication_enabled
         expose :job_artifacts_count
         expose :job_artifacts_synced_count
         expose :job_artifacts_failed_count
@@ -42,6 +45,7 @@ module EE
           number_to_percentage(node.job_artifacts_synced_in_percentage, precision: 2)
         end
 
+        expose :container_repositories_replication_enabled
         expose :container_repositories_count
         expose :container_repositories_synced_count
         expose :container_repositories_failed_count
@@ -49,6 +53,7 @@ module EE
           number_to_percentage(node.container_repositories_synced_in_percentage, precision: 2)
         end
 
+        expose :design_repositories_replication_enabled
         expose :design_repositories_count
         expose :design_repositories_synced_count
         expose :design_repositories_failed_count
@@ -56,6 +61,7 @@ module EE
           number_to_percentage(node.design_repositories_synced_in_percentage, precision: 2)
         end
 
+        expose :repositories_replication_enabled
         expose :projects_count
 
         expose :repositories_failed_count

ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json

@@ -6,25 +6,31 @@
     "health",
     "health_status",
     "missing_oauth_application",
+    "attachments_replication_enabled",
     "attachments_count",
     "attachments_failed_count",
     "attachments_synced_count",
     "attachments_synced_missing_on_primary_count",
+    "lfs_objects_replication_enabled",
     "lfs_objects_count",
     "lfs_objects_failed_count",
     "lfs_objects_synced_count",
     "lfs_objects_synced_missing_on_primary_count",
+    "job_artifacts_replication_enabled",
     "job_artifacts_count",
     "job_artifacts_failed_count",
     "job_artifacts_synced_count",
     "job_artifacts_synced_missing_on_primary_count",
     "db_replication_lag_seconds",
+    "container_repositories_replication_enabled",
     "container_repositories_count",
     "container_repositories_failed_count",
     "container_repositories_synced_count",
+    "design_repositories_replication_enabled",
     "design_repositories_count",
     "design_repositories_failed_count",
     "design_repositories_synced_count",
+    "repositories_replication_enabled",
     "projects_count",
     "repositories_failed_count",
     "repositories_synced_count",
@@ -71,30 +77,36 @@
     "health": { "type": ["string", "null"] },
     "health_status": { "type": "string" },
     "missing_oauth_application": { "type": "boolean" },
+    "attachments_replication_enabled": { "type": "boolean" },
     "attachments_count": { "type": "integer" },
     "attachments_failed_count": { "type": ["integer", "null"] },
     "attachments_synced_count": { "type": ["integer", "null"] },
     "attachments_synced_missing_on_primary_count": { "type": ["integer", "null"] },
     "attachments_synced_in_percentage": { "type": "string" },
     "db_replication_lag_seconds": { "type": ["integer", "null"] },
+    "lfs_objects_replication_enabled": { "type": "boolean" },
     "lfs_objects_count": { "type": "integer" },
     "lfs_objects_failed_count": { "type": ["integer", "null"] },
     "lfs_objects_synced_count": { "type": ["integer", "null"] },
     "lfs_objects_synced_missing_on_primary_count": { "type": ["integer", "null"] },
     "lfs_objects_synced_in_percentage": { "type": "string" },
+    "job_artifacts_replication_enabled": { "type": "boolean" },
     "job_artifacts_count": { "type": "integer" },
     "job_artifacts_failed_count": { "type": ["integer", "null"] },
     "job_artifacts_synced_count": { "type": ["integer", "null"] },
     "job_artifacts_synced_missing_on_primary_count": { "type": ["integer", "null"] },
     "job_artifacts_synced_in_percentage": { "type": "string" },
+    "container_repositories_replication_enabled": { "type": "boolean" },
     "container_repositories_count": { "type": "integer" },
     "container_repositories_failed_count": { "type": ["integer", "null"] },
     "container_repositories_synced_count": { "type": ["integer", "null"] },
     "container_repositories_synced_in_percentage": { "type": "string" },
+    "design_repositories_replication_enabled": { "type": "boolean" },
     "design_repositories_count": { "type": "integer" },
     "design_repositories_failed_count": { "type": ["integer", "null"] },
     "design_repositories_synced_count": { "type": ["integer", "null"] },
     "design_repositories_synced_in_percentage": { "type": "string" },
+    "repositories_replication_enabled": { "type": "boolean" },
     "projects_count": { "type": "integer" },
     "repositories_failed_count": { "type": ["integer", "null"] },
     "repository_verification_enabled": { "type": "boolean" },

ee/spec/models/geo/container_repository_registry_spec.rb

@@ -73,4 +73,18 @@ describe Geo::ContainerRepositoryRegistry, :geo do
       end
     end
   end
+
+  describe '.replication_enabled?' do
+    it 'returns true when registry replication is enabled' do
+      stub_geo_setting(registry_replication: { enabled: true })
+
+      expect(Geo::ContainerRepositoryRegistry.replication_enabled?).to be_truthy
+    end
+
+    it 'returns false when registry replication is disabled' do
+      stub_geo_setting(registry_replication: { enabled: false })
+
+      expect(Geo::ContainerRepositoryRegistry.replication_enabled?).to be_falsey
+    end
+  end
 end

ee/spec/models/geo/job_artifact_registry_spec.rb

@@ -3,8 +3,36 @@
 require 'spec_helper'
 
 describe Geo::JobArtifactRegistry, :geo do
+  include EE::GeoHelpers
+
   it_behaves_like 'a BulkInsertSafe model', Geo::JobArtifactRegistry do
     let(:valid_items_for_bulk_insertion) { build_list(:geo_job_artifact_registry, 10) }
     let(:invalid_items_for_bulk_insertion) { [] } # class does not have any validations defined
   end
+
+  describe '.replication_enabled?' do
+    context 'when Object Storage is enabled' do
+      before do
+        allow(JobArtifactUploader).to receive(:object_store_enabled?).and_return(true)
+      end
+
+      it 'returns true when Geo Object Storage replication is enabled' do
+        stub_current_geo_node(double(sync_object_storage?: true))
+
+        expect(Geo::JobArtifactRegistry.replication_enabled?).to be_truthy
+      end
+
+      it 'returns false when Geo Object Storage replication is disabled' do
+        stub_current_geo_node(double(sync_object_storage?: false))
+
+        expect(Geo::JobArtifactRegistry.replication_enabled?).to be_falsey
+      end
+    end
+
+    it 'returns true when Object Storage is disabled' do
+      allow(JobArtifactUploader).to receive(:object_store_enabled?).and_return(false)
+
+      expect(Geo::JobArtifactRegistry.replication_enabled?).to be_truthy
+    end
+  end
 end

ee/spec/models/geo/upload_registry_spec.rb

@@ -3,6 +3,8 @@
 require 'spec_helper'
 
 describe Geo::UploadRegistry, :geo, :geo_fdw do
+  include EE::GeoHelpers
+
   let!(:failed) { create(:geo_upload_registry, :failed) }
   let!(:synced) { create(:geo_upload_registry) }
 
@@ -104,4 +106,28 @@ describe Geo::UploadRegistry, :geo, :geo_fdw do
       expect(failed.synchronization_state).to eq(:failed)
     end
   end
+
+  describe '.replication_enabled?' do
+    context 'when Object Storage is enabled' do
+      before do
+        allow(FileUploader).to receive(:object_store_enabled?).and_return(true)
+      end
+
+      it 'returns true when Geo Object Storage replication is enabled' do
+        stub_current_geo_node(double(sync_object_storage?: true))
+
+        expect(Geo::UploadRegistry.replication_enabled?).to be_truthy
+      end
+
+      it 'returns false when Geo Object Storage replication is disabled' do
+        stub_current_geo_node(double(sync_object_storage?: false))
+
+        expect(Geo::UploadRegistry.replication_enabled?).to be_falsey
+      end
+    end
+
+    it 'returns true when Object Storage is disabled' do
+      expect(Geo::UploadRegistry.replication_enabled?).to be_truthy
+    end
+  end
 end

ee/spec/models/geo_node_status_spec.rb

@@ -664,51 +664,117 @@ describe GeoNodeStatus, :geo, :geo_fdw do
   end
 
   describe '#container_repositories_count' do
-    it 'counts all the repositories' do
-      create(:container_repository)
-      create(:container_repository)
+    let!(:container_1) { create(:container_repository) }
+    let!(:container_2) { create(:container_repository) }
+
+    context 'when container repositories replication is active' do
+      before do
+        stub_geo_setting(registry_replication: { enabled: true })
+      end
 
+      it 'counts all the repositories' do
         expect(subject.container_repositories_count).to eq(2)
       end
     end
 
+    context 'when container repositories replication is inactive' do
+      before do
+        stub_geo_setting(registry_replication: { enabled: false })
+      end
+
+      it 'returns nil' do
+        expect(subject.container_repositories_count).to be_nil
+      end
+    end
+  end
+
   describe '#container_repositories_synced_count' do
-    it 'counts synced repositories' do
-      create(:container_repository_registry, :synced)
-      create(:container_repository_registry, :synced)
-      create(:container_repository_registry, :sync_failed)
+    let!(:container_1) { create(:container_repository_registry, :synced) }
+    let!(:container_2) { create(:container_repository_registry, :synced) }
+    let!(:container_3) { create(:container_repository_registry, :sync_failed) }
 
+    context 'when container repositories replication is active' do
+      before do
+        stub_geo_setting(registry_replication: { enabled: true })
+      end
+
+      it 'counts synced repositories' do
         expect(subject.container_repositories_synced_count).to eq(2)
       end
     end
 
+    context 'when container repositories replication is inactive' do
+      before do
+        stub_geo_setting(registry_replication: { enabled: false })
+      end
+
+      it 'returns nil' do
+        expect(subject.container_repositories_synced_count).to be_nil
+      end
+    end
+  end
+
   describe '#container_repositories_failed_count' do
-    it 'counts failed to sync repositories' do
-      create(:container_repository_registry, :synced)
-      create(:container_repository_registry, :sync_failed)
-      create(:container_repository_registry, :sync_failed)
+    let!(:container_1) { create(:container_repository_registry, :synced) }
+    let!(:container_2) { create(:container_repository_registry, :sync_failed) }
+    let!(:container_3) { create(:container_repository_registry, :sync_failed) }
 
+    context 'when container repositories replication is active' do
+      before do
+        stub_geo_setting(registry_replication: { enabled: true })
+      end
+
+      it 'counts failed to sync repositories' do
         expect(subject.container_repositories_failed_count).to eq(2)
       end
     end
 
+    context 'when container repositories replication is inactive' do
+      before do
+        stub_geo_setting(registry_replication: { enabled: false })
+      end
+
+      it 'returns nil' do
+        expect(subject.container_repositories_failed_count).to be_nil
+      end
+    end
+  end
+
   describe '#container_repositories_registry_count' do
-    it 'counts number of registries for repositories' do
-      create(:container_repository_registry, :synced)
-      create(:container_repository_registry, :sync_failed)
-      create(:container_repository_registry, :sync_failed)
-      create(:container_repository)
+    let!(:container_1) { create(:container_repository_registry, :synced) }
+    let!(:container_2) { create(:container_repository_registry, :sync_failed) }
+    let!(:container_3) { create(:container_repository_registry, :sync_failed) }
+    let!(:container_4) { create(:container_repository) }
+
+    context 'when container repositories replication is active' do
+      before do
+        stub_geo_setting(registry_replication: { enabled: true })
+      end
 
+      it 'counts number of registries for repositories' do
         expect(subject.container_repositories_registry_count).to eq(3)
       end
     end
 
+    context 'when container repositories replication is inactive' do
+      before do
+        stub_geo_setting(registry_replication: { enabled: false })
+      end
+
+      it 'returns nil' do
+        expect(subject.container_repositories_registry_count).to be_nil
+      end
+    end
+  end
+
   describe '#container_repositories_synced_in_percentage' do
-    let(:container_repository) { create(:container_repository, project: project_1) }
+    let!(:container_repository_1) { create(:container_repository, project: project_1) }
+    let!(:container_repository_2) { create(:container_repository, project: project_1) }
+    let!(:container_list) { create_list(:container_repository, 2, project: project_3) }
 
+    context 'when container repositories replication is active' do
       before do
-      create(:container_repository, project: project_1)
-      create_list(:container_repository, 2, project: project_3)
+        stub_geo_setting(registry_replication: { enabled: true })
       end
 
       it 'returns 0 when no objects are available' do
@@ -716,19 +782,27 @@ describe GeoNodeStatus, :geo, :geo_fdw do
       end
 
       it 'returns the right percentage with no group restrictions' do
-      create(:container_repository_registry, :synced, container_repository: container_repository)
+        create(:container_repository_registry, :synced, container_repository: container_repository_1)
 
         expect(subject.container_repositories_synced_in_percentage).to be_within(0.0001).of(25)
       end
 
       it 'returns the right percentage with group restrictions' do
         secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
-      create(:container_repository_registry, :synced, container_repository: container_repository)
+        create(:container_repository_registry, :synced, container_repository: container_repository_1)
 
         expect(subject.container_repositories_synced_in_percentage).to be_within(0.0001).of(50)
       end
     end
 
+    it 'when container repositories replication is inactive returns 0' do
+      stub_geo_setting(registry_replication: { enabled: false })
+      create(:container_repository_registry, :synced, container_repository: container_repository_1)
+
+      expect(subject.container_repositories_synced_in_percentage).to eq(0)
+    end
+  end
+
   describe '#design_repositories_count' do
     it 'counts all the designs' do
       create(:design)