Commit bfbfe13c authored by Zack Cuddy's avatar Zack Cuddy

Expose Enabled Replication to API

From the UI we currently do not know
if a replication is active or disabled.

Currently, we just show that nothing is synced.

However, as part of https://gitlab.com/gitlab-org/gitlab/-/issues/13309
we are going to differentiate between nothing synced
and synced disabled.

This adds the ability to do that by exposing the data through the API.
parent d28f6821
......@@ -17,6 +17,10 @@ class Geo::BaseRegistry < Geo::TrackingBase
where.not(self::MODEL_FOREIGN_KEY => ids)
end
def self.replication_enabled?
true
end
def self.insert_for_model_ids(ids)
records = ids.map do |id|
new(self::MODEL_FOREIGN_KEY => id, created_at: Time.zone.now)
......
......@@ -38,6 +38,10 @@ class Geo::ContainerRepositoryRegistry < Geo::BaseRegistry
where(nil).pluck(:container_repository_id)
end
def self.replication_enabled?
Gitlab.config.geo.registry_replication.enabled
end
def fail_sync!(message, error)
new_retry_count = retry_count + 1
......
......@@ -30,4 +30,8 @@ class Geo::JobArtifactRegistry < Geo::BaseRegistry
def self.has_create_events?
false
end
def self.replication_enabled?
JobArtifactUploader.object_store_enabled? ? Gitlab::Geo.current_node.sync_object_storage? : true
end
end
......@@ -51,6 +51,10 @@ class Geo::UploadRegistry < Geo::BaseRegistry
end
end
def self.replication_enabled?
FileUploader.object_store_enabled? ? Gitlab::Geo.current_node.sync_object_storage? : true
end
def file
upload&.path || s_('Removed %{type} with id %{id}') % { type: file_type, id: file_id }
end
......
......@@ -11,7 +11,9 @@ class GeoNodeStatus < ApplicationRecord
attr_accessor :storage_shards
attr_accessor :repository_verification_enabled
attr_accessor :attachments_replication_enabled, :lfs_objects_replication_enabled, :job_artifacts_replication_enabled,
:container_repositories_replication_enabled, :design_repositories_replication_enabled, :repositories_replication_enabled,
:repository_verification_enabled
# Prometheus metrics, no need to store them in the database
attr_accessor :event_log_max_id, :repository_created_max_id, :repository_updated_max_id,
......@@ -35,6 +37,7 @@ class GeoNodeStatus < ApplicationRecord
# Be sure to keep this consistent with Prometheus naming conventions
PROMETHEUS_METRICS = {
db_replication_lag_seconds: 'Database replication lag (seconds)',
repositories_replication_enabled: 'Boolean denoting if replication is enabled for Repositories',
repositories_count: 'Total number of repositories available on primary',
repositories_synced_count: 'Number of repositories synced on secondary',
repositories_failed_count: 'Number of repositories failed to sync on secondary',
......@@ -50,16 +53,19 @@ class GeoNodeStatus < ApplicationRecord
wikis_verified_count: 'Number of wikis verified on secondary',
wikis_verification_failed_count: 'Number of wikis failed to verify on secondary',
wikis_checksum_mismatch_count: 'Number of wikis that checksum mismatch on secondary',
lfs_objects_replication_enabled: 'Boolean denoting if replication is enabled for LFS Objects',
lfs_objects_count: 'Total number of syncable LFS objects available on primary',
lfs_objects_synced_count: 'Number of syncable LFS objects synced on secondary',
lfs_objects_failed_count: 'Number of syncable LFS objects failed to sync on secondary',
lfs_objects_registry_count: 'Number of LFS objects in the registry',
lfs_objects_synced_missing_on_primary_count: 'Number of LFS objects marked as synced due to the file missing on the primary',
job_artifacts_replication_enabled: 'Boolean denoting if replication is enabled for Job Artifacts',
job_artifacts_count: 'Total number of syncable job artifacts available on primary',
job_artifacts_synced_count: 'Number of syncable job artifacts synced on secondary',
job_artifacts_failed_count: 'Number of syncable job artifacts failed to sync on secondary',
job_artifacts_registry_count: 'Number of job artifacts in the registry',
job_artifacts_synced_missing_on_primary_count: 'Number of job artifacts marked as synced due to the file missing on the primary',
attachments_replication_enabled: 'Boolean denoting if replication is enabled for Attachments',
attachments_count: 'Total number of syncable file attachments available on primary',
attachments_synced_count: 'Number of syncable file attachments synced on secondary',
attachments_failed_count: 'Number of syncable file attachments failed to sync on secondary',
......@@ -88,10 +94,12 @@ class GeoNodeStatus < ApplicationRecord
repositories_checked_failed_count: 'Number of failed repositories checked',
repositories_retrying_verification_count: 'Number of repositories verification failures that Geo is actively trying to correct on secondary',
wikis_retrying_verification_count: 'Number of wikis verification failures that Geo is actively trying to correct on secondary',
container_repositories_replication_enabled: 'Boolean denoting if replication is enabled for Container Repositories',
container_repositories_count: 'Total number of syncable container repositories available on primary',
container_repositories_synced_count: 'Number of syncable container repositories synced on secondary',
container_repositories_failed_count: 'Number of syncable container repositories failed to sync on secondary',
container_repositories_registry_count: 'Number of container repositories in the registry',
design_repositories_replication_enabled: 'Boolean denoting if replication is enabled for Design Repositories',
design_repositories_count: 'Total number of syncable design repositories available on primary',
design_repositories_synced_count: 'Number of syncable design repositories synced on secondary',
design_repositories_failed_count: 'Number of syncable design repositories failed to sync on secondary',
......@@ -152,6 +160,12 @@ class GeoNodeStatus < ApplicationRecord
end
def initialize_feature_flags
self.attachments_replication_enabled = Geo::UploadRegistry.replication_enabled?
self.container_repositories_replication_enabled = Geo::ContainerRepositoryRegistry.replication_enabled?
self.design_repositories_replication_enabled = Geo::DesignRegistry.replication_enabled?
self.job_artifacts_replication_enabled = Geo::JobArtifactRegistry.replication_enabled?
self.lfs_objects_replication_enabled = Geo::LfsObjectRegistry.replication_enabled?
self.repositories_replication_enabled = Geo::ProjectRegistry.replication_enabled?
self.repository_verification_enabled = Gitlab::Geo.repository_verification_enabled?
end
......@@ -312,6 +326,8 @@ class GeoNodeStatus < ApplicationRecord
end
def load_lfs_objects_data
return unless lfs_objects_replication_enabled
self.lfs_objects_count = lfs_objects_finder.count_syncable
self.lfs_objects_synced_count = lfs_objects_finder.count_synced
self.lfs_objects_failed_count = lfs_objects_finder.count_failed
......@@ -320,6 +336,8 @@ class GeoNodeStatus < ApplicationRecord
end
def load_job_artifacts_data
return unless job_artifacts_replication_enabled
self.job_artifacts_count = job_artifacts_finder.count_syncable
self.job_artifacts_synced_count = job_artifacts_finder.count_synced
self.job_artifacts_failed_count = job_artifacts_finder.count_failed
......@@ -328,6 +346,8 @@ class GeoNodeStatus < ApplicationRecord
end
def load_attachments_data
return unless attachments_replication_enabled
self.attachments_count = attachments_finder.count_syncable
self.attachments_synced_count = attachments_finder.count_synced
self.attachments_failed_count = attachments_finder.count_failed
......@@ -336,6 +356,8 @@ class GeoNodeStatus < ApplicationRecord
end
def load_container_registry_data
return unless container_repositories_replication_enabled
self.container_repositories_count = container_registry_finder.count_syncable
self.container_repositories_synced_count = container_registry_finder.count_synced
self.container_repositories_failed_count = container_registry_finder.count_failed
......@@ -343,6 +365,8 @@ class GeoNodeStatus < ApplicationRecord
end
def load_designs_data
return unless design_repositories_replication_enabled
self.design_repositories_count = design_registry_finder.count_syncable
self.design_repositories_synced_count = design_registry_finder.count_synced
self.design_repositories_failed_count = design_registry_finder.count_failed
......
......@@ -16,6 +16,7 @@ module EE
expose :health_status
expose :missing_oauth_application
expose :attachments_replication_enabled
expose :attachments_count
expose :attachments_synced_count
expose :attachments_failed_count
......@@ -26,6 +27,7 @@ module EE
expose :db_replication_lag_seconds
expose :lfs_objects_replication_enabled
expose :lfs_objects_count
expose :lfs_objects_synced_count
expose :lfs_objects_failed_count
......@@ -34,6 +36,7 @@ module EE
number_to_percentage(node.lfs_objects_synced_in_percentage, precision: 2)
end
expose :job_artifacts_replication_enabled
expose :job_artifacts_count
expose :job_artifacts_synced_count
expose :job_artifacts_failed_count
......@@ -42,6 +45,7 @@ module EE
number_to_percentage(node.job_artifacts_synced_in_percentage, precision: 2)
end
expose :container_repositories_replication_enabled
expose :container_repositories_count
expose :container_repositories_synced_count
expose :container_repositories_failed_count
......@@ -49,6 +53,7 @@ module EE
number_to_percentage(node.container_repositories_synced_in_percentage, precision: 2)
end
expose :design_repositories_replication_enabled
expose :design_repositories_count
expose :design_repositories_synced_count
expose :design_repositories_failed_count
......@@ -56,6 +61,7 @@ module EE
number_to_percentage(node.design_repositories_synced_in_percentage, precision: 2)
end
expose :repositories_replication_enabled
expose :projects_count
expose :repositories_failed_count
......
......@@ -6,25 +6,31 @@
"health",
"health_status",
"missing_oauth_application",
"attachments_replication_enabled",
"attachments_count",
"attachments_failed_count",
"attachments_synced_count",
"attachments_synced_missing_on_primary_count",
"lfs_objects_replication_enabled",
"lfs_objects_count",
"lfs_objects_failed_count",
"lfs_objects_synced_count",
"lfs_objects_synced_missing_on_primary_count",
"job_artifacts_replication_enabled",
"job_artifacts_count",
"job_artifacts_failed_count",
"job_artifacts_synced_count",
"job_artifacts_synced_missing_on_primary_count",
"db_replication_lag_seconds",
"container_repositories_replication_enabled",
"container_repositories_count",
"container_repositories_failed_count",
"container_repositories_synced_count",
"design_repositories_replication_enabled",
"design_repositories_count",
"design_repositories_failed_count",
"design_repositories_synced_count",
"repositories_replication_enabled",
"projects_count",
"repositories_failed_count",
"repositories_synced_count",
......@@ -71,30 +77,36 @@
"health": { "type": ["string", "null"] },
"health_status": { "type": "string" },
"missing_oauth_application": { "type": "boolean" },
"attachments_replication_enabled": { "type": "boolean" },
"attachments_count": { "type": "integer" },
"attachments_failed_count": { "type": ["integer", "null"] },
"attachments_synced_count": { "type": ["integer", "null"] },
"attachments_synced_missing_on_primary_count": { "type": ["integer", "null"] },
"attachments_synced_in_percentage": { "type": "string" },
"db_replication_lag_seconds": { "type": ["integer", "null"] },
"lfs_objects_replication_enabled": { "type": "boolean" },
"lfs_objects_count": { "type": "integer" },
"lfs_objects_failed_count": { "type": ["integer", "null"] },
"lfs_objects_synced_count": { "type": ["integer", "null"] },
"lfs_objects_synced_missing_on_primary_count": { "type": ["integer", "null"] },
"lfs_objects_synced_in_percentage": { "type": "string" },
"job_artifacts_replication_enabled": { "type": "boolean" },
"job_artifacts_count": { "type": "integer" },
"job_artifacts_failed_count": { "type": ["integer", "null"] },
"job_artifacts_synced_count": { "type": ["integer", "null"] },
"job_artifacts_synced_missing_on_primary_count": { "type": ["integer", "null"] },
"job_artifacts_synced_in_percentage": { "type": "string" },
"container_repositories_replication_enabled": { "type": "boolean" },
"container_repositories_count": { "type": "integer" },
"container_repositories_failed_count": { "type": ["integer", "null"] },
"container_repositories_synced_count": { "type": ["integer", "null"] },
"container_repositories_synced_in_percentage": { "type": "string" },
"design_repositories_replication_enabled": { "type": "boolean" },
"design_repositories_count": { "type": "integer" },
"design_repositories_failed_count": { "type": ["integer", "null"] },
"design_repositories_synced_count": { "type": ["integer", "null"] },
"design_repositories_synced_in_percentage": { "type": "string" },
"repositories_replication_enabled": { "type": "boolean" },
"projects_count": { "type": "integer" },
"repositories_failed_count": { "type": ["integer", "null"] },
"repository_verification_enabled": { "type": "boolean" },
......
......@@ -73,4 +73,18 @@ describe Geo::ContainerRepositoryRegistry, :geo do
end
end
end
describe '.replication_enabled?' do
it 'returns true when registry replication is enabled' do
stub_geo_setting(registry_replication: { enabled: true })
expect(Geo::ContainerRepositoryRegistry.replication_enabled?).to be_truthy
end
it 'returns false when registry replication is disabled' do
stub_geo_setting(registry_replication: { enabled: false })
expect(Geo::ContainerRepositoryRegistry.replication_enabled?).to be_falsey
end
end
end
......@@ -3,8 +3,36 @@
require 'spec_helper'
describe Geo::JobArtifactRegistry, :geo do
include EE::GeoHelpers
it_behaves_like 'a BulkInsertSafe model', Geo::JobArtifactRegistry do
let(:valid_items_for_bulk_insertion) { build_list(:geo_job_artifact_registry, 10) }
let(:invalid_items_for_bulk_insertion) { [] } # class does not have any validations defined
end
describe '.replication_enabled?' do
context 'when Object Storage is enabled' do
before do
allow(JobArtifactUploader).to receive(:object_store_enabled?).and_return(true)
end
it 'returns true when Geo Object Storage replication is enabled' do
stub_current_geo_node(double(sync_object_storage?: true))
expect(Geo::JobArtifactRegistry.replication_enabled?).to be_truthy
end
it 'returns false when Geo Object Storage replication is disabled' do
stub_current_geo_node(double(sync_object_storage?: false))
expect(Geo::JobArtifactRegistry.replication_enabled?).to be_falsey
end
end
it 'returns true when Object Storage is disabled' do
allow(JobArtifactUploader).to receive(:object_store_enabled?).and_return(false)
expect(Geo::JobArtifactRegistry.replication_enabled?).to be_truthy
end
end
end
......@@ -3,6 +3,8 @@
require 'spec_helper'
describe Geo::UploadRegistry, :geo, :geo_fdw do
include EE::GeoHelpers
let!(:failed) { create(:geo_upload_registry, :failed) }
let!(:synced) { create(:geo_upload_registry) }
......@@ -104,4 +106,28 @@ describe Geo::UploadRegistry, :geo, :geo_fdw do
expect(failed.synchronization_state).to eq(:failed)
end
end
describe '.replication_enabled?' do
context 'when Object Storage is enabled' do
before do
allow(FileUploader).to receive(:object_store_enabled?).and_return(true)
end
it 'returns true when Geo Object Storage replication is enabled' do
stub_current_geo_node(double(sync_object_storage?: true))
expect(Geo::UploadRegistry.replication_enabled?).to be_truthy
end
it 'returns false when Geo Object Storage replication is disabled' do
stub_current_geo_node(double(sync_object_storage?: false))
expect(Geo::UploadRegistry.replication_enabled?).to be_falsey
end
end
it 'returns true when Object Storage is disabled' do
expect(Geo::UploadRegistry.replication_enabled?).to be_truthy
end
end
end
......@@ -664,68 +664,142 @@ describe GeoNodeStatus, :geo, :geo_fdw do
end
describe '#container_repositories_count' do
it 'counts all the repositories' do
create(:container_repository)
create(:container_repository)
let!(:container_1) { create(:container_repository) }
let!(:container_2) { create(:container_repository) }
expect(subject.container_repositories_count).to eq(2)
context 'when container repositories replication is active' do
before do
stub_geo_setting(registry_replication: { enabled: true })
end
it 'counts all the repositories' do
expect(subject.container_repositories_count).to eq(2)
end
end
context 'when container repositories replication is inactive' do
before do
stub_geo_setting(registry_replication: { enabled: false })
end
it 'returns nil' do
expect(subject.container_repositories_count).to be_nil
end
end
end
describe '#container_repositories_synced_count' do
it 'counts synced repositories' do
create(:container_repository_registry, :synced)
create(:container_repository_registry, :synced)
create(:container_repository_registry, :sync_failed)
let!(:container_1) { create(:container_repository_registry, :synced) }
let!(:container_2) { create(:container_repository_registry, :synced) }
let!(:container_3) { create(:container_repository_registry, :sync_failed) }
context 'when container repositories replication is active' do
before do
stub_geo_setting(registry_replication: { enabled: true })
end
expect(subject.container_repositories_synced_count).to eq(2)
it 'counts synced repositories' do
expect(subject.container_repositories_synced_count).to eq(2)
end
end
context 'when container repositories replication is inactive' do
before do
stub_geo_setting(registry_replication: { enabled: false })
end
it 'returns nil' do
expect(subject.container_repositories_synced_count).to be_nil
end
end
end
describe '#container_repositories_failed_count' do
it 'counts failed to sync repositories' do
create(:container_repository_registry, :synced)
create(:container_repository_registry, :sync_failed)
create(:container_repository_registry, :sync_failed)
let!(:container_1) { create(:container_repository_registry, :synced) }
let!(:container_2) { create(:container_repository_registry, :sync_failed) }
let!(:container_3) { create(:container_repository_registry, :sync_failed) }
expect(subject.container_repositories_failed_count).to eq(2)
context 'when container repositories replication is active' do
before do
stub_geo_setting(registry_replication: { enabled: true })
end
it 'counts failed to sync repositories' do
expect(subject.container_repositories_failed_count).to eq(2)
end
end
context 'when container repositories replication is inactive' do
before do
stub_geo_setting(registry_replication: { enabled: false })
end
it 'returns nil' do
expect(subject.container_repositories_failed_count).to be_nil
end
end
end
describe '#container_repositories_registry_count' do
it 'counts number of registries for repositories' do
create(:container_repository_registry, :synced)
create(:container_repository_registry, :sync_failed)
create(:container_repository_registry, :sync_failed)
create(:container_repository)
let!(:container_1) { create(:container_repository_registry, :synced) }
let!(:container_2) { create(:container_repository_registry, :sync_failed) }
let!(:container_3) { create(:container_repository_registry, :sync_failed) }
let!(:container_4) { create(:container_repository) }
context 'when container repositories replication is active' do
before do
stub_geo_setting(registry_replication: { enabled: true })
end
expect(subject.container_repositories_registry_count).to eq(3)
it 'counts number of registries for repositories' do
expect(subject.container_repositories_registry_count).to eq(3)
end
end
context 'when container repositories replication is inactive' do
before do
stub_geo_setting(registry_replication: { enabled: false })
end
it 'returns nil' do
expect(subject.container_repositories_registry_count).to be_nil
end
end
end
describe '#container_repositories_synced_in_percentage' do
let(:container_repository) { create(:container_repository, project: project_1) }
let!(:container_repository_1) { create(:container_repository, project: project_1) }
let!(:container_repository_2) { create(:container_repository, project: project_1) }
let!(:container_list) { create_list(:container_repository, 2, project: project_3) }
before do
create(:container_repository, project: project_1)
create_list(:container_repository, 2, project: project_3)
end
context 'when container repositories replication is active' do
before do
stub_geo_setting(registry_replication: { enabled: true })
end
it 'returns 0 when no objects are available' do
expect(subject.container_repositories_synced_in_percentage).to eq(0)
end
it 'returns 0 when no objects are available' do
expect(subject.container_repositories_synced_in_percentage).to eq(0)
end
it 'returns the right percentage with no group restrictions' do
create(:container_repository_registry, :synced, container_repository: container_repository)
it 'returns the right percentage with no group restrictions' do
create(:container_repository_registry, :synced, container_repository: container_repository_1)
expect(subject.container_repositories_synced_in_percentage).to be_within(0.0001).of(25)
expect(subject.container_repositories_synced_in_percentage).to be_within(0.0001).of(25)
end
it 'returns the right percentage with group restrictions' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
create(:container_repository_registry, :synced, container_repository: container_repository_1)
expect(subject.container_repositories_synced_in_percentage).to be_within(0.0001).of(50)
end
end
it 'returns the right percentage with group restrictions' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
create(:container_repository_registry, :synced, container_repository: container_repository)
it 'when container repositories replication is inactive returns 0' do
stub_geo_setting(registry_replication: { enabled: false })
create(:container_repository_registry, :synced, container_repository: container_repository_1)
expect(subject.container_repositories_synced_in_percentage).to be_within(0.0001).of(50)
expect(subject.container_repositories_synced_in_percentage).to eq(0)
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment