Show the sync information for design repositories

Adds Geo Node API and sync status in Admin panel

changelogs/unreleased/13055-show-the-sync-information-for-design-repositories.yml

+---
+title: 'Geo: Add resigns-related fields to Geo Node Status table'
+merge_request: 18379
+author:
+type: other

db/migrate/20191009100244_add_geo_design_repository_counters.rb

+# frozen_string_literal: true
+
+class AddGeoDesignRepositoryCounters < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    change_table :geo_node_statuses do |t|
+      t.column :design_repositories_count, :integer
+      t.column :design_repositories_synced_count, :integer
+      t.column :design_repositories_failed_count, :integer
+      t.column :design_repositories_registry_count, :integer
+    end
+  end
+end

db/schema.rb

@@ -1650,6 +1650,10 @@ ActiveRecord::Schema.define(version: 2019_10_17_045817) do
     t.integer "container_repositories_synced_count"
     t.integer "container_repositories_failed_count"
     t.integer "container_repositories_registry_count"
+    t.integer "design_repositories_count"
+    t.integer "design_repositories_synced_count"
+    t.integer "design_repositories_failed_count"
+    t.integer "design_repositories_registry_count"
     t.index ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true
   end
 

doc/api/geo_nodes.md

@@ -237,6 +237,10 @@ Example response:
     "container_repositories_synced_count": nil,
     "container_repositories_failed_count": nil,
     "container_repositories_synced_in_percentage": "0.00%",
+    "design_repositories_count": 3,
+    "design_repositories_synced_count": nil,
+    "design_repositories_failed_count": nil,
+    "design_repositories_synced_in_percentage": "0.00%",
     "projects_count": 41,
     "repositories_failed_count": nil,
     "repositories_synced_count": nil,
@@ -304,6 +308,10 @@ Example response:
     "container_repositories_synced_count": nil,
     "container_repositories_failed_count": nil,
     "container_repositories_synced_in_percentage": "0.00%",
+    "design_repositories_count": 3,
+    "design_repositories_synced_count": nil,
+    "design_repositories_failed_count": nil,
+    "design_repositories_synced_in_percentage": "0.00%",
     "projects_count": 41,
     "repositories_failed_count": 1,
     "repositories_synced_count": 40,
@@ -387,6 +395,10 @@ Example response:
   "container_repositories_synced_count": nil,
   "container_repositories_failed_count": nil,
   "container_repositories_synced_in_percentage": "0.00%",
+  "design_repositories_count": 3,
+  "design_repositories_synced_count": nil,
+  "design_repositories_failed_count": nil,
+  "design_repositories_synced_in_percentage": "0.00%",
   "projects_count": 41,
   "repositories_failed_count": 1,
   "repositories_synced_count": 40,

ee/app/assets/javascripts/geo_nodes/components/geo_node_detail_item.vue

@@ -79,6 +79,11 @@ export default {
       required: false,
       default: false,
     },
+    featureDisabled: {
+      type: Boolean,
+      required: false,
+      default: false,
+    },
   },
   computed: {
     hasHelpInfo() {
@@ -121,9 +126,9 @@ export default {
 </script>
 
 <template>
-  <div class="prepend-top-15 prepend-left-10 node-detail-item">
+  <div v-if="!featureDisabled" class="prepend-top-15 prepend-left-10 node-detail-item">
     <div class="node-detail-title">
-      <span> {{ itemTitle }} </span>
+      <span>{{ itemTitle }}</span>
       <icon
         v-if="hasHelpInfo"
         v-popover="popoverConfig"

ee/app/assets/javascripts/geo_nodes/components/node_detail_sections/node_details_section_sync.vue

@@ -61,6 +61,12 @@ export default {
           itemValue: this.nodeDetails.containerRepositories,
           itemValueType: VALUE_TYPE.GRAPH,
         },
+        {
+          itemTitle: s__('GeoNodes|Design repositories'),
+          itemValue: this.nodeDetails.designRepositories,
+          itemValueType: VALUE_TYPE.GRAPH,
+          featureDisabled: !gon.features.enableGeoDesignSync,
+        },
         {
           itemTitle: s__('GeoNodes|Data replication lag'),
           itemValue: this.dbReplicationLag(),
@@ -146,6 +152,7 @@ export default {
         :item-value-stale-tooltip="statusInfoStaleMessage"
         :custom-type="nodeDetailItem.customType"
         :event-type-log-status="nodeDetailItem.eventTypeLogStatus"
+        :feature-disabled="nodeDetailItem.featureDisabled"
       />
     </div>
   </div>

ee/app/assets/javascripts/geo_nodes/store/geo_nodes_store.js

@@ -124,6 +124,11 @@ export default class GeoNodesStore {
         successCount: rawNodeDetails.container_repositories_synced_count || 0,
         failureCount: rawNodeDetails.container_repositories_failed_count || 0,
       },
+      designRepositories: {
+        totalCount: rawNodeDetails.design_repositories_count || 0,
+        successCount: rawNodeDetails.design_repositories_synced_count || 0,
+        failureCount: rawNodeDetails.design_repositories_failed_count || 0,
+      },
       attachments: {
         totalCount: rawNodeDetails.attachments_count || 0,
         successCount: rawNodeDetails.attachments_synced_count || 0,

ee/app/controllers/admin/geo/nodes_controller.rb

@@ -3,6 +3,9 @@
 class Admin::Geo::NodesController < Admin::Geo::ApplicationController
   before_action :check_license!, except: :index
   before_action :load_node, only: [:edit, :update]
+  before_action only: [:index] do
+    push_frontend_feature_flag(:enable_geo_design_sync)
+  end
 
   # rubocop: disable CodeReuse/ActiveRecord
   def index

ee/app/finders/geo/design_registry_finder.rb

+# frozen_string_literal: true
+
+module Geo
+  class DesignRegistryFinder < RegistryFinder
+    def count_syncable
+      designs_repositories.count
+    end
+
+    def count_synced
+      registries_for_design_repositories
+        .merge(Geo::DesignRegistry.synced).count
+    end
+
+    def count_failed
+      registries_for_design_repositories
+        .merge(Geo::DesignRegistry.failed).count
+    end
+
+    def count_registry
+      registries_for_design_repositories.count
+    end
+
+    private
+
+    def designs_repositories
+      current_node.projects.inner_join_design_management
+    end
+
+    def registries_for_design_repositories
+      designs_repositories
+        .inner_join_design_registry
+    end
+  end
+end

ee/app/models/geo/fdw/design_management_design.rb

+# frozen_string_literal: true
+
+module Geo
+  module Fdw
+    class DesignManagementDesign < ::Geo::BaseFdw
+      self.table_name = Gitlab::Geo::Fdw.foreign_table_name('design_management_designs')
+      self.primary_key = :id
+    end
+  end
+end

ee/app/models/geo/fdw/project.rb

@@ -15,6 +15,7 @@ module Geo
       has_many :container_repositories, class_name: 'Geo::Fdw::ContainerRepository'
 
       belongs_to :namespace, class_name: 'Geo::Fdw::Namespace'
+      belongs_to :design_management_designs, class_name: 'Geo::Fdw::DesignManagementDesign'
 
       scope :outside_shards, -> (shard_names) { where.not(repository_storage: Array(shard_names)) }
 
@@ -80,6 +81,24 @@ module Geo
           joins(join_statement.join_sources)
         end
 
+        def inner_join_design_registry
+          join_statement =
+            arel_table
+              .join(Geo::DesignRegistry.arel_table, Arel::Nodes::InnerJoin)
+              .on(arel_table[:id].eq(Geo::DesignRegistry.arel_table[:project_id]))
+
+          joins(join_statement.join_sources)
+        end
+
+        def inner_join_design_management
+          join_statement =
+            arel_table
+              .join(Geo::Fdw::DesignManagementDesign.arel_table, Arel::Nodes::InnerJoin)
+              .on(arel_table[:id].eq(Geo::Fdw::DesignManagementDesign.arel_table[:project_id]))
+
+          joins(join_statement.join_sources)
+        end
+
         private
 
         def left_outer_join_project_registry

ee/app/models/geo_node_status.rb

@@ -91,7 +91,11 @@ class GeoNodeStatus < ApplicationRecord
     container_repositories_count: 'Total number of syncable container repositories available on primary',
     container_repositories_synced_count: 'Number of syncable container repositories synced on secondary',
     container_repositories_failed_count: 'Number of syncable container repositories failed to sync on secondary',
-    container_repositories_registry_count: 'Number of container repositories in the registry'
+    container_repositories_registry_count: 'Number of container repositories in the registry',
+    design_repositories_count: 'Total number of syncable design repositories available on primary',
+    design_repositories_synced_count: 'Number of syncable design repositories synced on secondary',
+    design_repositories_failed_count: 'Number of syncable design repositories failed to sync on secondary',
+    design_repositories_registry_count: 'Number of design repositories in the registry'
   }.freeze
 
   EXPIRATION_IN_MINUTES = 5
@@ -240,6 +244,7 @@ class GeoNodeStatus < ApplicationRecord
   attr_in_percentage :attachments_synced,            :attachments_synced_count,            :attachments_count
   attr_in_percentage :replication_slots_used,        :replication_slots_used_count,        :replication_slots_count
   attr_in_percentage :container_repositories_synced, :container_repositories_synced_count, :container_repositories_count
+  attr_in_percentage :design_repositories_synced,    :design_repositories_synced_count,    :design_repositories_count
 
   def storage_shards_match?
     return true if geo_node.primary?
@@ -303,6 +308,7 @@ class GeoNodeStatus < ApplicationRecord
     load_job_artifacts_data
     load_attachments_data
     load_container_registry_data
+    load_designs_data
   end
 
   def load_lfs_objects_data
@@ -330,10 +336,17 @@ class GeoNodeStatus < ApplicationRecord
   end
 
   def load_container_registry_data
-    self.container_repositories_count = container_repository_finder.count_syncable
-    self.container_repositories_synced_count = container_repository_finder.count_synced
-    self.container_repositories_failed_count = container_repository_finder.count_failed
-    self.container_repositories_registry_count = container_repository_finder.count_registry
+    self.container_repositories_count = container_registry_finder.count_syncable
+    self.container_repositories_synced_count = container_registry_finder.count_synced
+    self.container_repositories_failed_count = container_registry_finder.count_failed
+    self.container_repositories_registry_count = container_registry_finder.count_registry
+  end
+
+  def load_designs_data
+    self.design_repositories_count = design_registry_finder.count_syncable
+    self.design_repositories_synced_count = design_registry_finder.count_synced
+    self.design_repositories_failed_count = design_registry_finder.count_failed
+    self.design_repositories_registry_count = design_registry_finder.count_registry
   end
 
   def load_repository_check_data
@@ -382,8 +395,12 @@ class GeoNodeStatus < ApplicationRecord
     @job_artifacts_finder ||= Geo::JobArtifactRegistryFinder.new(current_node_id: geo_node.id)
   end
 
-  def container_repository_finder
-    @container_repository_finder ||= Geo::ContainerRepositoryRegistryFinder.new(current_node_id: geo_node.id)
+  def container_registry_finder
+    @container_registry_finder ||= Geo::ContainerRepositoryRegistryFinder.new(current_node_id: geo_node.id)
+  end
+
+  def design_registry_finder
+    @design_registry_finder ||= Geo::DesignRegistryFinder.new(current_node_id: geo_node.id)
   end
 
   def registries_for_synced_projects(type)

ee/lib/ee/api/entities.rb

@@ -649,6 +649,13 @@ module EE
           number_to_percentage(node.container_repositories_synced_in_percentage, precision: 2)
         end
 
+        expose :design_repositories_count
+        expose :design_repositories_synced_count
+        expose :design_repositories_failed_count
+        expose :design_repositories_synced_in_percentage do |node|
+          number_to_percentage(node.design_repositories_synced_in_percentage, precision: 2)
+        end
+
         expose :projects_count
 
         expose :repositories_failed_count

ee/lib/tasks/geo.rake

@@ -339,6 +339,13 @@ namespace :geo do
       puts using_percentage(current_node_status.container_repositories_synced_in_percentage)
     end
 
+    if Feature.enabled?(:enable_geo_design_sync)
+      print 'Design repositories: '.rjust(GEO_STATUS_COLUMN_WIDTH)
+      show_failed_value(current_node_status.design_repositories_failed_count)
+      print "#{current_node_status.design_repositories_synced_count || 0}/#{current_node_status.design_repositories_count || 0} "
+      puts using_percentage(current_node_status.design_repositories_synced_in_percentage)
+    end
+
     if Gitlab::CurrentSettings.repository_checks_enabled
       print 'Repositories Checked: '.rjust(GEO_STATUS_COLUMN_WIDTH)
       show_failed_value(current_node_status.repositories_checked_failed_count)

ee/spec/factories/geo/design_registry.rb

@@ -7,6 +7,10 @@ FactoryBot.define do
     last_synced_at { nil }
     state { :pending }
 
+    after(:create) do |registry, evaluator|
+      create(:design, project: registry.project)
+    end
+
     trait :synced do
       state { :synced }
       last_synced_at { 5.days.ago }

ee/spec/factories/geo_node_statuses.rb

@@ -22,6 +22,9 @@ FactoryBot.define do
       container_repositories_count { 400 }
       container_repositories_failed_count { 3 }
       container_repositories_synced_count { 200 }
+      design_repositories_count { 400 }
+      design_repositories_failed_count { 3 }
+      design_repositories_synced_count { 200 }
       projects_count { 10 }
       repositories_synced_count { 5 }
       repositories_failed_count { 0 }

ee/spec/finders/geo/design_registry_finder_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Geo::DesignRegistryFinder, :geo, :geo_fdw do
+  include ::EE::GeoHelpers
+
+  let!(:secondary) { create(:geo_node) }
+  let!(:failed_registry) { create(:geo_design_registry, :sync_failed) }
+  let!(:synced_registry) { create(:geo_design_registry, :synced) }
+
+  let(:synced_group) { create(:group) }
+  let(:unsynced_group) { create(:group) }
+  let(:synced_project) { create(:project, group: synced_group) }
+  let(:unsynced_project) { create(:project, :broken_storage, group: unsynced_group) }
+
+  subject { described_class.new(current_node_id: secondary.id) }
+
+  before do
+    stub_current_geo_node(secondary)
+  end
+
+  context 'count all the things' do
+    describe '#count_syncable' do
+      it 'returns number of container repositories' do
+        result = subject.count_syncable
+
+        expect(result).to eq(2)
+      end
+    end
+
+    describe '#count_synced' do
+      it 'returns only synced registry' do
+        result = subject.count_synced
+
+        expect(result).to eq(1)
+      end
+    end
+
+    describe '#count_failed' do
+      it 'returns only failed registry' do
+        result = subject.count_failed
+
+        expect(result).to eq(1)
+      end
+    end
+
+    describe '#count_registry' do
+      it 'returns number of all registries' do
+        result = subject.count_registry
+
+        expect(result).to eq(2)
+      end
+    end
+
+    context 'selective sync' do
+      let(:unsynced_project2) { create(:project, group: unsynced_group) }
+      let(:synced_project2) { create(:project, group: synced_group) }
+
+      before do
+        create(:geo_design_registry, :synced, project: synced_project)
+        create(:geo_design_registry, :sync_failed, project: synced_project2)
+        create(:geo_design_registry, :synced, project: unsynced_project)
+        create(:geo_design_registry, :sync_failed, project: unsynced_project2)
+
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
+      end
+
+      context 'count all the things' do
+        describe '#count_syncable' do
+          it 'returns number of container repositories' do
+            result = subject.count_syncable
+
+            expect(result).to eq(2)
+          end
+        end
+
+        describe '#count_synced' do
+          it 'returns only synced registry' do
+            result = subject.count_synced
+
+            expect(result).to eq(1)
+          end
+        end
+
+        describe '#count_failed' do
+          it 'returns only failed registry' do
+            result = subject.count_failed
+
+            expect(result).to eq(1)
+          end
+        end
+
+        describe '#count_registry' do
+          it 'returns number of all registries' do
+            result = subject.count_registry
+
+            expect(result).to eq(2)
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json

@@ -22,6 +22,9 @@
     "container_repositories_count",
     "container_repositories_failed_count",
     "container_repositories_synced_count",
+    "design_repositories_count",
+    "design_repositories_failed_count",
+    "design_repositories_synced_count",
     "projects_count",
     "repositories_failed_count",
     "repositories_synced_count",
@@ -88,6 +91,10 @@
     "container_repositories_failed_count": { "type": ["integer", "null"] },
     "container_repositories_synced_count": { "type": ["integer", "null"] },
     "container_repositories_synced_in_percentage": { "type": "string" },
+    "design_repositories_count": { "type": "integer" },
+    "design_repositories_failed_count": { "type": ["integer", "null"] },
+    "design_repositories_synced_count": { "type": ["integer", "null"] },
+    "design_repositories_synced_in_percentage": { "type": "string" },
     "projects_count": { "type": "integer" },
     "repositories_failed_count": { "type": ["integer", "null"] },
     "repository_verification_enabled": { "type": "boolean" },

ee/spec/javascripts/geo_nodes/components/geo_node_detail_item_spec.js

@@ -117,5 +117,14 @@ describe('GeoNodeDetailItemComponent', () => {
       expect(vm.$el.querySelectorAll('.event-status-timestamp').length).not.toBe(0);
       vm.$destroy();
     });
+
+    it('does not render if featureDisabled is true', () => {
+      const vm = createComponent({
+        featureDisabled: true,
+      });
+
+      expect(vm.$el.innerHTML).toBeUndefined();
+      vm.$destroy();
+    });
   });
 });

ee/spec/javascripts/geo_nodes/components/node_detail_sections/node_details_section_sync_spec.js

@@ -16,6 +16,7 @@ describe('NodeDetailsSectionSync', () => {
   let vm;
 
   beforeEach(() => {
+    gon.features = gon.features || {};
     vm = createComponent();
   });
 

ee/spec/javascripts/geo_nodes/mock_data.js

@@ -83,6 +83,10 @@ export const rawMockNodeDetails = {
   container_repositories_synced_count: 0,
   container_repositories_failed_count: 0,
   container_repositories_synced_in_percentage: '0.00%',
+  design_repositories_count: 0,
+  design_repositories_synced_count: 0,
+  design_repositories_failed_count: 0,
+  design_repositories_synced_in_percentage: '0.00%',
   repositories_failed_count: 0,
   repositories_synced_count: 12,
   repositories_synced_in_percentage: '100.00%',
@@ -193,6 +197,11 @@ export const mockNodeDetails = {
     successCount: 0,
     failureCount: 0,
   },
+  designRepositories: {
+    totalCount: 0,
+    successCount: 0,
+    failureCount: 0,
+  },
   attachments: {
     totalCount: 0,
     successCount: 0,

ee/spec/lib/ee/api/entities/geo_node_status_spec.rb

@@ -97,6 +97,18 @@ describe EE::API::Entities::GeoNodeStatus do
     end
   end
 
+  describe '#design_repositories_synced_in_percentage' do
+    it 'formats as percentage' do
+      geo_node_status.assign_attributes(
+        design_repositories_count: 256,
+        design_repositories_failed_count: 12,
+        design_repositories_synced_count: 123
+      )
+
+      expect(subject[:design_repositories_synced_in_percentage]).to eq '48.05%'
+    end
+  end
+
   describe '#repositories_synced_in_percentage' do
     it 'formats as percentage' do
       geo_node_status.assign_attributes(projects_count: 10,

ee/spec/models/geo_node_status_spec.rb

@@ -705,6 +705,67 @@ describe GeoNodeStatus, :geo, :geo_fdw do
     end
   end
 
+  describe '#design_repositories_count' do
+    it 'counts all the designs' do
+      create(:design)
+      create(:design)
+
+      expect(subject.design_repositories_count).to eq(2)
+    end
+  end
+
+  describe '#design_repositories_synced_count' do
+    it 'counts synced repositories' do
+      create(:geo_design_registry, :synced)
+      create(:geo_design_registry, :sync_failed)
+
+      expect(subject.design_repositories_synced_count).to eq(1)
+    end
+  end
+
+  describe '#design_repositories_failed_count' do
+    it 'counts failed to sync repositories' do
+      create(:geo_design_registry, :sync_failed)
+      create(:geo_design_registry, :synced)
+
+      expect(subject.design_repositories_failed_count).to eq(1)
+    end
+  end
+
+  describe '#design_repositories_registry_count' do
+    it 'counts number of registries for repositories' do
+      create(:geo_design_registry, :sync_failed)
+      create(:geo_design_registry)
+      create(:geo_design_registry, :synced)
+
+      expect(subject.design_repositories_registry_count).to eq(3)
+    end
+  end
+
+  describe '#design_repositories_synced_in_percentage' do
+    it 'returns 0 when no objects are available' do
+      expect(subject.design_repositories_synced_in_percentage).to eq(0)
+    end
+
+    it 'returns the right percentage with no group restrictions' do
+      create(:geo_design_registry, :synced)
+      create(:geo_design_registry, :sync_failed)
+
+      expect(subject.design_repositories_synced_in_percentage).to be_within(0.0001).of(50)
+    end
+
+    it 'returns the right percentage with group restrictions' do
+      secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
+
+      create(:geo_design_registry, :synced, project: project_1)
+      create(:geo_design_registry, :sync_failed, project: project_2)
+      create(:geo_design_registry, :sync_failed, project: project_3)
+      create(:geo_design_registry, :sync_failed, project: project_4)
+
+      expect(subject.design_repositories_synced_in_percentage).to be_within(0.0001).of(50)
+    end
+  end
+
   describe '#repositories_verified_count' do
     before do
       stub_current_geo_node(secondary)

ee/spec/requests/api/geo_spec.rb

@@ -270,6 +270,9 @@ describe API::Geo do
         container_repositories_count: 100,
         container_repositories_synced_count: 50,
         container_repositories_failed_count: 12,
+        design_repositories_count: 100,
+        design_repositories_synced_count: 50,
+        design_repositories_failed_count: 12,
         attachments_count: 30,
         attachments_synced_count: 30,
         attachments_failed_count: 25,

ee/spec/services/geo/design_repository_sync_service_spec.rb

@@ -40,6 +40,9 @@ describe Geo::DesignRepositorySyncService do
 
       allow_any_instance_of(Geo::ProjectHousekeepingService).to receive(:execute)
         .and_return(nil)
+
+      allow_any_instance_of(Users::RefreshAuthorizedProjectsService).to receive(:execute)
+        .and_return(nil)
     end
 
     include_context 'lease handling'

ee/spec/services/geo/metrics_update_service_spec.rb

@@ -33,6 +33,9 @@ describe Geo::MetricsUpdateService, :geo, :prometheus do
       container_repositories_count: 100,
       container_repositories_synced_count: 50,
       container_repositories_failed_count: 12,
+      design_repositories_count: 100,
+      design_repositories_synced_count: 50,
+      design_repositories_failed_count: 12,
       attachments_count: 30,
       attachments_synced_count: 30,
       attachments_failed_count: 25,

locale/gitlab.pot

@@ -7493,6 +7493,9 @@ msgstr ""
 msgid "GeoNodes|Data replication lag"
 msgstr ""
 
+msgid "GeoNodes|Design repositories"
+msgstr ""
+
 msgid "GeoNodes|Does not match the primary storage configuration"
 msgstr ""