Check if shard configuration is same across Geo nodes

This merge request will allow the primary to compare its storage configuration
against the secondaries. Not that the status only works if you view the admin
page on the primary, since the secondaries don't have access to the primary.

How it works:

1. The primary will populate the GeoNode index page with a JSON of the current configuration
2. The primary will download the latest configuration with the `/api/v4/geo/status` endpoint
3. The JavaScript on the frontend will do a simple equality comparison between the arrays

Since the shard configuration is currently stored in `gitlab.yml`, I've
purposely tried avoided storing the shard configuration in the database to
avoid multiple sources of the truth.

Closes #3243

app/assets/javascripts/geo_nodes.js

@@ -15,6 +15,8 @@ const unknownIcon = 'fa-times';
 const notAvailable = 'Not Available';
 const versionMismatch = 'Does not match the primary node version';
 const versionMismatchClass = 'geo-node-version-mismatch';
+const storageMismatch = 'Does not match the primary storage configuration';
+const storageMismatchClass = 'geo-node-storage-mismatch';
 
 class GeoNodeStatus {
   constructor(el) {
@@ -34,11 +36,13 @@ class GeoNodeStatus {
     this.$health = $('.js-health-message', this.$status.parent());
     this.$version = $('.js-gitlab-version', this.$status);
     this.$secondaryVersion = $('.js-secondary-version', this.$status);
+    this.$secondaryStorage = $('.js-secondary-storage-shards', this.$status);
     this.endpoint = this.$el.data('status-url');
     this.$advancedStatus = $('.js-advanced-geo-node-status-toggler', this.$status.parent());
     this.$advancedStatus.on('click', GeoNodeStatus.toggleShowAdvancedStatus.bind(this));
     this.primaryVersion = $('.js-primary-version').text();
     this.primaryRevision = $('.js-primary-revision').text().replace(/\W/g, '');
+    this.primaryStorageConfiguration = $('.primary-node').data('storageShards');
 
     this.statusInterval = new SmartInterval({
       callback: this.getStatus.bind(this),
@@ -74,6 +78,15 @@ class GeoNodeStatus {
     };
   }
 
+  static sortByStorageName(config) {
+    return config.sort((a, b) => a.name.localeCompare(b.name));
+  }
+
+  static storageConfigEquals(first, second) {
+    return _.isEqual(GeoNodeStatus.sortByStorageName(first),
+                     GeoNodeStatus.sortByStorageName(second));
+  }
+
   static renderSyncGraph($itemEl, syncStats) {
     const graphItems = [
       {
@@ -211,6 +224,17 @@ class GeoNodeStatus {
       this.$secondaryVersion.text(`${status.version} (${status.revision}) - ${versionMismatch}`);
     }
 
+    if (!this.primaryStorageConfiguration || !status.storage_shards) {
+      this.$secondaryStorage.text('UNKNOWN');
+    } else if (GeoNodeStatus.storageConfigEquals(
+      this.primaryStorageConfiguration, status.storage_shards)) {
+      this.$secondaryStorage.removeClass(`${storageMismatchClass}`);
+      this.$secondaryStorage.text('OK');
+    } else {
+      this.$secondaryStorage.addClass(`${storageMismatchClass}`);
+      this.$secondaryStorage.text(storageMismatch);
+    }
+
     if (status.repositories_count > 0) {
       const repositoriesStats = GeoNodeStatus.getSyncStatistics({
         syncedCount: status.repositories_synced_count,

app/assets/stylesheets/pages/geo_nodes.scss

@@ -131,6 +131,10 @@
   color: $gl-danger;
 }
 
+.geo-node-storage-mismatch {
+  color: $gl-danger;
+}
+
 .node-badge {
   color: $white-light;
   display: inline-block;

app/models/storage_shard.rb

+# This is an in-memory structure only. The repository storage configuration is
+# in gitlab.yml and not in the database. This model makes it easier to work
+# with the configuration.
+class StorageShard
+  include ActiveModel::Model
+
+  attr_accessor :name, :path, :gitaly_address, :gitaly_token
+  attr_accessor :failure_count_threshold, :failure_reset_time, :failure_wait_time
+  attr_accessor :storage_timeout
+
+  validates :name, presence: true
+  validates :path, presence: true
+
+  # Generates an array of StorageShard objects from the currrent storage
+  # configuration using the gitlab.yml array of key/value pairs:
+  #
+  # {"default"=>{"path"=>"/home/git/repositories", ...}
+  #
+  # The key is the shard name, and the values are the parameters for that shard.
+  def self.current_shards
+    Settings.repositories.storages.map do |name, params|
+      config = params.symbolize_keys.merge(name: name)
+      StorageShard.new(config)
+    end
+  end
+end

changelogs/unreleased-ee/sh-api-shard-config.yml

+---
+title: Check if shard configuration is same across Geo nodes
+merge_request:
+author:
+type: added

ee/app/controllers/admin/geo_nodes_controller.rb

@@ -8,6 +8,7 @@ class Admin::GeoNodesController < Admin::ApplicationController
   def index
     @nodes = GeoNode.all.order(:id)
     @node = GeoNode.new
+    @current_storage_shards = StorageShardSerializer.new.represent(StorageShard.current_shards)
 
     unless Gitlab::Geo.license_allows?
       flash_now(:alert, 'You need a different license to enable Geo replication')

ee/app/models/geo_node_status.rb

@@ -4,6 +4,7 @@ class GeoNodeStatus < ActiveRecord::Base
   # Whether we were successful in reaching this node
   attr_accessor :success, :version, :revision
   attr_writer :health_status
+  attr_accessor :storage_shards
 
   # Be sure to keep this consistent with Prometheus naming conventions
   PROMETHEUS_METRICS = {
@@ -49,12 +50,12 @@ class GeoNodeStatus < ActiveRecord::Base
   def self.from_json(json_data)
     json_data.slice!(*allowed_params)
 
-    GeoNodeStatus.new(json_data)
+    GeoNodeStatus.new(HashWithIndifferentAccess.new(json_data))
   end
 
   def self.allowed_params
     excluded_params = %w(id created_at updated_at).freeze
-    extra_params = %w(success health health_status last_event_timestamp cursor_last_event_timestamp version revision).freeze
+    extra_params = %w(success health health_status last_event_timestamp cursor_last_event_timestamp version revision storage_shards).freeze
     self.column_names - excluded_params + extra_params
   end
 
@@ -74,6 +75,7 @@ class GeoNodeStatus < ActiveRecord::Base
     self.lfs_objects_count = lfs_objects_finder.count_lfs_objects
     self.attachments_count = attachments_finder.count_attachments
     self.last_successful_status_check_at = Time.now
+    self.storage_shards = StorageShard.current_shards
 
     if Gitlab::Geo.primary?
       self.replication_slots_count = geo_node.replication_slots_count

ee/app/serializers/geo_node_status_entity.rb

@@ -58,6 +58,17 @@ class GeoNodeStatusEntity < Grape::Entity
 
   expose :namespaces, using: NamespaceEntity
 
+  # We load GeoNodeStatus data in two ways:
+  #
+  # 1. Directly by asking a Geo node via an API call
+  # 2. Via cached state in the database
+  #
+  # We don't yet cached the state of the shard information in the database, so if
+  # we don't have this information omit from the serialization entirely.
+  expose :storage_shards, using: StorageShardEntity, if: ->(status, options) do
+    status.storage_shards.present?
+  end
+
   private
 
   def namespaces

ee/app/serializers/storage_shard_entity.rb

+class StorageShardEntity < Grape::Entity
+  expose :name, :path
+end

ee/app/serializers/storage_shard_serializer.rb

+class StorageShardSerializer < BaseSerializer
+  entity StorageShardEntity
+end

ee/app/views/admin/geo_nodes/index.html.haml

@@ -15,6 +15,7 @@
   %strong exact order
   they appear.
 
+- current_primary = Gitlab::Geo.primary?
 - if @nodes.any?
   .panel.panel-default
     .panel-heading
@@ -28,7 +29,10 @@
             - if node.current?
               .node-badge.current-node Current node
             - if node.primary?
-              .node-badge.primary-node Primary
+              - if node.current?
+                .node-badge.primary-node{ data: { storage_shards: @current_storage_shards.to_json } } Primary
+              - else
+                .node-badge.primary-node
               %p
                 %span.help-block Primary node
               %p
@@ -46,6 +50,13 @@
                       GitLab version:
                   %td
                     .node-info.prepend-top-5.prepend-left-5.js-secondary-version
+                - if current_primary
+                  %tr
+                    %td
+                      .help-block
+                        Storage config:
+                        %td
+                          .node-info.prepend-top-5.prepend-left-5.js-secondary-storage-shards
                 - if node.enabled?
                   %tr
                     %td

spec/ee/spec/models/geo_node_status_spec.rb

@@ -396,18 +396,26 @@ describe GeoNodeStatus, :geo do
     it_behaves_like 'timestamp parameters', :cursor_last_event_timestamp, :cursor_last_event_date
   end
 
+  describe '#storage_shards' do
+    it "returns the current node's shard config" do
+      expect(subject[:storage_shards].as_json).to eq(StorageShard.current_shards.as_json)
+    end
+  end
+
   describe '#from_json' do
     it 'returns a new GeoNodeStatus excluding parameters' do
       status = create(:geo_node_status)
 
       data = status.as_json
-      data[:id] = 10000
+      data['id'] = 10000
+      data['storage_shards'] = Settings.repositories.storages
 
       result = GeoNodeStatus.from_json(data)
 
       expect(result.id).to be_nil
       expect(result.attachments_count).to eq(status.attachments_count)
       expect(result.cursor_last_event_date).to eq(status.cursor_last_event_date)
+      expect(result.storage_shards.count).to eq(Settings.repositories.storages.count)
     end
   end
 end

spec/ee/spec/serializers/geo_node_status_entity_spec.rb

@@ -32,6 +32,7 @@ describe GeoNodeStatusEntity, :postgresql do
   it { is_expected.to have_key(:replication_slots_max_retained_wal_bytes) }
   it { is_expected.to have_key(:last_successful_status_check_timestamp) }
   it { is_expected.to have_key(:namespaces) }
+  it { is_expected.to have_key(:storage_shards) }
 
   describe '#healthy' do
     context 'when node is healthy' do
@@ -127,4 +128,11 @@ describe GeoNodeStatusEntity, :postgresql do
       expect(subject[:namespaces].first[:path]).to eq(namespace.path)
     end
   end
+
+  describe '#storage_shards' do
+    it 'returns the config' do
+      expect(subject[:storage_shards].first[:name]).to eq('default')
+      expect(subject[:storage_shards].first[:path]).to eq('/tmp/test')
+    end
+  end
 end

spec/factories/geo_node_statuses.rb

@@ -2,6 +2,7 @@ FactoryBot.define do
   factory :geo_node_status do
     sequence(:id)
     geo_node
+    storage_shards { [{ name: 'default', path: '/tmp/test' }] }
 
     trait :healthy do
       health nil

spec/fixtures/api/schemas/geo_node_status.json

@@ -62,6 +62,7 @@
     "cursor_last_event_timestamp": { "type": ["integer", "null"] },
     "last_successful_status_check_timestamp": { "type": ["integer", "null"] },
     "namespaces": { "type": "array" },
+    "storage_shards": { "type": "array" },
     "version": { "type": ["string"] },
     "revision": { "type": ["string"] }
   },

spec/models/storage_shard_spec.rb

+require 'spec_helper'
+
+describe StorageShard do
+  describe '.current_shards' do
+    it 'returns an array of StorageShard objects' do
+      shards = described_class.current_shards
+
+      expect(shards.count).to eq(Settings.repositories.storages.count)
+      expect(shards.map(&:name)).to match_array(Settings.repositories.storages.keys)
+      expect(shards.map(&:path)).to match_array(Settings.repositories.storages.values.map { |x| x.path })
+    end
+  end
+end