Merge branch '6851-geo-registry-details' into 'master'

Add Projects page under Admin > Geo Nodes to display detailed synchronization information

Closes #6851

See merge request gitlab-org/gitlab-ee!6452

app/views/layouts/nav/sidebar/_admin.html.haml

@@ -170,17 +170,27 @@
               %strong.fly-out-top-item-name
                 #{ _('Push Rules') }
 
-      = nav_link(controller: :geo_nodes) do
+      = nav_link(controller: [:geo_nodes, :geo_projects]) do
         = link_to admin_geo_nodes_path do
           .nav-icon-container
             = sprite_icon('location-dot')
           %span.nav-item-name
-            Geo Nodes
-        %ul.sidebar-sub-level-items.is-fly-out-only
-          = nav_link(controller: :geo_nodes, html_options: { class: "fly-out-top-item" } ) do
-            = link_to admin_geo_nodes_path do
-              %strong.fly-out-top-item-name
-                #{ _('Geo Nodes') }
+            #{ _('Geo Nodes') }
+        - if Gitlab::Geo.secondary?
+          %ul.sidebar-sub-level-items
+            = nav_link(controller: :geo_nodes, html_options: { class: "fly-out-top-item" } ) do
+              = link_to admin_geo_nodes_path do
+                %strong.fly-out-top-item-name
+                  #{ _('Geo Nodes') }
+            %li.divider.fly-out-top-item
+            = nav_link(path: 'geo_nodes#index') do
+              = link_to admin_geo_nodes_path, title: 'Nodes' do
+                %span
+                  #{ _('Nodes') }
+            = nav_link(path: 'geo_projects#index') do
+              = link_to admin_geo_projects_path, title: 'Projects' do
+                %span
+                  #{ _('Projects') }
 
       = nav_link(controller: :deploy_keys) do
         = link_to admin_deploy_keys_path do

config/routes/admin.rb

@@ -138,6 +138,14 @@ namespace :admin do
     end
   end
 
+  resources :geo_projects, only: [:index] do
+    member do
+      post :recheck
+      post :resync
+      post :force_redownload
+    end
+  end
+
   get '/dashboard/stats', to: 'dashboard#stats'
   ## EE-specific
 

ee/app/assets/stylesheets/pages/geo_nodes.scss

@@ -176,3 +176,76 @@
     }
   }
 }
+
+.geo-admin-projects {
+  .card-header {
+    .header-text-primary,
+    .header-text-secondary {
+      color: $gl-link-color;
+    }
+
+    .header-text-primary {
+      line-height: 28px;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+      overflow: hidden;
+    }
+  }
+
+  .btn-card-header {
+    &:hover,
+    &:focus {
+      text-decoration: none;
+    }
+
+    &.collapsed {
+      .card-collapse-icon {
+        display: none;
+      }
+    }
+
+    &:not(.collapsed) {
+      .card-expand-icon {
+        display: none;
+      }
+    }
+  }
+
+  .card-body {
+    .project-container {
+      margin-left: 0;
+      padding-left: 0;
+    }
+
+    .project-status-content {
+      &.status-type-success {
+        color: $gl-text-green;
+      }
+
+      &.status-type-failure {
+        color: $gl-text-red;
+      }
+    }
+
+    @include media-breakpoint-down(xs) {
+      .project-status-container + .project-status-container {
+        margin-top: 15px;
+      }
+    }
+  }
+
+  .errors-list {
+    li + li {
+      margin-top: 5px;
+    }
+
+    .error-icon,
+    .error-text {
+      color: $gl-text-red;
+    }
+
+    .error-text {
+      line-height: 18px;
+    }
+  }
+}

ee/app/controllers/admin/geo_projects_controller.rb

+# frozen_string_literal: true
+
+class Admin::GeoProjectsController < Admin::ApplicationController
+  before_action :check_license
+  before_action :load_registry, except: [:index]
+
+  helper ::EE::GeoHelper
+
+  def index
+    finder = ::Geo::ProjectRegistryStatusFinder.new
+
+    @registries = case params[:sync_status]
+                  when 'never'
+                    finder.never_synced_projects.page(params[:page])
+                  when 'failed'
+                    finder.failed_projects.page(params[:page])
+                  when 'pending'
+                    finder.pending_projects.page(params[:page])
+                  else
+                    finder.synced_projects.page(params[:page])
+                  end
+  end
+
+  def recheck
+    @registry.flag_repository_for_recheck!
+
+    redirect_back_or_admin_geo_projects(notice: s_('Geo|%{name} is scheduled for re-check') % { name: @registry.project.full_name })
+  end
+
+  def resync
+    @registry.flag_repository_for_resync!
+
+    redirect_back_or_admin_geo_projects(notice: s_('Geo|%{name} is scheduled for re-sync') % { name: @registry.project.full_name })
+  end
+
+  def force_redownload
+    @registry.flag_repository_for_redownload!
+
+    redirect_back_or_admin_geo_projects(notice: s_('Geo|%{name} is scheduled for forced re-download') % { name: @registry.project.full_name })
+  end
+
+  private
+
+  def check_license
+    unless Gitlab::Geo.license_allows?
+      redirect_to admin_license_path, alert: s_('Geo|You need a different license to use Geo replication')
+    end
+  end
+
+  def load_registry
+    @registry = ::Geo::ProjectRegistry.find_by_id(params[:id])
+  end
+
+  def redirect_back_or_admin_geo_projects(params)
+    redirect_back_or_default(default: admin_geo_projects_path, options: params)
+  end
+end

ee/app/finders/geo/project_registry_status_finder.rb

+# frozen_string_literal: true
+
+module Geo
+  # Finders specific for Project status listing and inspecting
+  #
+  # This finders works slightly different than the ones used to trigger
+  # synchronization, as we are concerned in filtering for displaying rather then
+  # filtering for processing.
+  class ProjectRegistryStatusFinder < RegistryFinder
+    # Returns any project registry which project is fully synced
+    #
+    # We consider fully synced any project without pending actions
+    # or failures
+    def synced_projects
+      no_repository_resync = project_registry[:resync_repository].eq(false)
+      no_repository_sync_failure = project_registry[:repository_retry_count].eq(nil)
+      repository_verified = project_registry[:repository_verification_checksum_sha].not_eq(nil)
+
+      Geo::ProjectRegistry.where(
+        no_repository_resync
+          .and(no_repository_sync_failure)
+          .and(repository_verified)
+      ).includes(project: :route).includes(project: { namespace: :route })
+    end
+
+    # Return any project registry which project is pending to update
+    #
+    # We include here only projects that have successfully synced before.
+    # We exclude projects that have tried to re-sync or re-check already and had failures
+    def pending_projects
+      no_repository_sync_failure = project_registry[:repository_retry_count].eq(nil)
+      repository_successfully_synced_before = project_registry[:last_repository_successful_sync_at].not_eq(nil)
+      repository_pending_verification = project_registry[:repository_verification_checksum_sha].eq(nil)
+      repository_without_verification_failure_before = project_registry[:last_repository_verification_failure].eq(nil)
+      flagged_for_resync = project_registry[:resync_repository].eq(true)
+
+      Geo::ProjectRegistry.where(
+        no_repository_sync_failure
+          .and(repository_successfully_synced_before)
+          .and(flagged_for_resync
+            .or(repository_pending_verification
+              .and(repository_without_verification_failure_before)))
+      ).includes(project: :route).includes(project: { namespace: :route })
+    end
+
+    # Return any project registry which project has a failure
+    #
+    # Both types of failures are included: Synchronization and Verification
+    def failed_projects
+      repository_sync_failed = project_registry[:repository_retry_count].gt(0)
+      repository_verification_failed = project_registry[:last_repository_verification_failure].not_eq(nil)
+      repository_checksum_mismatch = project_registry[:repository_checksum_mismatch].eq(true)
+
+      Geo::ProjectRegistry.where(
+        repository_sync_failed
+          .or(repository_verification_failed)
+          .or(repository_checksum_mismatch)
+      ).includes(project: :route).includes(project: { namespace: :route })
+    end
+
+    # Return any project registry that has never been fully synced
+    #
+    # We don't include projects without a corresponding ProjectRegistry
+    # for performance reasons.
+    def never_synced_projects
+      Geo::ProjectRegistry.where(last_repository_successful_sync_at: nil)
+        .includes(project: :route)
+        .includes(project: { namespace: :route })
+    end
+
+    private
+
+    def project_registry
+      Geo::ProjectRegistry.arel_table
+    end
+  end
+end

ee/app/models/ee/project.rb

@@ -23,6 +23,7 @@ module EE
       belongs_to :mirror_user, foreign_key: 'mirror_user_id', class_name: 'User'
 
       has_one :repository_state, class_name: 'ProjectRepositoryState', inverse_of: :project
+      has_one :project_registry, class_name: 'Geo::ProjectRegistry', inverse_of: :project
       has_one :push_rule, ->(project) { project&.feature_available?(:push_rules) ? all : none }
       has_one :index_status
       has_one :jenkins_service

ee/app/models/geo/project_registry.rb

@@ -21,6 +21,7 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
 
   validates :project, presence: true, uniqueness: true
 
+  scope :never_synced, -> { where(last_repository_synced_at: nil) }
   scope :dirty, -> { where(arel_table[:resync_repository].eq(true).or(arel_table[:resync_wiki].eq(true))) }
   scope :synced_repos, -> { where(resync_repository: false) }
   scope :synced_wikis, -> { where(resync_wiki: false) }
@@ -77,7 +78,12 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
   end
 
   # Must be run before fetching the repository to avoid a race condition
+  #
+  # @param [String] type must be one of the values in TYPES
+  # @see REGISTRY_TYPES
   def start_sync!(type)
+    ensure_valid_type!(type)
+
     new_count = retry_count(type) + 1
 
     update!(
@@ -86,7 +92,12 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
       "#{type}_retry_at" => next_retry_time(new_count))
   end
 
+  # Is called when synchronization finishes without any issue
+  #
+  # @param [String] type must be one of the values in TYPES
+  # @see REGISTRY_TYPES
   def finish_sync!(type, missing_on_primary = false)
+    ensure_valid_type!(type)
     update!(
       # Indicate that the sync succeeded (but separately mark as synced atomically)
       "last_#{type}_successful_sync_at" => Time.now,
@@ -104,7 +115,16 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
     mark_synced_atomically(type)
   end
 
+  # Is called when synchronization fails with an exception
+  #
+  # @param [String] type must be one of the values in TYPES
+  # @param [String] message with a human readable description of the failure
+  # @param [Exception] error the exception
+  # @param [Hash] attrs attributes to update the database with
+  # @see REGISTRY_TYPES
   def fail_sync!(type, message, error, attrs = {})
+    ensure_valid_type!(type)
+
     attrs["resync_#{type}"] = true
     attrs["last_#{type}_sync_failure"] = "#{message}: #{error.message}"
     attrs["#{type}_retry_count"] = retry_count(type) + 1
@@ -120,9 +140,13 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
   # Marks the project as dirty.
   #
   # resync_#{type}_was_scheduled_at tracks scheduled_at to avoid a race condition.
-  # See the method #mark_synced_atomically.
-  def repository_updated!(repository_updated_event, scheduled_at)
-    type = repository_updated_event.source
+  # @see #mark_synced_atomically
+  #
+  # @param [String] type must be one of the values in TYPES
+  # @param [Time] scheduled_at when it was scheduled
+  # @see REGISTRY_TYPES
+  def repository_updated!(type, scheduled_at)
+    ensure_valid_type!(type)
 
     update!(
       "resync_#{type}" => true,
@@ -144,6 +168,33 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
     project.wiki_enabled? && (never_synced_wiki? || wiki_sync_needed?(scheduled_time))
   end
 
+  # Returns whether repository is pending verification check
+  #
+  # This will check for missing verification checksum sha
+  #
+  # @return [Boolean] whether repository is pending verification
+  def repository_verification_pending?
+    self.repository_verification_checksum_sha.nil?
+  end
+
+  # Returns whether wiki is pending verification check
+  #
+  # This will check for missing verification checksum sha
+  #
+  # @return [Boolean] whether wiki is pending verification
+  def wiki_verification_pending?
+    self.wiki_verification_checksum_sha.nil?
+  end
+
+  # Returns wheter verification is pending for either wiki or repository
+  #
+  # This will check for missing verification checksum sha for both wiki and repository
+  #
+  # @return [Boolean] whether verification is pending for either wiki or repository
+  def verification_pending?
+    repository_verification_pending? || wiki_verification_pending?
+  end
+
   def syncs_since_gc
     Gitlab::Redis::SharedState.with { |redis| redis.get(fetches_since_gc_redis_key).to_i }
   end
@@ -162,7 +213,12 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
     Gitlab::Redis::SharedState.with { |redis| redis.set(fetches_since_gc_redis_key, value) }
   end
 
+  # Check if we should re-download *type*
+  #
+  # @param [String] type must be one of the values in TYPES
+  # @see REGISTRY_TYPES
   def should_be_redownloaded?(type)
+    ensure_valid_type!(type)
     return true if public_send("force_to_redownload_#{type}")  # rubocop:disable GitlabSecurity/PublicSend
 
     retry_count(type) > RETRIES_BEFORE_REDOWNLOAD
@@ -172,6 +228,37 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
     public_send("#{type}_verification_retry_count").to_i # rubocop:disable GitlabSecurity/PublicSend
   end
 
+  # Flag the repository to be re-checked
+  #
+  # This operation happens only in the database and the recheck will be triggered after by the cron job
+  def flag_repository_for_recheck!
+    self.update(repository_verification_checksum_sha: nil, last_repository_verification_failure: nil, repository_checksum_mismatch: false)
+  end
+
+  # Flag the repository to be re-synced
+  #
+  # This operation happens only in the database and the resync will be triggered after by the cron job
+  def flag_repository_for_resync!
+    repository_updated!(:repository, Time.now)
+  end
+
+  # Flag the repository to perform a full re-download
+  #
+  # This operation happens only in the database and the forced re-download will be triggered after by the cron job
+  def flag_repository_for_redownload!
+    self.update(resync_repository: true, force_to_redownload_repository: true)
+  end
+
+  # A registry becomes candidate for re-download after first failed retries
+  #
+  # This is used by the Admin > Geo Nodes > Projects UI interface to choose
+  # when to display the re-download button
+  #
+  # @return [Boolean] whether the registry is candidate for a re-download
+  def candidate_for_redownload?
+    self.repository_retry_count && self.repository_retry_count > 1
+  end
+
   private
 
   def fetches_since_gc_redis_key
@@ -200,11 +287,19 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
     last_wiki_synced_at && timestamp > last_wiki_synced_at
   end
 
+  # How many times have we retried syncing it?
+  #
+  # @param [String] type must be one of the values in TYPES
+  # @see REGISTRY_TYPES
   def retry_count(type)
     public_send("#{type}_retry_count") || -1 # rubocop:disable GitlabSecurity/PublicSend
   end
 
+  # Mark repository as synced using atomic conditions
+  #
   # @return [Boolean] whether the update was successful
+  # @param [String] type must be one of the values in TYPES
+  # @see REGISTRY_TYPES
   def mark_synced_atomically(type)
     # Indicates whether the project is dirty (needs to be synced).
     #
@@ -233,4 +328,12 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
 
     num_rows > 0
   end
+
+  # Make sure informed type is one of the allowed values
+  #
+  # @param [String] type must be one of the values in TYPES otherwise it will fail
+  # @see REGISTRY_TYPES
+  def ensure_valid_type!(type)
+    raise ArgumentError, "Invalid type: '#{type.inspect}' informed. Must be one of the following: #{REGISTRY_TYPES.map { |type| "'#{type}'" }.join(', ')}" unless REGISTRY_TYPES.include?(type.to_sym)
+  end
 end

ee/app/views/admin/geo_projects/_failed.html.haml

+- @registries.each do |project_registry|
+  .card.project-card.prepend-top-15
+    .card-header{ id: "project-#{project_registry.project.id}-header" }
+      .d-flex
+        %strong.header-text-primary.flex-fill
+          = link_to project_registry.project.full_name, admin_namespace_project_path(project_registry.project.namespace, project_registry.project)
+        - if project_registry.candidate_for_redownload?
+          = link_to(force_redownload_admin_geo_project_path(project_registry), method: :post, class: 'btn btn-outline btn-sm mr-2') do
+            = s_('Geo|Redownload')
+        = link_to(resync_admin_geo_project_path(project_registry), method: :post, class: 'btn btn-outline-primary btn-sm') do
+          = s_('Geo|Resync')
+
+    .card-body
+      .container.project-container
+        .row
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Status')
+            .project-status-content.status-type-failure
+              = s_('Geo|Failed')
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Next sync scheduled at')
+            .project-status-content
+              - if project_registry.repository_retry_at
+                = distance_of_time_in_words(Time.now, project_registry.repository_retry_at)
+              - else
+                = s_('Geo|Waiting for scheduler')
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Last sync attempt')
+            .project-status-content
+              - if project_registry.last_repository_synced_at
+                = time_ago_with_tooltip(project_registry.last_repository_synced_at, placement: 'bottom')
+              - else
+                = s_('Geo|Waiting for scheduler')
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Retry count')
+            .project-status-content
+              = project_registry.repository_retry_count.nil? ? 0 : project_registry.repository_retry_count
+
+    .project-card-errors
+      .card-header.bg-transparent.border-bottom-0.border-top
+        %button.btn.btn-link.btn-card-header.collapsed.d-flex{ type: 'button',
+          data: { toggle: 'collapse', target: "#project-errors-#{project_registry.project.id}" },
+          'aria-expanded' => 'false',
+          'aria-controls' => "project-errors-#{project_registry.project.id}" }
+          = sprite_icon('chevron-down', size: 18, css_class: 'append-right-5 card-expand-icon')
+          = sprite_icon('chevron-up', size: 18, css_class: 'append-right-5 card-collapse-icon')
+          .header-text-secondary
+            More
+      .collapse{ id: "project-errors-#{project_registry.project.id}",
+        'aria-labelledby' => "project-#{project_registry.project.id}-header" }
+        .card-body
+          .container.project-container
+            %ul.unstyled-list.errors-list
+              - if project_registry.last_repository_sync_failure
+                %li.p-0.d-flex
+                  = sprite_icon('warning', size: 18, css_class: 'error-icon')
+                  %span.error-text.prepend-left-5
+                    = s_('Geo|Synchronization failed - %{error}') % { error: project_registry.last_repository_sync_failure }
+              - if project_registry.last_repository_verification_failure
+                %li.p-0.d-flex
+                  = sprite_icon('warning', size: 18, css_class: 'error-icon')
+                  %span.error-text.prepend-left-5
+                    = s_('Geo|Verification failed - %{error}') % { error: project_registry.last_repository_verification_failure }
+
+= paginate @registries, theme: 'gitlab'

ee/app/views/admin/geo_projects/_never.html.haml

+- @registries.each do |project_registry|
+  .card.project-card.prepend-top-15
+    .card-header{ id: "project-#{project_registry.project.id}-header" }
+      .d-flex
+        %strong.header-text-primary.flex-fill
+          = link_to project_registry.project.full_name, admin_namespace_project_path(project_registry.project.namespace, project_registry.project)
+
+    .card-body
+      .container.project-container
+        .row
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Next sync scheduled at')
+            .project-status-content
+              - if project_registry.repository_retry_at
+                = distance_of_time_in_words(Time.now, project_registry.repository_retry_at)
+              - else
+                = s_('Geo|Waiting for scheduler')
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Last sync attempt')
+            .project-status-content
+              - if project_registry.last_repository_synced_at
+                = time_ago_with_tooltip(project_registry.last_repository_synced_at, placement: 'bottom')
+              - else
+                = s_('Geo|Waiting for scheduler')
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Retry counts')
+            .project-status-content
+              = project_registry.repository_retry_count
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Error message')
+            .project-status-content.font-weight-bold
+              - if project_registry
+                = project_registry.last_repository_sync_failure
+              - else
+                = s_('Geo|No errors')
+
+= paginate @registries, theme: 'gitlab'

ee/app/views/admin/geo_projects/_pending.html.haml

+- @registries.each do |project_registry|
+  .card.project-card.prepend-top-15
+    .card-header{ id: "project-#{project_registry.project.id}-header" }
+      .d-flex
+        %strong.header-text-primary.flex-fill
+          = link_to project_registry.project.full_name, admin_namespace_project_path(project_registry.project.namespace, project_registry.project)
+        - unless project_registry.verification_pending?
+          = link_to(recheck_admin_geo_project_path(project_registry), method: :post, class: 'btn btn-outline btn-sm mr-2') do
+            = s_('Geo|Recheck')
+        - unless project_registry.resync_repository?
+          = link_to(resync_admin_geo_project_path(project_registry), method: :post, class: 'btn btn-outline-primary btn-sm') do
+            = s_('Geo|Resync')
+
+    .card-body
+      .container.project-container
+        .row
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Status')
+            .project-status-content
+              - if project_registry.resync_repository?
+                = s_('Geo|Pending synchronization')
+              - elsif project_registry.verification_pending?
+                = s_('Geo|Pending verification')
+              - else
+                = s_('Geo|Unknown state') # should never reach this state, unless we introduce new behavior
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Next sync scheduled at')
+            .project-status-content
+              - if project_registry.repository_retry_at
+                = distance_of_time_in_words(Time.now, project_registry.repository_retry_at)
+              - else
+                = s_('Geo|Waiting for scheduler')
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Last sync attempt')
+            .project-status-content
+              - if project_registry.last_repository_synced_at
+                = time_ago_with_tooltip(project_registry.last_repository_synced_at, placement: 'bottom')
+              - else
+                = s_('Geo|Never')
+          .col-sm.d-sm-none.d-md-block
+
+= paginate @registries, theme: 'gitlab'

ee/app/views/admin/geo_projects/_synced.html.haml

+- @registries.each do |project_registry|
+  .card.project-card.prepend-top-15
+    .card-header{ id: "project-#{project_registry.project.id}-header" }
+      .d-flex
+        %strong.header-text-primary.flex-fill
+          = link_to project_registry.project.full_name, admin_namespace_project_path(project_registry.project.namespace, project_registry.project)
+        = link_to(recheck_admin_geo_project_path(project_registry), method: :post, class: 'btn btn-outline btn-sm mr-2') do
+          = s_('Geo|Recheck')
+        = link_to(resync_admin_geo_project_path(project_registry), method: :post, class: 'btn btn-outline-primary btn-sm') do
+          = s_('Geo|Resync')
+
+    .card-body
+      .container.project-container
+        .row
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Status')
+            .project-status-content
+              = s_('Geo|In sync')
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Last successful sync')
+            .project-status-content
+              - if project_registry.last_repository_successful_sync_at
+                = time_ago_with_tooltip(project_registry.last_repository_successful_sync_at, placement: 'bottom')
+              - else
+                = s_('Geo|Never')
+          .col-sm.project-status-container
+            .project-status-title.text-muted
+              = s_('Geo|Last time verified')
+            .project-status-content
+              - if project_registry.last_repository_check_at
+                = time_ago_with_tooltip(project_registry.last_repository_check_at, placement: 'bottom')
+              - else
+                = s_('Geo|Never')
+          .col-sm.d-sm-none.d-md-block
+
+= paginate @registries, theme: 'gitlab'

ee/app/views/admin/geo_projects/index.html.haml

+- page_title 'Projects'
+- @no_container = true
+- @content_class = "geo-admin-container geo-admin-projects"
+- params[:sync_status] ||= []
+
+%div{ class: container_class }
+  .top-area.scrolling-tabs-container.inner-page-scroll-tabs
+    %ul.nav-links.nav.nav-tabs
+      - opts = params[:sync_status].present? ? {} : { page: admin_geo_projects_path }
+      = nav_link(opts) do
+        = link_to admin_geo_projects_path do
+          = s_('Geo|Synced')
+      = nav_link(html_options: { class: active_when(params[:sync_status] == 'pending') }) do
+        = link_to admin_geo_projects_path(sync_status: 'pending') do
+          = s_('Geo|Pending')
+      = nav_link(html_options: { class: active_when(params[:sync_status] == 'failed') }) do
+        = link_to admin_geo_projects_path(sync_status: 'failed') do
+          = s_('Geo|Failed')
+      = nav_link(html_options: { class: active_when(params[:sync_status] == 'never') }) do
+        = link_to admin_geo_projects_path(sync_status: 'never') do
+          = s_('Geo|Never')
+
+  - case params[:sync_status]
+  - when 'never'
+    = render(partial: 'never')
+  - when 'failed'
+    = render(partial: 'failed')
+  - when 'pending'
+    = render(partial: 'pending')
+  - else
+    = render(partial: 'synced')

ee/changelogs/unreleased/6851-geo-registry-details.yml

+---
+title: Projects page under Admin > Geo Nodes to display detailed synchronization information
+merge_request: 6452
+author:
+type: added

ee/db/geo/migrate/20180803160048_add_synced_repositories_partial_index.rb

+# frozen_string_literal: true
+
+class AddSyncedRepositoriesPartialIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  REPOSITORY_SYNCED_INDEX_NAME = 'idx_project_registry_synced_repositories_partial'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(
+      :project_registry,
+      :last_repository_successful_sync_at,
+      where: "resync_repository = 'f' AND repository_retry_count IS NULL AND repository_verification_checksum_sha IS NOT NULL",
+      name: REPOSITORY_SYNCED_INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index_by_name(:project_registry, REPOSITORY_SYNCED_INDEX_NAME)
+  end
+end

ee/db/geo/migrate/20180806011909_add_failed_synchronizations_partial_index.rb

+# frozen_string_literal: true
+
+class AddFailedSynchronizationsPartialIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  REPOSITORY_FAILED_INDEX_NAME = 'idx_project_registry_failed_repositories_partial'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(
+      :project_registry,
+      :repository_retry_count,
+      where: "repository_retry_count > 0 OR last_repository_verification_failure IS NOT NULL OR repository_checksum_mismatch",
+      name: REPOSITORY_FAILED_INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index_by_name(:project_registry, REPOSITORY_FAILED_INDEX_NAME)
+  end
+end

ee/db/geo/migrate/20180806020615_add_pending_synchronizations_partial_index.rb

+# frozen_string_literal: true
+
+class AddPendingSynchronizationsPartialIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  REPOSITORY_PENDING_INDEX_NAME = 'idx_project_registry_pending_repositories_partial'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(
+      :project_registry,
+      :repository_retry_count,
+      where: "repository_retry_count IS NULL AND last_repository_successful_sync_at IS NOT NULL AND (resync_repository = 't' OR repository_verification_checksum_sha IS NULL AND last_repository_verification_failure IS NULL)",
+      name: REPOSITORY_PENDING_INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index_by_name(:project_registry, REPOSITORY_PENDING_INDEX_NAME)
+  end
+end

ee/db/geo/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180802215313) do
+ActiveRecord::Schema.define(version: 20180806020615) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -84,6 +84,8 @@ ActiveRecord::Schema.define(version: 20180802215313) do
     t.integer "wiki_verification_retry_count"
   end
 
+  add_index "project_registry", ["last_repository_successful_sync_at"], name: "idx_project_registry_synced_repositories_partial", where: "((resync_repository = false) AND (repository_retry_count IS NULL) AND (repository_verification_checksum_sha IS NOT NULL))", using: :btree
+  add_index "project_registry", ["last_repository_successful_sync_at"], name: "idx_unsynced_repositories_partial", where: "(last_repository_successful_sync_at IS NULL)", using: :btree
   add_index "project_registry", ["last_repository_successful_sync_at"], name: "index_project_registry_on_last_repository_successful_sync_at", using: :btree
   add_index "project_registry", ["last_repository_synced_at"], name: "index_project_registry_on_last_repository_synced_at", using: :btree
   add_index "project_registry", ["project_id"], name: "idx_project_registry_on_repo_checksums_and_failure_partial", where: "((repository_verification_checksum_sha IS NULL) AND (last_repository_verification_failure IS NULL))", using: :btree
@@ -94,6 +96,8 @@ ActiveRecord::Schema.define(version: 20180802215313) do
   add_index "project_registry", ["project_id"], name: "idx_wiki_checksum_mismatch", where: "(wiki_checksum_mismatch = true)", using: :btree
   add_index "project_registry", ["project_id"], name: "index_project_registry_on_project_id", unique: true, using: :btree
   add_index "project_registry", ["repository_retry_at"], name: "index_project_registry_on_repository_retry_at", using: :btree
+  add_index "project_registry", ["repository_retry_count"], name: "idx_project_registry_failed_repositories_partial", where: "((repository_retry_count > 0) OR (last_repository_verification_failure IS NOT NULL) OR repository_checksum_mismatch)", using: :btree
+  add_index "project_registry", ["repository_retry_count"], name: "idx_project_registry_pending_repositories_partial", where: "((repository_retry_count IS NULL) AND (last_repository_successful_sync_at IS NOT NULL) AND ((resync_repository = true) OR ((repository_verification_checksum_sha IS NULL) AND (last_repository_verification_failure IS NULL))))", using: :btree
   add_index "project_registry", ["repository_verification_checksum_sha"], name: "idx_project_registry_on_repository_checksum_sha_partial", where: "(repository_verification_checksum_sha IS NULL)", using: :btree
   add_index "project_registry", ["resync_repository"], name: "index_project_registry_on_resync_repository", using: :btree
   add_index "project_registry", ["resync_wiki"], name: "index_project_registry_on_resync_wiki", using: :btree

ee/lib/ee/gitlab/middleware/read_only/controller.rb

@@ -9,6 +9,10 @@ module EE
             'admin/geo_nodes' => %w{update}
           }.freeze
 
+          WHITELISTED_GEO_ROUTES_TRACKING_DB = {
+            'admin/geo_projects' => %w{resync recheck force_redownload}
+          }.freeze
+
           private
 
           override :whitelisted_routes
@@ -18,10 +22,16 @@ module EE
 
           def geo_node_update_route
             # Calling route_hash may be expensive. Only do it if we think there's a possible match
-            return false unless request.path =~ %r{/admin/geo_nodes}
+            return false unless request.path =~ %r{/admin/geo_}
+
+            controller = route_hash[:controller]
+            action = route_hash[:action]
 
-            ::Gitlab::Database.db_read_write? &&
-              WHITELISTED_GEO_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
+            if WHITELISTED_GEO_ROUTES[controller]&.include?(action)
+              ::Gitlab::Database.db_read_write?
+            else
+              WHITELISTED_GEO_ROUTES_TRACKING_DB[controller]&.include?(action)
+            end
           end
         end
       end

ee/lib/gitlab/geo/log_cursor/events/repository_updated_event.rb

@@ -6,7 +6,7 @@ module Gitlab
           include BaseEvent
 
           def process
-            registry.repository_updated!(event, scheduled_at)
+            registry.repository_updated!(event.source, scheduled_at)
 
             job_id = enqueue_job_if_shard_healthy(event) do
               ::Geo::ProjectSyncWorker.perform_async(event.project_id, scheduled_at)

ee/spec/controllers/admin/geo_projects_controller_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Admin::GeoProjectsController, :geo do
+  set(:admin) { create(:admin) }
+  let(:synced_registry) { create(:geo_project_registry, :synced) }
+
+  before do
+    sign_in(admin)
+  end
+
+  shared_examples 'license required' do
+    context 'without a valid license' do
+      it 'redirects to license page with a flash message' do
+        expect(subject).to redirect_to(admin_license_path)
+        expect(flash[:alert]).to include('You need a different license to use Geo replication')
+      end
+    end
+  end
+
+  describe '#index' do
+    subject { get :index }
+
+    it_behaves_like 'license required'
+
+    context 'with a valid license' do
+      before do
+        allow(Gitlab::Geo).to receive(:license_allows?).and_return(true)
+      end
+
+      it 'renders synced template when no extra get params is specified' do
+        expect(subject).to have_gitlab_http_status(200)
+        expect(subject).to render_template(:index, partial: :synced)
+      end
+
+      context 'with sync_status=pending' do
+        subject { get :index, sync_status: 'pending' }
+
+        it 'renders pending template' do
+          expect(subject).to have_gitlab_http_status(200)
+          expect(subject).to render_template(:index, partial: :pending)
+        end
+      end
+
+      context 'with sync_status=failed' do
+        subject { get :index, sync_status: 'failed' }
+
+        it 'renders failed template' do
+          expect(subject).to have_gitlab_http_status(200)
+          expect(subject).to render_template(:index, partial: :failed)
+        end
+      end
+
+      context 'with sync_status=never' do
+        subject { get :index, sync_status: 'never' }
+
+        it 'renders failed template' do
+          expect(subject).to have_gitlab_http_status(200)
+          expect(subject).to render_template(:index, partial: :never)
+        end
+      end
+    end
+  end
+
+  describe '#recheck' do
+    subject { post :recheck, id: synced_registry }
+
+    it_behaves_like 'license required'
+
+    context 'with a valid license' do
+      before do
+        allow(Gitlab::Geo).to receive(:license_allows?).and_return(true)
+      end
+
+      it 'flags registry for recheck' do
+        expect(subject).to redirect_to(admin_geo_projects_path)
+        expect(flash[:notice]).to include('is scheduled for re-check')
+        expect(synced_registry.reload.verification_pending?).to be_truthy
+      end
+    end
+  end
+
+  describe '#resync' do
+    subject { post :resync, id: synced_registry }
+
+    it_behaves_like 'license required'
+
+    context 'with a valid license' do
+      before do
+        allow(Gitlab::Geo).to receive(:license_allows?).and_return(true)
+      end
+
+      it 'flags registry for resync' do
+        expect(subject).to redirect_to(admin_geo_projects_path)
+        expect(flash[:notice]).to include('is scheduled for re-sync')
+        expect(synced_registry.reload.resync_repository?).to be_truthy
+      end
+    end
+  end
+
+  describe '#force_redownload' do
+    subject { post :force_redownload, id: synced_registry }
+
+    it_behaves_like 'license required'
+
+    context 'with a valid license' do
+      before do
+        allow(Gitlab::Geo).to receive(:license_allows?).and_return(true)
+      end
+
+      it 'flags registry for re-download' do
+        expect(subject).to redirect_to(admin_geo_projects_path)
+        expect(flash[:notice]).to include('is scheduled for forced re-download')
+        expect(synced_registry.reload.should_be_redownloaded?('repository')).to be_truthy
+      end
+    end
+  end
+end

ee/spec/factories/geo/project_registry.rb

@@ -44,13 +44,17 @@ FactoryBot.define do
     end
 
     trait :repository_sync_failed do
-      last_repository_synced_at { 5.days.ago }
-      last_repository_successful_sync_at nil
-      last_wiki_synced_at { 5.days.ago }
+      sync_failed
+
       last_wiki_successful_sync_at  { 5.days.ago }
-      resync_repository true
       resync_wiki false
-      repository_retry_count 1
+      wiki_retry_count nil
+    end
+
+    trait :existing_repository_sync_failed do
+      repository_sync_failed
+
+      last_repository_successful_sync_at { 5.days.ago }
     end
 
     trait :repository_syncing do
@@ -59,13 +63,11 @@ FactoryBot.define do
     end
 
     trait :wiki_sync_failed do
-      last_repository_synced_at { 5.days.ago }
+      sync_failed
+
       last_repository_successful_sync_at { 5.days.ago }
-      last_wiki_synced_at { 5.days.ago }
-      last_wiki_successful_sync_at nil
       resync_repository false
-      resync_wiki true
-      wiki_retry_count 2
+      repository_retry_count nil
     end
 
     trait :wiki_syncing do

ee/spec/finders/geo/project_registry_status_finder_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Geo::ProjectRegistryStatusFinder, :geo do
+  include ::EE::GeoHelpers
+
+  set(:secondary) { create(:geo_node) }
+
+  set(:synced_registry) { create(:geo_project_registry, :synced) }
+  set(:synced_and_verified_registry) { create(:geo_project_registry, :synced, :repository_verified) }
+  set(:sync_pending_registry) { create(:geo_project_registry, :synced, :repository_dirty) }
+  set(:sync_failed_registry) { create(:geo_project_registry, :existing_repository_sync_failed) }
+
+  set(:verify_outdated_registry) { create(:geo_project_registry, :synced, :repository_verification_outdated) }
+  set(:verify_failed_registry) { create(:geo_project_registry, :synced, :repository_verification_failed) }
+  set(:verify_checksum_mismatch_registry) { create(:geo_project_registry, :synced, :repository_checksum_mismatch) }
+
+  set(:never_synced_registry) { create(:geo_project_registry) }
+  set(:never_synced_registry_with_failure) { create(:geo_project_registry, :repository_sync_failed) }
+
+  subject { described_class.new(current_node: secondary) }
+
+  before do
+    skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo::Fdw.enabled?
+    stub_current_geo_node(secondary)
+  end
+
+  describe '#synced_projects' do
+    it 'returns only synced registry' do
+      result = subject.synced_projects
+
+      expect(result).to contain_exactly(synced_and_verified_registry)
+    end
+  end
+
+  describe '#pending_projects' do
+    it 'returns only pending registry' do
+      result = subject.pending_projects
+
+      expect(result).to contain_exactly(
+        synced_registry,
+        sync_pending_registry,
+        verify_outdated_registry
+      )
+    end
+  end
+
+  describe '#failed_projects' do
+    it 'returns only failed registry' do
+      result = subject.failed_projects
+
+      expect(result).to contain_exactly(
+        sync_failed_registry,
+        never_synced_registry_with_failure,
+        verify_failed_registry,
+        verify_checksum_mismatch_registry
+      )
+    end
+  end
+
+  describe '#never_synced_projects' do
+    it 'returns only never fully synced registries' do
+      result = subject.never_synced_projects
+
+      expect(result).to contain_exactly(
+        never_synced_registry,
+        never_synced_registry_with_failure
+      )
+    end
+  end
+end

ee/spec/models/geo/project_registry_spec.rb

@@ -699,7 +699,7 @@ describe Geo::ProjectRegistry do
                         repository_retry_count: 1,
                         repository_verification_retry_count: 1)
 
-        subject.repository_updated!(event, Time.now)
+        subject.repository_updated!(event.source, Time.now)
       end
 
       it 'resets sync state' do
@@ -737,7 +737,7 @@ describe Geo::ProjectRegistry do
                         wiki_retry_count: 1,
                         wiki_verification_retry_count: 1)
 
-        subject.repository_updated!(event, Time.now)
+        subject.repository_updated!(event.source, Time.now)
       end
 
       it 'resets sync state' do
@@ -762,4 +762,117 @@ describe Geo::ProjectRegistry do
       end
     end
   end
+
+  describe '#repository_verification_pending?' do
+    it 'returns true when outdated' do
+      registry = create(:geo_project_registry, :repository_verification_outdated)
+
+      expect(registry.repository_verification_pending?).to be_truthy
+    end
+
+    it 'returns true when we are missing checksum sha' do
+      registry = create(:geo_project_registry, :repository_verification_failed)
+
+      expect(registry.repository_verification_pending?).to be_truthy
+    end
+
+    it 'returns false when checksum is present' do
+      registry = create(:geo_project_registry, :repository_verified)
+
+      expect(registry.repository_verification_pending?).to be_falsey
+    end
+  end
+
+  describe '#wiki_verification_pending?' do
+    it 'returns true when outdated' do
+      registry = create(:geo_project_registry, :wiki_verification_outdated)
+
+      expect(registry.wiki_verification_pending?).to be_truthy
+    end
+
+    it 'returns true when we are missing checksum sha' do
+      registry = create(:geo_project_registry, :wiki_verification_failed)
+
+      expect(registry.wiki_verification_pending?).to be_truthy
+    end
+
+    it 'returns false when checksum is present' do
+      registry = create(:geo_project_registry, :wiki_verified)
+
+      expect(registry.wiki_verification_pending?).to be_falsey
+    end
+  end
+
+  describe 'verification_pending?' do
+    it 'returns true when either wiki or repository verification is pending' do
+      repo_registry = create(:geo_project_registry, :repository_verification_outdated)
+      wiki_registry = create(:geo_project_registry, :wiki_verification_failed)
+
+      expect(repo_registry.verification_pending?).to be_truthy
+      expect(wiki_registry.verification_pending?).to be_truthy
+    end
+
+    it 'returns false when both wiki and repository verification is present' do
+      registry = create(:geo_project_registry, :repository_verified, :wiki_verified)
+
+      expect(registry.verification_pending?).to be_falsey
+    end
+  end
+
+  describe '#flag_repository_for_recheck!' do
+    it 'modified record to a recheck state' do
+      registry = create(:geo_project_registry, :repository_verified)
+      registry.flag_repository_for_recheck!
+
+      expect(registry).to have_attributes(
+        repository_verification_checksum_sha: nil,
+        last_repository_verification_failure: nil,
+        repository_checksum_mismatch: false
+      )
+    end
+  end
+
+  describe '#flag_repository_for_resync!' do
+    it 'modified record to a resync state' do
+      registry = create(:geo_project_registry, :synced)
+      registry.flag_repository_for_resync!
+
+      expect(registry).to have_attributes(
+        resync_repository: true,
+        repository_verification_checksum_sha: nil,
+        last_repository_verification_failure: nil,
+        repository_checksum_mismatch: false,
+        repository_verification_retry_count: nil,
+        repository_retry_count: nil,
+        repository_retry_at: nil
+
+      )
+    end
+  end
+
+  describe '#flag_repository_for_redownload!' do
+    it 'modified record to a recheck state' do
+      registry = create(:geo_project_registry, :repository_verified)
+      registry.flag_repository_for_redownload!
+
+      expect(registry).to have_attributes(
+        resync_repository: true,
+        force_to_redownload_repository: true
+      )
+    end
+  end
+
+  describe '#candidate_for_redownload?' do
+    it 'returns false when repository_retry_count is 1 or less' do
+      registry = create(:geo_project_registry, :sync_failed)
+
+      expect(registry.candidate_for_redownload?).to be_falsey
+    end
+
+    it 'returns true when repository_retry_count is > 1' do
+      registry = create(:geo_project_registry, :sync_failed, repository_retry_count: 2)
+
+      expect(registry.candidate_for_redownload?).to be_truthy
+    end
+  end
 end

ee/spec/routing/admin_routing_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe 'EE-specific admin routing' do
+  describe Admin::GeoProjectsController, 'routing' do
+    let(:project_registry) { create(:geo_project_registry) }
+
+    it 'routes ../ to #index' do
+      expect(get('/admin/geo_projects')).to route_to('admin/geo_projects#index')
+    end
+
+    it 'routes ../:id/recheck to #recheck' do
+      expect(post("admin/geo_projects/#{project_registry.id}/recheck")).to route_to('admin/geo_projects#recheck', id: project_registry.id.to_s)
+    end
+
+    it 'routes ../id:/resync to #resync' do
+      expect(post("admin/geo_projects/#{project_registry.id}/resync")).to route_to('admin/geo_projects#resync', id: project_registry.id.to_s)
+    end
+
+    it 'routes ../id:/force_redownload to #force_redownload' do
+      expect(post("admin/geo_projects/#{project_registry.id}/force_redownload")).to route_to('admin/geo_projects#force_redownload', id: project_registry.id.to_s)
+    end
+  end
+end

locale/gitlab.pot

@@ -3220,33 +3220,114 @@ msgstr ""
 msgid "GeoNodes|You have configured Geo nodes using an insecure HTTP connection. We recommend the use of HTTPS."
 msgstr ""
 
+msgid "Geo|%{name} is scheduled for forced re-download"
+msgstr ""
+
+msgid "Geo|%{name} is scheduled for re-check"
+msgstr ""
+
+msgid "Geo|%{name} is scheduled for re-sync"
+msgstr ""
+
 msgid "Geo|All projects"
 msgstr ""
 
+msgid "Geo|Error message"
+msgstr ""
+
+msgid "Geo|Failed"
+msgstr ""
+
 msgid "Geo|File sync capacity"
 msgstr ""
 
 msgid "Geo|Groups to synchronize"
 msgstr ""
 
+msgid "Geo|In sync"
+msgstr ""
+
+msgid "Geo|Last successful sync"
+msgstr ""
+
+msgid "Geo|Last sync attempt"
+msgstr ""
+
+msgid "Geo|Last time verified"
+msgstr ""
+
+msgid "Geo|Never"
+msgstr ""
+
+msgid "Geo|Next sync scheduled at"
+msgstr ""
+
+msgid "Geo|No errors"
+msgstr ""
+
+msgid "Geo|Pending"
+msgstr ""
+
+msgid "Geo|Pending synchronization"
+msgstr ""
+
+msgid "Geo|Pending verification"
+msgstr ""
+
 msgid "Geo|Projects in certain groups"
 msgstr ""
 
 msgid "Geo|Projects in certain storage shards"
 msgstr ""
 
+msgid "Geo|Recheck"
+msgstr ""
+
+msgid "Geo|Redownload"
+msgstr ""
+
 msgid "Geo|Repository sync capacity"
 msgstr ""
 
+msgid "Geo|Resync"
+msgstr ""
+
+msgid "Geo|Retry count"
+msgstr ""
+
+msgid "Geo|Retry counts"
+msgstr ""
+
 msgid "Geo|Select groups to replicate."
 msgstr ""
 
 msgid "Geo|Shards to synchronize"
 msgstr ""
 
+msgid "Geo|Status"
+msgstr ""
+
+msgid "Geo|Synced"
+msgstr ""
+
+msgid "Geo|Synchronization failed - %{error}"
+msgstr ""
+
+msgid "Geo|Unknown state"
+msgstr ""
+
 msgid "Geo|Verification capacity"
 msgstr ""
 
+msgid "Geo|Verification failed - %{error}"
+msgstr ""
+
+msgid "Geo|Waiting for scheduler"
+msgstr ""
+
+msgid "Geo|You need a different license to use Geo replication"
+msgstr ""
+
 msgid "Git"
 msgstr ""
 
@@ -4515,6 +4596,9 @@ msgstr ""
 msgid "No, directly import the existing email addresses and usernames."
 msgstr ""
 
+msgid "Nodes"
+msgstr ""
+
 msgid "None"
 msgstr ""
 

spec/lib/gitlab/import_export/all_models.yml

@@ -339,6 +339,7 @@ project:
 - prometheus_alerts
 - software_license_policies
 - repository_languages
+- project_registry
 award_emoji:
 - awardable
 - user