Merge branch 'da-create-reset-checksum-event' into 'master'

Add an event to reset checksums on Geo secondary nodes

See merge request gitlab-org/gitlab-ee!7394

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180907015926) do
+ActiveRecord::Schema.define(version: 20180917214204) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1085,6 +1085,7 @@ ActiveRecord::Schema.define(version: 20180907015926) do
     t.integer "hashed_storage_attachments_event_id", limit: 8
     t.integer "job_artifact_deleted_event_id", limit: 8
     t.integer "upload_deleted_event_id", limit: 8
+    t.integer "reset_checksum_event_id", limit: 8
   end
 
   add_index "geo_event_log", ["repositories_changed_event_id"], name: "index_geo_event_log_on_repositories_changed_event_id", using: :btree
@@ -1092,6 +1093,7 @@ ActiveRecord::Schema.define(version: 20180907015926) do
   add_index "geo_event_log", ["repository_deleted_event_id"], name: "index_geo_event_log_on_repository_deleted_event_id", using: :btree
   add_index "geo_event_log", ["repository_renamed_event_id"], name: "index_geo_event_log_on_repository_renamed_event_id", using: :btree
   add_index "geo_event_log", ["repository_updated_event_id"], name: "index_geo_event_log_on_repository_updated_event_id", using: :btree
+  add_index "geo_event_log", ["reset_checksum_event_id"], name: "index_geo_event_log_on_reset_checksum_event_id", using: :btree
 
   create_table "geo_hashed_storage_attachments_events", id: :bigserial, force: :cascade do |t|
     t.integer "project_id", null: false
@@ -1263,6 +1265,12 @@ ActiveRecord::Schema.define(version: 20180907015926) do
   add_index "geo_repository_updated_events", ["project_id"], name: "index_geo_repository_updated_events_on_project_id", using: :btree
   add_index "geo_repository_updated_events", ["source"], name: "index_geo_repository_updated_events_on_source", using: :btree
 
+  create_table "geo_reset_checksum_events", id: :bigserial, force: :cascade do |t|
+    t.integer "project_id", null: false
+  end
+
+  add_index "geo_reset_checksum_events", ["project_id"], name: "index_geo_reset_checksum_events_on_project_id", using: :btree
+
   create_table "geo_upload_deleted_events", id: :bigserial, force: :cascade do |t|
     t.integer "upload_id", null: false
     t.string "file_path", null: false
@@ -3079,6 +3087,7 @@ ActiveRecord::Schema.define(version: 20180907015926) do
   add_foreign_key "geo_event_log", "geo_repository_deleted_events", column: "repository_deleted_event_id", name: "fk_c4b1c1f66e", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_renamed_events", column: "repository_renamed_event_id", name: "fk_86c84214ec", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", on_delete: :cascade
+  add_foreign_key "geo_event_log", "geo_reset_checksum_events", column: "reset_checksum_event_id", name: "fk_cff7185ad2", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_upload_deleted_events", column: "upload_deleted_event_id", name: "fk_c1f241c70d", on_delete: :cascade
   add_foreign_key "geo_hashed_storage_attachments_events", "projects", on_delete: :cascade
   add_foreign_key "geo_hashed_storage_migrated_events", "projects", on_delete: :cascade
@@ -3089,6 +3098,7 @@ ActiveRecord::Schema.define(version: 20180907015926) do
   add_foreign_key "geo_repository_created_events", "projects", on_delete: :cascade
   add_foreign_key "geo_repository_renamed_events", "projects", on_delete: :cascade
   add_foreign_key "geo_repository_updated_events", "projects", on_delete: :cascade
+  add_foreign_key "geo_reset_checksum_events", "projects", on_delete: :cascade
   add_foreign_key "gpg_key_subkeys", "gpg_keys", on_delete: :cascade
   add_foreign_key "gpg_keys", "users", on_delete: :cascade
   add_foreign_key "gpg_signatures", "gpg_key_subkeys", on_delete: :nullify

ee/app/models/geo/event_log.rb

@@ -8,6 +8,7 @@ module Geo
                        Geo::RepositoryDeletedEvent
                        Geo::RepositoryRenamedEvent
                        Geo::RepositoriesChangedEvent
+                       Geo::ResetChecksumEvent
                        Geo::HashedStorageMigratedEvent
                        Geo::HashedStorageAttachmentsEvent
                        Geo::LfsObjectDeletedEvent
@@ -54,6 +55,10 @@ module Geo
       class_name: 'Geo::UploadDeletedEvent',
       foreign_key: :upload_deleted_event_id
 
+    belongs_to :reset_checksum_event,
+      class_name: 'Geo::ResetChecksumEvent',
+      foreign_key: :reset_checksum_event_id
+
     def self.latest_event
       order(id: :desc).first
     end
@@ -76,7 +81,8 @@ module Geo
         hashed_storage_attachments_event ||
         lfs_object_deleted_event ||
         job_artifact_deleted_event ||
-        upload_deleted_event
+        upload_deleted_event ||
+        reset_checksum_event
     end
 
     def project_id

ee/app/models/geo/project_registry.rb

@@ -160,6 +160,21 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
     )
   end
 
+  # Resets repository/wiki verification state. Is called when a Geo
+  # secondary node process a Geo::ResetChecksymEvent.
+  def reset_checksum!
+    update!(
+      repository_verification_checksum_sha: nil,
+      wiki_verification_checksum_sha: nil,
+      repository_checksum_mismatch: false,
+      wiki_checksum_mismatch: false,
+      last_repository_verification_failure: nil,
+      last_wiki_verification_failure: nil,
+      repository_verification_retry_count: nil,
+      wiki_verification_retry_count: nil
+    )
+  end
+
   def repository_sync_due?(scheduled_time)
     never_synced_repository? || repository_sync_needed?(scheduled_time)
   end

ee/app/models/geo/reset_checksum_event.rb

+# frozen_string_literal: true
+
+module Geo
+  class ResetChecksumEvent < ActiveRecord::Base
+    include Geo::Model
+    include Geo::Eventable
+
+    belongs_to :project
+
+    validates :project, presence: true
+  end
+end

ee/app/services/geo/reset_checksum_event_store.rb

+# frozen_string_literal: true
+
+module Geo
+  class ResetChecksumEventStore < EventStore
+    self.event_type = :reset_checksum_event
+
+    private
+
+    def build_event
+      Geo::ResetChecksumEvent.new(project: project)
+    end
+  end
+end

ee/changelogs/unreleased/da-create-reset-checksum-event.yml

+---
+title: Geo - Add an event to reset checksums on Geo secondary nodes
+merge_request: 7394
+author:
+type: added

ee/db/migrate/20180917213751_create_geo_reset_checksum_events.rb

+# frozen_string_literal: true
+
+class CreateGeoResetChecksumEvents < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :geo_reset_checksum_events, id: :bigserial do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }, null: false
+    end
+
+    add_column :geo_event_log, :reset_checksum_event_id, :integer, limit: 8
+  end
+end

ee/db/migrate/20180917214204_add_geo_reset_checksum_events_foreign_key.rb

+# frozen_string_literal: true
+
+class AddGeoResetChecksumEventsForeignKey < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :geo_event_log, :geo_reset_checksum_events,
+                               column: :reset_checksum_event_id, on_delete: :cascade
+    add_concurrent_index :geo_event_log, :reset_checksum_event_id
+  end
+
+  def down
+    remove_foreign_key :geo_event_log, column: :reset_checksum_event_id
+    remove_concurrent_index :geo_event_log, :reset_checksum_event_id
+  end
+end

ee/lib/gitlab/geo/log_cursor/events/reset_checksum_event.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Geo
+    module LogCursor
+      module Events
+        class ResetChecksumEvent
+          include BaseEvent
+
+          def process
+            registry.reset_checksum! unless skippable?
+            log_event
+          end
+
+          private
+
+          def log_event
+            logger.event_info(
+              created_at,
+              'Reset checksum',
+              project_id: event.project_id,
+              skippable: skippable?
+            )
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/factories/geo/event_log.rb

@@ -35,6 +35,10 @@ FactoryBot.define do
     trait :upload_deleted_event do
       upload_deleted_event factory: :geo_upload_deleted_event
     end
+
+    trait :reset_checksum_event do
+      reset_checksum_event factory: :geo_reset_checksum_event
+    end
   end
 
   factory :geo_repository_created_event, class: Geo::RepositoryCreatedEvent do
@@ -138,4 +142,8 @@ FactoryBot.define do
       upload { create(:upload, :namespace_upload) }
     end
   end
+
+  factory :geo_reset_checksum_event, class: Geo::ResetChecksumEvent do
+    project
+  end
 end

ee/spec/lib/gitlab/geo/log_cursor/events/reset_checksum_event_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Geo::LogCursor::Events::ResetChecksumEvent, :postgresql, :clean_gitlab_redis_shared_state do
+  let(:logger) { Gitlab::Geo::LogCursor::Logger.new(described_class, Logger::INFO) }
+  let(:event_log) { create(:geo_event_log, :reset_checksum_event) }
+  let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
+  let(:reset_checksum_event) { event_log.reset_checksum_event }
+  let(:project) { reset_checksum_event.project }
+
+  subject { described_class.new(reset_checksum_event, Time.now, logger) }
+
+  around do |example|
+    Sidekiq::Testing.fake! { example.run }
+  end
+
+  describe '#process' do
+    context 'when a tracking entry does not exist' do
+      it 'does not create a tracking entry' do
+        expect { subject.process }.not_to change(Geo::ProjectRegistry, :count)
+      end
+
+      it 'logs an info event' do
+        data = {
+          class: described_class.name,
+          message: 'Reset checksum',
+          project_id: project.id,
+          skippable: true
+        }
+
+        expect(::Gitlab::Logger)
+          .to receive(:info)
+          .with(hash_including(data))
+
+        subject.process
+      end
+    end
+
+    context 'when a tracking entry exists' do
+      let!(:registry) { create(:geo_project_registry, :repository_verified, :wiki_verified, project: project) }
+
+      it 'resets repository/wiki verification state' do
+        subject.process
+
+        expect(registry.reload).to have_attributes(
+          repository_verification_checksum_sha: nil,
+          wiki_verification_checksum_sha: nil
+        )
+      end
+
+      it 'logs an info event' do
+        data = {
+          class: described_class.name,
+          message: 'Reset checksum',
+          project_id: project.id,
+          skippable: false
+        }
+
+        expect(::Gitlab::Logger)
+          .to receive(:info)
+          .with(hash_including(data))
+
+        subject.process
+      end
+    end
+  end
+end

ee/spec/models/geo/event_log_spec.rb

@@ -7,6 +7,7 @@ RSpec.describe Geo::EventLog, type: :model do
     it { is_expected.to belong_to(:repository_deleted_event).class_name('Geo::RepositoryDeletedEvent').with_foreign_key('repository_deleted_event_id') }
     it { is_expected.to belong_to(:repository_renamed_event).class_name('Geo::RepositoryRenamedEvent').with_foreign_key('repository_renamed_event_id') }
     it { is_expected.to belong_to(:repository_updated_event).class_name('Geo::RepositoryUpdatedEvent').with_foreign_key('repository_updated_event_id') }
+    it { is_expected.to belong_to(:reset_checksum_event).class_name('Geo::ResetChecksumEvent').with_foreign_key('reset_checksum_event_id') }
     it { is_expected.to belong_to(:hashed_storage_migrated_event).class_name('Geo::HashedStorageMigratedEvent').with_foreign_key('hashed_storage_migrated_event_id') }
     it { is_expected.to belong_to(:hashed_storage_attachments_event).class_name('Geo::HashedStorageAttachmentsEvent').with_foreign_key('hashed_storage_attachments_event_id') }
     it { is_expected.to belong_to(:lfs_object_deleted_event).class_name('Geo::LfsObjectDeletedEvent').with_foreign_key('lfs_object_deleted_event_id') }
@@ -95,6 +96,13 @@ RSpec.describe Geo::EventLog, type: :model do
 
       expect(subject.event).to eq upload_deleted_event
     end
+
+    it 'returns reset_checksum_event when set' do
+      reset_checksum_event = build(:geo_reset_checksum_event)
+      subject.reset_checksum_event = reset_checksum_event
+
+      expect(subject.event).to eq reset_checksum_event
+    end
   end
 
   describe '#project_id' do

ee/spec/models/geo/every_geo_event_spec.rb

+require 'spec_helper'
+
+describe 'Every Geo event' do
+  subject { events }
+
+  it 'includes Geo::Eventable' do
+    is_expected.to all( satisfy { |klass| klass.ancestors.include?(Geo::Eventable)  })
+  end
+
+  it 'has its class in Geo::EventLog::EVENT_CLASSES' do
+    expect(subject.map(&:name)).to match_array(Geo::EventLog::EVENT_CLASSES)
+  end
+
+  def events
+    root  = Rails.root.join('ee', 'app', 'models')
+    geo   = root.join('geo')
+
+    events = Dir[geo.join('**', '*.rb')]
+      .select { |path| path.end_with?('_event.rb') }
+
+    events.map! do |path|
+      ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
+      ns.camelize.constantize
+    end
+
+    # Skip things that aren't models
+    events.select { |event| event < ActiveRecord::Base }
+  end
+end

ee/spec/models/geo/project_registry_spec.rb

@@ -753,6 +753,34 @@ describe Geo::ProjectRegistry do
     end
   end
 
+  describe '#reset_checksum!' do
+    it 'resets repository/wiki verification state' do
+      subject.update!(
+        repository_verification_checksum_sha: 'abc123',
+        wiki_verification_checksum_sha: 'abc123',
+        repository_checksum_mismatch: true,
+        wiki_checksum_mismatch: true,
+        last_repository_verification_failure: 'foo',
+        last_wiki_verification_failure: 'foo',
+        repository_verification_retry_count: 1,
+        wiki_verification_retry_count: 1
+      )
+
+      subject.reset_checksum!
+
+      expect(subject).to have_attributes(
+        repository_verification_checksum_sha: nil,
+        wiki_verification_checksum_sha: nil,
+        repository_checksum_mismatch: false,
+        wiki_checksum_mismatch: false,
+        last_repository_verification_failure: nil,
+        last_wiki_verification_failure: nil,
+        repository_verification_retry_count: nil,
+        wiki_verification_retry_count: nil
+      )
+    end
+  end
+
   describe '#repository_verification_pending?' do
     it 'returns true when outdated' do
       registry = create(:geo_project_registry, :repository_verification_outdated)

ee/spec/models/geo/reset_checksum_event_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::ResetChecksumEvent, type: :model do
+  describe 'relationships' do
+    it { is_expected.to belong_to(:project) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:project) }
+  end
+end

ee/spec/services/geo/prune_event_log_service_spec.rb

@@ -31,16 +31,9 @@ describe Geo::PruneEventLogService do
   end
 
   it 'prunes all event tables' do
-    expect(service).to receive(:prune!).with(Geo::RepositoryCreatedEvent, anything)
-    expect(service).to receive(:prune!).with(Geo::RepositoryUpdatedEvent, anything)
-    expect(service).to receive(:prune!).with(Geo::RepositoryDeletedEvent, anything)
-    expect(service).to receive(:prune!).with(Geo::RepositoryRenamedEvent, anything)
-    expect(service).to receive(:prune!).with(Geo::RepositoriesChangedEvent, anything)
-    expect(service).to receive(:prune!).with(Geo::HashedStorageMigratedEvent, anything)
-    expect(service).to receive(:prune!).with(Geo::HashedStorageAttachmentsEvent, anything)
-    expect(service).to receive(:prune!).with(Geo::LfsObjectDeletedEvent, anything)
-    expect(service).to receive(:prune!).with(Geo::JobArtifactDeletedEvent, anything)
-    expect(service).to receive(:prune!).with(Geo::UploadDeletedEvent, anything)
+    Geo::EventLog::EVENT_CLASSES.each do |event_class|
+      expect(service).to receive(:prune!).with(event_class.constantize, anything)
+    end
 
     service.execute
   end

ee/spec/services/geo/reset_checksum_event_store_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Geo::ResetChecksumEventStore do
+  include EE::GeoHelpers
+
+  set(:project) { create(:project) }
+  set(:secondary_node) { create(:geo_node) }
+
+  subject { described_class.new(project) }
+
+  describe '#create' do
+    it_behaves_like 'a Geo event store', Geo::ResetChecksumEvent
+
+    context 'when running on a primary node' do
+      before do
+        stub_primary_node
+      end
+
+      it 'tracks the project that checksum must be wiped' do
+        subject.create
+
+        expect(Geo::ResetChecksumEvent.last).to have_attributes(project_id: project.id)
+      end
+    end
+  end
+end

ee/spec/support/shared_examples/geo_event_store_shared_examples.rb

 # frozen_string_literal: true
 
 shared_examples_for 'a Geo event store' do |event_class|
-  context 'running on a secondary node' do
+  context 'when running on a secondary node' do
     before do
       stub_secondary_node
     end