Geo: Gracefully handle deleted events from Geo event log

When a project is deleted, its Geo event log and associated events may be deleted
from the database. We should ignore these and move along.

Relates to https://gitlab.com/gitlab-com/migration/issues/295#note_87920718

ee/changelogs/unreleased/sh-geo-add-more-geo-event-debug-logs.yml

+---
+title: 'Geo: Gracefully handle deleted events from Geo event log'
+merge_request: 6506
+author:
+type: fixed

ee/lib/gitlab/geo/log_cursor/daemon.rb

@@ -45,13 +45,21 @@ module Gitlab
 
         def handle_events(batch)
           batch.each do |event_log|
+            event = event_log.event
+
+            # If a project is deleted, the event log and its associated event data
+            # could be purged from the log. We ignore this and move along.
+            unless event
+              logger.warn("Unknown event", event_log_id: event_log.id)
+              next
+            end
+
             unless can_replay?(event_log)
               logger.event_info(event_log.created_at, 'Skipped event', event_data(event_log))
               next
             end
 
             begin
-              event = event_log.event
               event_klass_for(event).new(event, event_log.created_at, logger).process
             rescue NoMethodError => e
               logger.error(e.message)

ee/lib/gitlab/geo/log_cursor/logger.rb

@@ -25,6 +25,10 @@ module Gitlab
           geo_logger.error(base_log_data(message, params))
         end
 
+        def warn(message, params = {})
+          geo_logger.warn(base_log_data(message, params))
+        end
+
         def debug(message, params = {})
           geo_logger.debug(base_log_data(message, params))
         end

ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb

@@ -115,6 +115,20 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
         daemon.run_once!
       end
 
+      it "logs a message if an associated event can't be found" do
+        new_event = create(:geo_event_log)
+
+        expect(Gitlab::Geo::Logger).to receive(:warn)
+                                        .with(hash_including(
+                                                class: 'Gitlab::Geo::LogCursor::Daemon',
+                                                message: 'Unknown event',
+                                                event_log_id: new_event.id))
+
+        daemon.run_once!
+
+        expect(::Geo::EventLogState.last_processed.id).to eq(new_event.id)
+      end
+
       it 'logs a message for skipped events' do
         secondary.update!(selective_sync_type: 'namespaces', namespaces: [group_2])
 

ee/spec/lib/gitlab/geo/log_cursor/logger_spec.rb

@@ -5,22 +5,26 @@ describe Gitlab::Geo::LogCursor::Logger do
 
   subject(:logger) { described_class.new(LoggerSpec) }
 
+  let(:data) { { pid: 111, class: 'LoggerSpec', message: 'Test' } }
+
   before do
     stub_const("#{described_class.name}::PID", 111)
   end
 
   it 'logs an info event' do
-    expect(::Gitlab::Logger).to receive(:info).with(pid: 111,
-                                                    class: "LoggerSpec",
-                                                    message: 'Test')
+    expect(::Gitlab::Logger).to receive(:info).with(data)
 
     logger.info('Test')
   end
 
+  it 'logs a warning event' do
+    expect(::Gitlab::Logger).to receive(:warn).with(data)
+
+    logger.warn('Test')
+  end
+
   it 'logs an error event' do
-    expect(::Gitlab::Logger).to receive(:error).with(pid: 111,
-                                                     class: "LoggerSpec",
-                                                     message: 'Test')
+    expect(::Gitlab::Logger).to receive(:error).with(data)
 
     logger.error('Test')
   end

lib/gitlab/logger.rb

@@ -12,6 +12,10 @@ module Gitlab
       build.error(message)
     end
 
+    def self.warn(message)
+      build.warn(message)
+    end
+
     def self.info(message)
       build.info(message)
     end