Toon Claes: Return last known status from db if it was not able to contact node

app/services/geo/node_status_fetch_service.rb

@@ -6,7 +6,8 @@ module Geo
     def call(geo_node)
       return GeoNodeStatus.current_node_status if geo_node.current?
 
-      data = { success: false }
+      data = GeoNodeStatus.find_or_initialize_by(geo_node: geo_node).attributes
+      data = data.merge(success: false, health_status: 'Offline')
 
       begin
         response = self.class.get(geo_node.status_url, headers: headers, timeout: timeout)
@@ -29,8 +30,10 @@ module Geo
         end
       rescue Gitlab::Geo::GeoNodeNotFoundError
         data[:health] = 'This GitLab instance does not appear to be configured properly as a Geo node. Make sure the URLs are using the correct fully-qualified domain names.'
+        data[:health_status] = 'Unhealthy'
       rescue OpenSSL::Cipher::CipherError
         data[:health] = 'Error decrypting the Geo secret from the database. Check that the primary uses the correct db_key_base.'
+        data[:health_status] = 'Unhealthy'
       rescue HTTParty::Error, Timeout::Error, SocketError, SystemCallError, OpenSSL::SSL::SSLError => e
         data[:health] = e.message
       end

spec/services/geo/node_status_fetch_service_spec.rb

@@ -121,5 +121,29 @@ describe Geo::NodeStatusFetchService, :geo do
 
       expect(status.status_message).to eq('This GitLab instance does not appear to be configured properly as a Geo node. Make sure the URLs are using the correct fully-qualified domain names.')
     end
+
+    it 'returns the status from database if it could not fetch it' do
+      allow(described_class).to receive(:get).and_raise(Errno::ECONNREFUSED.new('bad connection'))
+      db_status = create(:geo_node_status, :healthy, geo_node: secondary)
+
+      status = subject.call(secondary)
+
+      expect(status.status_message).to eq('Connection refused - bad connection')
+      expect(status).not_to be_healthy
+      expect(status.attachments_count).to eq(db_status.attachments_count)
+      expect(status.attachments_failed_count).to eq(db_status.attachments_failed_count)
+      expect(status.attachments_synced_count).to eq(db_status.attachments_synced_count)
+      expect(status.lfs_objects_count).to eq(db_status.lfs_objects_count)
+      expect(status.lfs_objects_failed_count).to eq(db_status.lfs_objects_failed_count)
+      expect(status.lfs_objects_synced_count).to eq(db_status.lfs_objects_synced_count)
+      expect(status.repositories_count).to eq(db_status.repositories_count)
+      expect(status.repositories_synced_count).to eq(db_status.repositories_synced_count)
+      expect(status.repositories_failed_count).to eq(db_status.repositories_failed_count)
+      expect(status.last_event_id).to eq(db_status.last_event_id)
+      expect(status.last_event_timestamp).to eq(db_status.last_event_timestamp)
+      expect(status.cursor_last_event_id).to eq(db_status.cursor_last_event_id)
+      expect(status.cursor_last_event_timestamp).to eq(db_status.cursor_last_event_timestamp)
+      expect(status.last_successful_status_check_timestamp).to eq(db_status.last_successful_status_check_timestamp)
+    end
   end
 end