Cap the Geo retry_at value to prevent out of range timestamps

Part of #4083

app/services/geo/base_sync_service.rb

@@ -136,7 +136,7 @@ module Geo
       if started_at
         attrs["last_#{type}_synced_at"] = started_at
         attrs["#{type}_retry_count"] = retry_count + 1
-        attrs["#{type}_retry_at"] = Time.now + delay(attrs["#{type}_retry_count"]).seconds
+        attrs["#{type}_retry_at"] = next_retry_time(attrs["#{type}_retry_count"])
       end
 
       if finished_at
@@ -210,5 +210,14 @@ module Geo
         raise Gitlab::Shell::Error, 'Can not move temporary repository'
       end
     end
+
+    # To prevent the retry time from storing invalid dates in the database,
+    # cap the max time to a week plus some random jitter value.
+    def next_retry_time(retry_count)
+      proposed_time = Time.now + delay(retry_count).seconds
+      max_future_time = Time.now + 7.days + delay(1).seconds
+
+      [proposed_time, max_future_time].min
+    end
   end
 end

spec/services/geo/repository_sync_service_spec.rb

@@ -189,6 +189,25 @@ describe Geo::RepositorySyncService do
 
         subject.execute
       end
+
+      it 'successfully redownloads the file even if the retry time exceeds max value' do
+        timestamp = Time.now.utc
+        registry = create(
+          :geo_project_registry,
+          project: project,
+          repository_retry_count: Geo::BaseSyncService::RETRY_BEFORE_REDOWNLOAD + 2000,
+          repository_retry_at: timestamp,
+          force_to_redownload_repository: true
+        )
+
+        subject.execute
+
+        # The repository should be redownloaded and cleared without errors. If
+        # the timestamp were not capped, we would have seen a "timestamp out
+        # of range" in the first update to the project registry.
+        registry.reload
+        expect(registry.repository_retry_at).to be_nil
+      end
     end
 
     context 'secondary replicates over SSH' do