Commit af9cb10d authored by GitLab Bot's avatar GitLab Bot

Automatic merge of gitlab-org/gitlab master

parents 534082f2 362a96b2
...@@ -516,12 +516,14 @@ class MyMigration < ActiveRecord::Migration[6.0] ...@@ -516,12 +516,14 @@ class MyMigration < ActiveRecord::Migration[6.0]
disable_ddl_transaction! disable_ddl_transaction!
INDEX_NAME = 'index_name'
def up def up
add_concurrent_index :table, :column add_concurrent_index :table, :column, name: INDEX_NAME
end end
def down def down
remove_concurrent_index :table, :column, name: index_name remove_concurrent_index :table, :column, name: INDEX_NAME
end end
end end
``` ```
......
...@@ -79,9 +79,11 @@ module Geo ...@@ -79,9 +79,11 @@ module Geo
retry_later = !registry.success || registry.missing_on_primary retry_later = !registry.success || registry.missing_on_primary
if retry_later if retry_later
custom_max_wait_time = missing_on_primary ? 4.hours : nil
# We don't limit the amount of retries # We don't limit the amount of retries
registry.retry_count = (registry.retry_count || 0) + 1 registry.retry_count = (registry.retry_count || 0) + 1
registry.retry_at = next_retry_time(registry.retry_count) registry.retry_at = next_retry_time(registry.retry_count, custom_max_wait_time)
else else
registry.retry_count = 0 registry.retry_count = 0
registry.retry_at = nil registry.retry_at = nil
......
---
title: 'Geo: Increase backoff cap for Job artifacts, LFS objects, and Uploads which are missing on primary'
merge_request: 50812
author:
type: changed
...@@ -8,9 +8,10 @@ module Delay ...@@ -8,9 +8,10 @@ module Delay
# To prevent the retry time from storing invalid dates in the database, # To prevent the retry time from storing invalid dates in the database,
# cap the max time to a hour plus some random jitter value. # cap the max time to a hour plus some random jitter value.
def next_retry_time(retry_count) def next_retry_time(retry_count, custom_max_wait_time = nil)
proposed_time = Time.now + delay(retry_count).seconds proposed_time = Time.now + delay(retry_count).seconds
max_future_time = 1.hour.from_now + delay(1).seconds max_wait_time = custom_max_wait_time || 1.hour
max_future_time = max_wait_time.from_now + delay(1).seconds
[proposed_time, max_future_time].min [proposed_time, max_future_time].min
end end
......
...@@ -323,13 +323,13 @@ RSpec.describe Geo::FileDownloadService do ...@@ -323,13 +323,13 @@ RSpec.describe Geo::FileDownloadService do
end end
end end
it 'sets a retry date with a maximum of about 7 days' do it 'sets a retry date with a maximum of about 4 hours' do
registry_entry.update!(retry_count: 100, retry_at: 7.days.from_now) registry_entry.update!(retry_count: 100, retry_at: 1.minute.ago)
freeze_time do freeze_time do
execute! execute!
expect(registry_entry.reload.retry_at < 8.days.from_now).to be_truthy expect(registry_entry.reload.retry_at).to be_within(3.minutes).of(4.hours.from_now)
end end
end end
end end
...@@ -362,13 +362,13 @@ RSpec.describe Geo::FileDownloadService do ...@@ -362,13 +362,13 @@ RSpec.describe Geo::FileDownloadService do
end end
end end
it 'sets a retry date with a maximum of about 7 days' do it 'sets a retry date with a maximum of about 1 hour' do
registry_entry.update!(retry_count: 100, retry_at: 7.days.from_now) registry_entry.update!(retry_count: 100, retry_at: 1.minute.ago)
freeze_time do freeze_time do
execute! execute!
expect(registry_entry.reload.retry_at < 8.days.from_now).to be_truthy expect(registry_entry.reload.retry_at).to be_within(3.minutes).of(1.hour.from_now)
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment