Ensure freshness of settings with project creation

Application settings from the database are currently cached locally in
the thread for a full minute after the changes are made. That means if
an administrator attempts to change the default storages in the admin
page, it might take up to a full minute for those changes to take
effect.

Since we don't have a great cache invalidation strategy for local caches
yet, the simplest approach to solving this problem would be to expire
the local application setting cache whenever we create a project. This
causes every project creation to make an additional SELECT call for
`application_settings`.

Closes https://gitlab.com/gitlab-org/gitaly/-/issues/2147

app/models/project.rb

@@ -66,7 +66,13 @@ class Project < ApplicationRecord
   default_value_for :archived, false
   default_value_for :resolve_outdated_diff_discussions, false
   default_value_for :container_registry_enabled, gitlab_config_features.container_registry
-  default_value_for(:repository_storage) { Gitlab::CurrentSettings.pick_repository_storage }
+  default_value_for(:repository_storage) do
+    # We need to ensure application settings are fresh when we pick
+    # a repository storage to use.
+    Gitlab::CurrentSettings.expire_current_application_settings
+    Gitlab::CurrentSettings.pick_repository_storage
+  end
+
   default_value_for(:shared_runners_enabled) { Gitlab::CurrentSettings.shared_runners_enabled }
   default_value_for :issues_enabled, gitlab_config_features.issues
   default_value_for :merge_requests_enabled, gitlab_config_features.merge_requests

changelogs/unreleased/sh-ensure-fresh-project-settings.yml

+---
+title: Ensure freshness of settings with project creation
+merge_request: 27156
+author:
+type: fixed

spec/models/project_spec.rb

@@ -1401,6 +1401,22 @@ describe Project do
 
       expect(project.repository_storage).to eq('picked')
     end
+
+    it 'picks from the latest available storage', :request_store do
+      stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false')
+      Gitlab::CurrentSettings.current_application_settings
+
+      settings = ApplicationSetting.last
+      settings.repository_storages = %w(picked)
+      settings.save!
+
+      expect(Gitlab::CurrentSettings.repository_storages).to eq(%w(default))
+
+      project
+
+      expect(project.repository.storage).to eq('picked')
+      expect(Gitlab::CurrentSettings.repository_storages).to eq(%w(picked))
+    end
   end
 
   context 'shared runners by default' do