Commit c1388d0e authored by Nick Thomas's avatar Nick Thomas

Allow multiple repository storage shards to be enabled, and automatically round-robin between them

parent 3a8a7c12
...@@ -116,8 +116,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -116,8 +116,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:metrics_packet_size, :metrics_packet_size,
:send_user_confirmation_email, :send_user_confirmation_email,
:container_registry_token_expire_delay, :container_registry_token_expire_delay,
:repository_storage,
:enabled_git_access_protocol, :enabled_git_access_protocol,
repository_storages: [],
restricted_visibility_levels: [], restricted_visibility_levels: [],
import_sources: [], import_sources: [],
disabled_oauth_sign_in_sources: [] disabled_oauth_sign_in_sources: []
......
...@@ -93,11 +93,11 @@ module ApplicationSettingsHelper ...@@ -93,11 +93,11 @@ module ApplicationSettingsHelper
end end
end end
def repository_storage_options_for_select def repository_storages_options_for_select
options = Gitlab.config.repositories.storages.map do |name, path| options = Gitlab.config.repositories.storages.map do |name, path|
["#{name} - #{path}", name] ["#{name} - #{path}", name]
end end
options_for_select(options, @application_setting.repository_storage) options_for_select(options, @application_setting.repository_storages)
end end
end end
...@@ -18,6 +18,7 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -18,6 +18,7 @@ class ApplicationSetting < ActiveRecord::Base
serialize :disabled_oauth_sign_in_sources, Array serialize :disabled_oauth_sign_in_sources, Array
serialize :domain_whitelist, Array serialize :domain_whitelist, Array
serialize :domain_blacklist, Array serialize :domain_blacklist, Array
serialize :repository_storages
cache_markdown_field :sign_in_text cache_markdown_field :sign_in_text
cache_markdown_field :help_page_text cache_markdown_field :help_page_text
...@@ -74,9 +75,8 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -74,9 +75,8 @@ class ApplicationSetting < ActiveRecord::Base
presence: true, presence: true,
numericality: { only_integer: true, greater_than: 0 } numericality: { only_integer: true, greater_than: 0 }
validates :repository_storage, validates :repository_storages, presence: true
presence: true, validate :check_repository_storages
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
validates :enabled_git_access_protocol, validates :enabled_git_access_protocol,
inclusion: { in: %w(ssh http), allow_blank: true, allow_nil: true } inclusion: { in: %w(ssh http), allow_blank: true, allow_nil: true }
...@@ -166,7 +166,7 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -166,7 +166,7 @@ class ApplicationSetting < ActiveRecord::Base
disabled_oauth_sign_in_sources: [], disabled_oauth_sign_in_sources: [],
send_user_confirmation_email: false, send_user_confirmation_email: false,
container_registry_token_expire_delay: 5, container_registry_token_expire_delay: 5,
repository_storage: 'default', repository_storages: ['default'],
user_default_external: false, user_default_external: false,
) )
end end
...@@ -201,6 +201,29 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -201,6 +201,29 @@ class ApplicationSetting < ActiveRecord::Base
self.domain_blacklist_raw = file.read self.domain_blacklist_raw = file.read
end end
def repository_storages
value = read_attribute(:repository_storages)
value = [value] if value.is_a?(String)
value = [] if value.nil?
value
end
# repository_storage is still required in the API. Remove in 9.0
def repository_storage
repository_storages.first
end
def repository_storage=(value)
self.repository_storages = [value]
end
# Choose one of the available repository storage options. Currently all have
# equal weighting.
def pick_repository_storage
repository_storages.sample
end
def runners_registration_token def runners_registration_token
ensure_runners_registration_token! ensure_runners_registration_token!
end end
...@@ -208,4 +231,12 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -208,4 +231,12 @@ class ApplicationSetting < ActiveRecord::Base
def health_check_access_token def health_check_access_token
ensure_health_check_access_token! ensure_health_check_access_token!
end end
private
def check_repository_storages
invalid = repository_storages - Gitlab.config.repositories.storages.keys
errors.add(:repository_storages, "can't include: #{invalid.join(", ")}") unless
invalid.empty?
end
end end
...@@ -28,7 +28,7 @@ class Project < ActiveRecord::Base ...@@ -28,7 +28,7 @@ class Project < ActiveRecord::Base
default_value_for :archived, false default_value_for :archived, false
default_value_for :visibility_level, gitlab_config_features.visibility_level default_value_for :visibility_level, gitlab_config_features.visibility_level
default_value_for :container_registry_enabled, gitlab_config_features.container_registry default_value_for :container_registry_enabled, gitlab_config_features.container_registry
default_value_for(:repository_storage) { current_application_settings.repository_storage } default_value_for(:repository_storage) { current_application_settings.pick_repository_storage }
default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
default_value_for :issues_enabled, gitlab_config_features.issues default_value_for :issues_enabled, gitlab_config_features.issues
default_value_for :merge_requests_enabled, gitlab_config_features.merge_requests default_value_for :merge_requests_enabled, gitlab_config_features.merge_requests
......
...@@ -353,9 +353,9 @@ ...@@ -353,9 +353,9 @@
%fieldset %fieldset
%legend Repository Storage %legend Repository Storage
.form-group .form-group
= f.label :repository_storage, 'Storage path for new projects', class: 'control-label col-sm-2' = f.label :repository_storages, 'Storage paths for new projects', class: 'control-label col-sm-2'
.col-sm-10 .col-sm-10
= f.select :repository_storage, repository_storage_options_for_select, {}, class: 'form-control' = f.select :repository_storages, repository_storages_options_for_select, {include_hidden: false}, multiple: true, class: 'form-control'
.help-block .help-block
Manage repository storage paths. Learn more in the Manage repository storage paths. Learn more in the
= succeed "." do = succeed "." do
......
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class RenameRepositoryStorageColumn < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
# When a migration requires downtime you **must** uncomment the following
# constant and define a short and easy to understand explanation as to why the
# migration requires downtime.
# DOWNTIME_REASON = ''
# When using the methods "add_concurrent_index" or "add_column_with_default"
# you must disable the use of transactions as these methods can not run in an
# existing transaction. When using "add_concurrent_index" make sure that this
# method is the _only_ method called in the migration, any other changes
# should go in a separate migration. This ensures that upon failure _only_ the
# index creation fails and can be retried or reverted easily.
#
# To disable transactions uncomment the following line and remove these
# comments:
# disable_ddl_transaction!
def change
rename_column :application_settings, :repository_storage, :repository_storages
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20161025231710) do ActiveRecord::Schema.define(version: 20161103171205) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -88,7 +88,7 @@ ActiveRecord::Schema.define(version: 20161025231710) do ...@@ -88,7 +88,7 @@ ActiveRecord::Schema.define(version: 20161025231710) do
t.integer "container_registry_token_expire_delay", default: 5 t.integer "container_registry_token_expire_delay", default: 5
t.text "after_sign_up_text" t.text "after_sign_up_text"
t.boolean "user_default_external", default: false, null: false t.boolean "user_default_external", default: false, null: false
t.string "repository_storage", default: "default" t.string "repository_storages", default: "default"
t.string "enabled_git_access_protocol" t.string "enabled_git_access_protocol"
t.boolean "domain_blacklist_enabled", default: false t.boolean "domain_blacklist_enabled", default: false
t.text "domain_blacklist" t.text "domain_blacklist"
......
...@@ -509,6 +509,7 @@ module API ...@@ -509,6 +509,7 @@ module API
expose :after_sign_out_path expose :after_sign_out_path
expose :container_registry_token_expire_delay expose :container_registry_token_expire_delay
expose :repository_storage expose :repository_storage
expose :repository_storages
expose :koding_enabled expose :koding_enabled
expose :koding_url expose :koding_url
end end
......
...@@ -17,12 +17,12 @@ module API ...@@ -17,12 +17,12 @@ module API
present current_settings, with: Entities::ApplicationSetting present current_settings, with: Entities::ApplicationSetting
end end
# Modify applicaiton settings # Modify application settings
# #
# Example Request: # Example Request:
# PUT /application/settings # PUT /application/settings
put "application/settings" do put "application/settings" do
attributes = current_settings.attributes.keys - ["id"] attributes = ["repository_storage"] + current_settings.attributes.keys - ["id"]
attrs = attributes_for_keys(attributes) attrs = attributes_for_keys(attributes)
if current_settings.update_attributes(attrs) if current_settings.update_attributes(attrs)
......
...@@ -41,14 +41,62 @@ describe ApplicationSetting, models: true do ...@@ -41,14 +41,62 @@ describe ApplicationSetting, models: true do
subject { setting } subject { setting }
end end
context 'repository storages inclussion' do # Upgraded databases will have this sort of content
context 'repository_storages is a String, not an Array' do
before { setting.__send__(:raw_write_attribute, :repository_storages, 'default') }
it { expect(setting.repository_storages_before_type_cast).to eq('default') }
it { expect(setting.repository_storages).to eq(['default']) }
end
context 'repository storages' do
before do before do
storages = { 'custom' => 'tmp/tests/custom_repositories' } storages = {
'custom1' => 'tmp/tests/custom_repositories_1',
'custom2' => 'tmp/tests/custom_repositories_2',
'custom3' => 'tmp/tests/custom_repositories_3',
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end end
it { is_expected.to allow_value('custom').for(:repository_storage) } describe 'inclusion' do
it { is_expected.not_to allow_value('alternative').for(:repository_storage) } it { is_expected.to allow_value('custom1').for(:repository_storages) }
it { is_expected.to allow_value(['custom2', 'custom3']).for(:repository_storages) }
it { is_expected.not_to allow_value('alternative').for(:repository_storages) }
it { is_expected.not_to allow_value(['alternative', 'custom1']).for(:repository_storages) }
end
describe 'presence' do
it { is_expected.not_to allow_value([]).for(:repository_storages) }
it { is_expected.not_to allow_value("").for(:repository_storages) }
it { is_expected.not_to allow_value(nil).for(:repository_storages) }
end
describe '.pick_repository_storage' do
it 'uses Array#sample to pick a random storage' do
array = double('array', sample: 'random')
expect(setting).to receive(:repository_storages).and_return(array)
expect(setting.pick_repository_storage).to eq('random')
end
describe '#repository_storage' do
it 'returns the first storage' do
setting.repository_storages = ['good', 'bad']
expect(setting.repository_storage).to eq('good')
end
end
describe '#repository_storage=' do
it 'overwrites repository_storages' do
setting.repository_storage = 'overwritten'
expect(setting.repository_storages).to eq(['overwritten'])
end
end
end
end end
end end
......
...@@ -837,16 +837,19 @@ describe Project, models: true do ...@@ -837,16 +837,19 @@ describe Project, models: true do
context 'repository storage by default' do context 'repository storage by default' do
let(:project) { create(:empty_project) } let(:project) { create(:empty_project) }
subject { project.repository_storage }
before do before do
storages = { 'alternative_storage' => '/some/path' } storages = {
'default' => 'tmp/tests/repositories',
'picked' => 'tmp/tests/repositories',
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
stub_application_setting(repository_storage: 'alternative_storage')
allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(true)
end end
it { is_expected.to eq('alternative_storage') } it 'picks storage from ApplicationSetting' do
expect_any_instance_of(ApplicationSetting).to receive(:pick_repository_storage).and_return('picked')
expect(project.repository_storage).to eq('picked')
end
end end
context 'shared runners by default' do context 'shared runners by default' do
......
...@@ -33,6 +33,7 @@ describe API::API, 'Settings', api: true do ...@@ -33,6 +33,7 @@ describe API::API, 'Settings', api: true do
expect(json_response['default_projects_limit']).to eq(3) expect(json_response['default_projects_limit']).to eq(3)
expect(json_response['signin_enabled']).to be_falsey expect(json_response['signin_enabled']).to be_falsey
expect(json_response['repository_storage']).to eq('custom') expect(json_response['repository_storage']).to eq('custom')
expect(json_response['repository_storages']).to eq(['custom'])
expect(json_response['koding_enabled']).to be_truthy expect(json_response['koding_enabled']).to be_truthy
expect(json_response['koding_url']).to eq('http://koding.example.com') expect(json_response['koding_url']).to eq('http://koding.example.com')
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment