Commit 9b4a1db9 authored by Yorick Peterse's avatar Yorick Peterse

Merge branch 'ab-43887-concurrent-migration-helpers' into 'master'

Convenient use of concurrent migration helpers

Closes #43887

See merge request gitlab-org/gitlab-ce!17888
parents 30c480c2 c914883a
......@@ -154,7 +154,7 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
end
def add_foreign_key_if_not_exists(source, target, column:)
return if foreign_key_exists?(source, column)
return if foreign_key_exists?(source, target, column: column)
add_concurrent_foreign_key(source, target, column: column)
end
......@@ -175,12 +175,6 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
rescue ArgumentError
end
def foreign_key_exists?(table, column)
foreign_keys(table).any? do |key|
key.options[:column] == column.to_s
end
end
def connection
# Rails memoizes connection objects, but this causes them to be shared
# amongst threads; we don't want that.
......
......@@ -10,13 +10,13 @@ class AddStageIdForeignKeyToBuilds < ActiveRecord::Migration
add_concurrent_index(:ci_builds, :stage_id)
end
unless foreign_key_exists?(:ci_builds, :stage_id)
unless foreign_key_exists?(:ci_builds, :ci_stages, column: :stage_id)
add_concurrent_foreign_key(:ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade)
end
end
def down
if foreign_key_exists?(:ci_builds, :stage_id)
if foreign_key_exists?(:ci_builds, column: :stage_id)
remove_foreign_key(:ci_builds, column: :stage_id)
end
......@@ -24,12 +24,4 @@ class AddStageIdForeignKeyToBuilds < ActiveRecord::Migration
remove_concurrent_index(:ci_builds, :stage_id)
end
end
private
def foreign_key_exists?(table, column)
foreign_keys(:ci_builds).any? do |key|
key.options[:column] == column.to_s
end
end
end
......@@ -23,23 +23,15 @@ class AddForeignKeyToMergeRequests < ActiveRecord::Migration
merge_requests.update_all(head_pipeline_id: nil)
end
unless foreign_key_exists?(:merge_requests, :head_pipeline_id)
unless foreign_key_exists?(:merge_requests, column: :head_pipeline_id)
add_concurrent_foreign_key(:merge_requests, :ci_pipelines,
column: :head_pipeline_id, on_delete: :nullify)
end
end
def down
if foreign_key_exists?(:merge_requests, :head_pipeline_id)
if foreign_key_exists?(:merge_requests, column: :head_pipeline_id)
remove_foreign_key(:merge_requests, column: :head_pipeline_id)
end
end
private
def foreign_key_exists?(table, column)
foreign_keys(table).any? do |key|
key.options[:column] == column.to_s
end
end
end
......@@ -26,11 +26,11 @@ class BuildUserInteractedProjectsTable < ActiveRecord::Migration
def down
execute "TRUNCATE user_interacted_projects"
if foreign_key_exists?(:user_interacted_projects, :user_id)
if foreign_key_exists?(:user_interacted_projects, :users)
remove_foreign_key :user_interacted_projects, :users
end
if foreign_key_exists?(:user_interacted_projects, :project_id)
if foreign_key_exists?(:user_interacted_projects, :projects)
remove_foreign_key :user_interacted_projects, :projects
end
......@@ -115,7 +115,7 @@ class BuildUserInteractedProjectsTable < ActiveRecord::Migration
end
def create_fk(table, target, column)
return if foreign_key_exists?(table, column)
return if foreign_key_exists?(table, target, column: column)
add_foreign_key table, target, column: column, on_delete: :cascade
end
......@@ -158,11 +158,11 @@ class BuildUserInteractedProjectsTable < ActiveRecord::Migration
add_concurrent_index :user_interacted_projects, [:project_id, :user_id], unique: true, name: UNIQUE_INDEX_NAME
end
unless foreign_key_exists?(:user_interacted_projects, :user_id)
unless foreign_key_exists?(:user_interacted_projects, :users, column: :user_id)
add_concurrent_foreign_key :user_interacted_projects, :users, column: :user_id, on_delete: :cascade
end
unless foreign_key_exists?(:user_interacted_projects, :project_id)
unless foreign_key_exists?(:user_interacted_projects, :projects, column: :project_id)
add_concurrent_foreign_key :user_interacted_projects, :projects, column: :project_id, on_delete: :cascade
end
end
......
......@@ -136,11 +136,14 @@ class MyMigration < ActiveRecord::Migration
disable_ddl_transaction!
def up
remove_concurrent_index :table_name, :column_name if index_exists?(:table_name, :column_name)
remove_concurrent_index :table_name, :column_name
end
end
```
Note that it is not necessary to check if the index exists prior to
removing it.
## Adding indexes
If you need to add a unique index please keep in mind there is the possibility
......
......@@ -59,6 +59,11 @@ module Gitlab
disable_statement_timeout
end
if index_exists?(table_name, column_name, options)
Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}"
return
end
add_index(table_name, column_name, options)
end
......@@ -83,6 +88,11 @@ module Gitlab
disable_statement_timeout
end
unless index_exists?(table_name, column_name, options)
Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}"
return
end
remove_index(table_name, options.merge({ column: column_name }))
end
......@@ -107,6 +117,11 @@ module Gitlab
disable_statement_timeout
end
unless index_exists_by_name?(table_name, index_name)
Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}"
return
end
remove_index(table_name, options.merge({ name: index_name }))
end
......@@ -140,6 +155,13 @@ module Gitlab
# of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall
# back to the normal foreign key procedure.
if Database.mysql?
if foreign_key_exists?(source, target, column: column)
Rails.logger.warn "Foreign key not created because it exists already " \
"(this may be due to an aborted migration or similar): " \
"source: #{source}, target: #{target}, column: #{column}"
return
end
return add_foreign_key(source, target,
column: column,
on_delete: on_delete)
......@@ -151,25 +173,43 @@ module Gitlab
key_name = concurrent_foreign_key_name(source, column)
# Using NOT VALID allows us to create a key without immediately
# validating it. This means we keep the ALTER TABLE lock only for a
# short period of time. The key _is_ enforced for any newly created
# data.
execute <<-EOF.strip_heredoc
ALTER TABLE #{source}
ADD CONSTRAINT #{key_name}
FOREIGN KEY (#{column})
REFERENCES #{target} (id)
#{on_delete ? "ON DELETE #{on_delete.upcase}" : ''}
NOT VALID;
EOF
unless foreign_key_exists?(source, target, column: column)
Rails.logger.warn "Foreign key not created because it exists already " \
"(this may be due to an aborted migration or similar): " \
"source: #{source}, target: #{target}, column: #{column}"
# Using NOT VALID allows us to create a key without immediately
# validating it. This means we keep the ALTER TABLE lock only for a
# short period of time. The key _is_ enforced for any newly created
# data.
execute <<-EOF.strip_heredoc
ALTER TABLE #{source}
ADD CONSTRAINT #{key_name}
FOREIGN KEY (#{column})
REFERENCES #{target} (id)
#{on_delete ? "ON DELETE #{on_delete.upcase}" : ''}
NOT VALID;
EOF
end
# Validate the existing constraint. This can potentially take a very
# long time to complete, but fortunately does not lock the source table
# while running.
#
# Note this is a no-op in case the constraint is VALID already
execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};")
end
def foreign_key_exists?(source, target = nil, column: nil)
foreign_keys(source).any? do |key|
if column
key.options[:column].to_s == column.to_s
else
key.to_table.to_s == target.to_s
end
end
end
# Returns the name for a concurrent foreign key.
#
# PostgreSQL constraint names have a limit of 63 bytes. The logic used
......@@ -860,12 +900,6 @@ into similar problems in the future (e.g. when new tables are created).
end
end
def foreign_key_exists?(table, column)
foreign_keys(table).any? do |key|
key.options[:column] == column.to_s
end
end
# Rails' index_exists? doesn't work when you only give it a table and index
# name. As such we have to use some extra code to check if an index exists for
# a given name.
......
......@@ -67,17 +67,35 @@ describe Gitlab::Database::MigrationHelpers do
model.add_concurrent_index(:users, :foo, unique: true)
end
it 'does nothing if the index exists already' do
expect(model).to receive(:index_exists?)
.with(:users, :foo, { algorithm: :concurrently, unique: true }).and_return(true)
expect(model).not_to receive(:add_index)
model.add_concurrent_index(:users, :foo, unique: true)
end
end
context 'using MySQL' do
it 'creates a regular index' do
expect(Gitlab::Database).to receive(:postgresql?).and_return(false)
before do
allow(Gitlab::Database).to receive(:postgresql?).and_return(false)
end
it 'creates a regular index' do
expect(model).to receive(:add_index)
.with(:users, :foo, {})
model.add_concurrent_index(:users, :foo)
end
it 'does nothing if the index exists already' do
expect(model).to receive(:index_exists?)
.with(:users, :foo, { unique: true }).and_return(true)
expect(model).not_to receive(:add_index)
model.add_concurrent_index(:users, :foo, unique: true)
end
end
end
......@@ -95,6 +113,7 @@ describe Gitlab::Database::MigrationHelpers do
context 'outside a transaction' do
before do
allow(model).to receive(:transaction_open?).and_return(false)
allow(model).to receive(:index_exists?).and_return(true)
end
context 'using PostgreSQL' do
......@@ -103,18 +122,41 @@ describe Gitlab::Database::MigrationHelpers do
allow(model).to receive(:disable_statement_timeout)
end
it 'removes the index concurrently by column name' do
expect(model).to receive(:remove_index)
.with(:users, { algorithm: :concurrently, column: :foo })
describe 'by column name' do
it 'removes the index concurrently' do
expect(model).to receive(:remove_index)
.with(:users, { algorithm: :concurrently, column: :foo })
model.remove_concurrent_index(:users, :foo)
model.remove_concurrent_index(:users, :foo)
end
it 'does nothing if the index does not exist' do
expect(model).to receive(:index_exists?)
.with(:users, :foo, { algorithm: :concurrently, unique: true }).and_return(false)
expect(model).not_to receive(:remove_index)
model.remove_concurrent_index(:users, :foo, unique: true)
end
end
it 'removes the index concurrently by index name' do
expect(model).to receive(:remove_index)
.with(:users, { algorithm: :concurrently, name: "index_x_by_y" })
describe 'by index name' do
before do
allow(model).to receive(:index_exists_by_name?).with(:users, "index_x_by_y").and_return(true)
end
it 'removes the index concurrently by index name' do
expect(model).to receive(:remove_index)
.with(:users, { algorithm: :concurrently, name: "index_x_by_y" })
model.remove_concurrent_index_by_name(:users, "index_x_by_y")
end
it 'does nothing if the index does not exist' do
expect(model).to receive(:index_exists_by_name?).with(:users, "index_x_by_y").and_return(false)
expect(model).not_to receive(:remove_index)
model.remove_concurrent_index_by_name(:users, "index_x_by_y")
model.remove_concurrent_index_by_name(:users, "index_x_by_y")
end
end
end
......@@ -141,6 +183,10 @@ describe Gitlab::Database::MigrationHelpers do
end
describe '#add_concurrent_foreign_key' do
before do
allow(model).to receive(:foreign_key_exists?).and_return(false)
end
context 'inside a transaction' do
it 'raises an error' do
expect(model).to receive(:transaction_open?).and_return(true)
......@@ -157,14 +203,23 @@ describe Gitlab::Database::MigrationHelpers do
end
context 'using MySQL' do
it 'creates a regular foreign key' do
before do
allow(Gitlab::Database).to receive(:mysql?).and_return(true)
end
it 'creates a regular foreign key' do
expect(model).to receive(:add_foreign_key)
.with(:projects, :users, column: :user_id, on_delete: :cascade)
model.add_concurrent_foreign_key(:projects, :users, column: :user_id)
end
it 'does not create a foreign key if it exists already' do
expect(model).to receive(:foreign_key_exists?).with(:projects, :users, column: :user_id).and_return(true)
expect(model).not_to receive(:add_foreign_key)
model.add_concurrent_foreign_key(:projects, :users, column: :user_id)
end
end
context 'using PostgreSQL' do
......@@ -189,6 +244,14 @@ describe Gitlab::Database::MigrationHelpers do
column: :user_id,
on_delete: :nullify)
end
it 'does not create a foreign key if it exists already' do
expect(model).to receive(:foreign_key_exists?).with(:projects, :users, column: :user_id).and_return(true)
expect(model).not_to receive(:execute).with(/ADD CONSTRAINT/)
expect(model).to receive(:execute).with(/VALIDATE CONSTRAINT/)
model.add_concurrent_foreign_key(:projects, :users, column: :user_id)
end
end
end
end
......@@ -203,6 +266,29 @@ describe Gitlab::Database::MigrationHelpers do
end
end
describe '#foreign_key_exists?' do
before do
key = ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new(:projects, :users, { column: :non_standard_id })
allow(model).to receive(:foreign_keys).with(:projects).and_return([key])
end
it 'finds existing foreign keys by column' do
expect(model.foreign_key_exists?(:projects, :users, column: :non_standard_id)).to be_truthy
end
it 'finds existing foreign keys by target table only' do
expect(model.foreign_key_exists?(:projects, :users)).to be_truthy
end
it 'compares by column name if given' do
expect(model.foreign_key_exists?(:projects, :users, column: :user_id)).to be_falsey
end
it 'compares by target if no column given' do
expect(model.foreign_key_exists?(:projects, :other_table)).to be_falsey
end
end
describe '#disable_statement_timeout' do
context 'using PostgreSQL' do
it 'disables statement timeouts' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment