Skip to content

G
gitlab-ce
Commit d43602d8 authored by Lin Jen-Shin's avatar Lin Jen-Shin
Browse files
Options

Merge branch '225494-background-migrations-aren-t-currently-tested' into 'master' 

Include background migrations in the migration test level [RUN AS-IF-FOSS]

Closes #225494

See merge request gitlab-org/gitlab!35757
parents c5cb2100 600603d5
Hide whitespace changes
Inline Side-by-side
Showing with 129 additions and 606 deletions
.gitlab/ci/rules.gitlab-ci.yml
View file @ d43602d8
......@@ -118,8 +118,8 @@
- "{,ee/}spec/**/*.rb"
.db-patterns: &db-patterns
- "{,ee/}{db}/**/*"
- "{,ee}/spec/{db,migrations}/**/*"
- "{,ee/}{,spec/}{db,migrations}/**/*"
- "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration/**/*"
.backstage-patterns: &backstage-patterns
- "Dangerfile"
......
app/models/project_statistics.rb
View file @ d43602d8
......@@ -75,7 +75,12 @@ class ProjectStatistics < ApplicationRecord
end
def update_storage_size
self.storage_size = repository_size + wiki_size + lfs_objects_size + build_artifacts_size + packages_size + snippets_size
storage_size = repository_size + wiki_size + lfs_objects_size + build_artifacts_size + packages_size
# The `snippets_size` column was added on 20200622095419 but db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb
# might try to update project statistics before the `snippets_size` column has been created.
storage_size += snippets_size if self.class.column_names.include?('snippets_size')
self.storage_size = storage_size
end
# Since this incremental update method does not call update_storage_size above,
......
db/migrate/20190225160300_steal_encrypt_runners_tokens.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
class StealEncryptRunnersTokens < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
# This cleans after `EncryptRunnersTokens`
DOWNTIME = false
disable_ddl_transaction!
def up
Gitlab::BackgroundMigration.steal('EncryptRunnersTokens')
end
def down
# no-op
end
end
db/structure.sql
View file @ d43602d8
......@@ -22530,7 +22530,6 @@ COPY "schema_migrations" (version) FROM STDIN;
20190220150130
20190222051615
20190225152525
20190225160300
20190225160301
20190228192410
20190301081611
......
ee/spec/lib/ee/gitlab/background_migration/remove_undefined_vulnerability_confidence_level_spec.rb
View file @ d43602d8
......@@ -5,11 +5,13 @@ require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel, :migration, schema: 20200511092714 do
let(:vulnerabilities) { table(:vulnerabilities) }
let(:identifiers) { table(:vulnerability_identifiers) }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:users) { table(:users) }
it 'updates undefined confidence level to unknown' do
projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
namespace = namespaces.create!(name: 'gitlab-org', path: 'gitlab-org')
projects.create!(id: 123, namespace_id: namespace.id, name: 'gitlab', path: 'gitlab')
users.create!(id: 13, email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active')
vul1 = vulnerabilities.create!(vuln_params)
......
ee/spec/lib/ee/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback_spec.rb
View file @ d43602d8
......@@ -69,7 +69,7 @@ RSpec.describe Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFe
expect { described_class.new.perform(project.id) }
.to change { vulnerability.reload.dismissed_at }
.from(nil)
.to(dismiss_feedback.created_at)
.to(dismiss_feedback.reload.created_at)
end
context 'project is set to be deleted' do
......
lib/gitlab/background_migration/digest_column.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class DigestColumn
class PersonalAccessToken < ActiveRecord::Base
self.table_name = 'personal_access_tokens'
end
def perform(model, attribute_from, attribute_to, start_id, stop_id)
model = model.constantize if model.is_a?(String)
model.transaction do
relation = model.where(id: start_id..stop_id).where.not(attribute_from => nil).lock
relation.each do |instance|
instance.update_columns(attribute_to => Gitlab::CryptoHelper.sha256(instance.read_attribute(attribute_from)),
attribute_from => nil)
end
end
end
end
end
end
lib/gitlab/background_migration/encrypt_columns.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# EncryptColumn migrates data from an unencrypted column - `foo`, say - to
# an encrypted column - `encrypted_foo`, say.
#
# To avoid depending on a particular version of the model in app/, add a
# model to `lib/gitlab/background_migration/models/encrypt_columns` and use
# it in the migration that enqueues the jobs, so code can be shared.
#
# For this background migration to work, the table that is migrated _has_ to
# have an `id` column as the primary key. Additionally, the encrypted column
# should be managed by attr_encrypted, and map to an attribute with the same
# name as the unencrypted column (i.e., the unencrypted column should be
# shadowed), unless you want to define specific methods / accessors in the
# temporary model in `/models/encrypt_columns/your_model.rb`.
#
class EncryptColumns
def perform(model, attributes, from, to)
model = model.constantize if model.is_a?(String)
# If sidekiq hasn't undergone a restart, its idea of what columns are
# present may be inaccurate, so ensure this is as fresh as possible
model.reset_column_information
model.define_attribute_methods
attributes = expand_attributes(model, Array(attributes).map(&:to_sym))
model.transaction do
# Use SELECT ... FOR UPDATE to prevent the value being changed while
# we are encrypting it
relation = model.where(id: from..to).lock
relation.each do |instance|
encrypt!(instance, attributes)
end
end
end
def clear_migrated_values?
true
end
private
# Build a hash of { attribute => encrypted column name }
def expand_attributes(klass, attributes)
expanded = attributes.flat_map do |attribute|
attr_config = klass.encrypted_attributes[attribute]
crypt_column_name = attr_config&.fetch(:attribute)
raise "Couldn't determine encrypted column for #{klass}##{attribute}" if
crypt_column_name.nil?
raise "#{klass} source column: #{attribute} is missing" unless
klass.column_names.include?(attribute.to_s)
# Running the migration without the destination column being present
# leads to data loss
raise "#{klass} destination column: #{crypt_column_name} is missing" unless
klass.column_names.include?(crypt_column_name.to_s)
[attribute, crypt_column_name]
end
Hash[*expanded]
end
# Generate ciphertext for each column and update the database
def encrypt!(instance, attributes)
to_clear = attributes
.map { |plain, crypt| apply_attribute!(instance, plain, crypt) }
.compact
.flat_map { |plain| [plain, nil] }
to_clear = Hash[*to_clear]
if instance.changed?
instance.save!
if clear_migrated_values?
instance.update_columns(to_clear)
end
end
end
def apply_attribute!(instance, plain_column, crypt_column)
plaintext = instance[plain_column]
ciphertext = instance[crypt_column]
# No need to do anything if the plaintext is nil, or an encrypted
# value already exists
return unless plaintext.present?
return if ciphertext.present?
# attr_encrypted will calculate and set the expected value for us
instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend
plain_column
end
end
end
end
lib/gitlab/background_migration/encrypt_runners_tokens.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# EncryptColumn migrates data from an unencrypted column - `foo`, say - to
# an encrypted column - `encrypted_foo`, say.
#
# We only create a subclass here because we want to isolate this migration
# (migrating unencrypted runner registration tokens to encrypted columns)
# from other `EncryptColumns` migration. This class name is going to be
# serialized and stored in Redis and later picked by Sidekiq, so we need to
# create a separate class name in order to isolate these migration tasks.
#
# We can solve this differently, see tech debt issue:
#
# https://gitlab.com/gitlab-org/gitlab-foss/issues/54328
#
class EncryptRunnersTokens < EncryptColumns
def perform(model, from, to)
resource = "::Gitlab::BackgroundMigration::Models::EncryptColumns::#{model.to_s.capitalize}"
model = resource.constantize
attributes = model.encrypted_attributes.keys
super(model, attributes, from, to)
end
def clear_migrated_values?
false
end
end
end
end
lib/gitlab/background_migration/models/encrypt_columns/namespace.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
module Models
module EncryptColumns
# This model is shared between synchronous and background migrations to
# encrypt the `runners_token` column in `namespaces` table.
#
class Namespace < ActiveRecord::Base
include ::EachBatch
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled
def runners_token=(value)
self.runners_token_encrypted =
::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
end
def self.encrypted_attributes
{ runners_token: { attribute: :runners_token_encrypted } }
end
end
end
end
end
end
lib/gitlab/background_migration/models/encrypt_columns/project.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
module Models
module EncryptColumns
# This model is shared between synchronous and background migrations to
# encrypt the `runners_token` column in `projects` table.
#
class Project < ActiveRecord::Base
include ::EachBatch
self.table_name = 'projects'
self.inheritance_column = :_type_disabled
def runners_token=(value)
self.runners_token_encrypted =
::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
end
def self.encrypted_attributes
{ runners_token: { attribute: :runners_token_encrypted } }
end
end
end
end
end
end
lib/gitlab/background_migration/models/encrypt_columns/runner.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
module Models
module EncryptColumns
# This model is shared between synchronous and background migrations to
# encrypt the `token` column in `ci_runners` table.
#
class Runner < ActiveRecord::Base
include ::EachBatch
self.table_name = 'ci_runners'
self.inheritance_column = :_type_disabled
def token=(value)
self.token_encrypted =
::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
end
def self.encrypted_attributes
{ token: { attribute: :token_encrypted } }
end
end
end
end
end
end
lib/gitlab/background_migration/models/encrypt_columns/settings.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
module Models
module EncryptColumns
# This model is shared between synchronous and background migrations to
# encrypt the `runners_token` column in `application_settings` table.
#
class Settings < ActiveRecord::Base
include ::EachBatch
include ::CacheableAttributes
self.table_name = 'application_settings'
self.inheritance_column = :_type_disabled
after_commit do
::ApplicationSetting.expire
end
def runners_registration_token=(value)
self.runners_registration_token_encrypted =
::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
end
def self.encrypted_attributes
{
runners_registration_token: {
attribute: :runners_registration_token_encrypted
}
}
end
end
end
end
end
end
lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
module Models
module EncryptColumns
# This model is shared between synchronous and background migrations to
# encrypt the `token` and `url` columns
class WebHook < ActiveRecord::Base
include ::EachBatch
self.table_name = 'web_hooks'
self.inheritance_column = :_type_disabled
attr_encrypted :token,
mode: :per_attribute_iv,
algorithm: 'aes-256-gcm',
key: ::Settings.attr_encrypted_db_key_base_32
attr_encrypted :url,
mode: :per_attribute_iv,
algorithm: 'aes-256-gcm',
key: ::Settings.attr_encrypted_db_key_base_32
end
end
end
end
end
lib/quality/test_level.rb
View file @ d43602d8
......@@ -93,8 +93,14 @@ module Quality
private
def migration_and_background_migration_folders
TEST_LEVEL_FOLDERS.fetch(:migration) + TEST_LEVEL_FOLDERS.fetch(:background_migration)
end
def folders_pattern(level)
case level
when :migration
"{#{migration_and_background_migration_folders.join(',')}}"
# Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
when :all, :geo
'**'
......@@ -105,6 +111,8 @@ module Quality
def folders_regex(level)
case level
when :migration
"(#{migration_and_background_migration_folders.join('|')})"
# Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
when :all, :geo
''
......
spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb
View file @ d43602d8
......@@ -2,13 +2,15 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount, schema: 20180105212544 do
RSpec.describe Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount do
let(:namespaces_table) { table(:namespaces) }
let(:projects_table) { table(:projects) }
let(:merge_requests_table) { table(:merge_requests) }
let(:merge_request_diffs_table) { table(:merge_request_diffs) }
let(:merge_request_diff_commits_table) { table(:merge_request_diff_commits) }
let(:project) { projects_table.create!(name: 'gitlab', path: 'gitlab-org/gitlab-ce') }
let(:namespace) { namespaces_table.create!(name: 'gitlab-org', path: 'gitlab-org') }
let(:project) { projects_table.create!(name: 'gitlab', path: 'gitlab-org/gitlab-ce', namespace_id: namespace.id) }
let(:merge_request) do
merge_requests_table.create!(target_project_id: project.id,
target_branch: 'master',
......
spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb
View file @ d43602d8
......@@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, schema: 20180529152628 do
RSpec.describe Gitlab::BackgroundMigration::ArchiveLegacyTraces do
include TraceHelpers
let(:namespaces) { table(:namespaces) }
......
spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb
View file @ d43602d8
......@@ -2,6 +2,6 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillHashedProjectRepositories, schema: 20181130102132 do
RSpec.describe Gitlab::BackgroundMigration::BackfillHashedProjectRepositories do
it_behaves_like 'backfill migration for project repositories', :hashed
end
spec/lib/gitlab/background_migration/backfill_legacy_project_repositories_spec.rb
View file @ d43602d8
......@@ -2,6 +2,6 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillLegacyProjectRepositories, schema: 20181212171634 do
RSpec.describe Gitlab::BackgroundMigration::BackfillLegacyProjectRepositories do
it_behaves_like 'backfill migration for project repositories', :legacy
end
spec/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config_spec.rb
View file @ d43602d8
......@@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig, schema: 20181010133639 do
RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:group) { namespaces.create!(name: 'foo', path: 'foo') }
......
spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb
View file @ d43602d8
......@@ -9,7 +9,6 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat
let(:snippet_repositories) { table(:snippet_repositories) }
let(:user_state) { 'active' }
let(:ghost) { false }
let(:user_type) { nil }
let(:user_name) { 'Test' }
......@@ -20,7 +19,6 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat
username: 'test',
name: user_name,
state: user_state,
ghost: ghost,
last_activity_on: 1.minute.ago,
user_type: user_type,
confirmed_at: 1.day.ago)
......@@ -113,8 +111,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat
end
context 'when user is a ghost' do
let(:ghost) { true }
let(:user_type) { 'ghost' }
let(:user_type) { HasUserType::USER_TYPES[:ghost] }
it_behaves_like 'migration_bot user commits files'
end
......@@ -255,7 +252,6 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat
username: 'test2',
name: 'Test2',
state: user_state,
ghost: ghost,
last_activity_on: 1.minute.ago,
user_type: user_type,
confirmed_at: 1.day.ago)
......
spec/lib/gitlab/background_migration/digest_column_spec.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::DigestColumn, schema: 20180913142237 do
let(:personal_access_tokens) { table(:personal_access_tokens) }
let(:users) { table(:users) }
subject { described_class.new }
describe '#perform' do
context 'token is not yet hashed' do
before do
users.create(id: 1, email: 'user@example.com', projects_limit: 10)
personal_access_tokens.create!(id: 1, user_id: 1, name: 'pat-01', token: 'token-01')
end
it 'saves token digest' do
expect { subject.perform(PersonalAccessToken, :token, :token_digest, 1, 2) }.to(
change { PersonalAccessToken.find(1).token_digest }.from(nil).to(Gitlab::CryptoHelper.sha256('token-01')))
end
it 'erases token' do
expect { subject.perform(PersonalAccessToken, :token, :token_digest, 1, 2) }.to(
change { PersonalAccessToken.find(1).read_attribute(:token) }.from('token-01').to(nil))
end
end
context 'token is already hashed' do
before do
users.create(id: 1, email: 'user@example.com', projects_limit: 10)
personal_access_tokens.create!(id: 1, user_id: 1, name: 'pat-01', token_digest: 'token-digest-01')
end
it 'does not change existing token digest' do
expect { subject.perform(PersonalAccessToken, :token, :token_digest, 1, 2) }.not_to(
change { PersonalAccessToken.find(1).token_digest })
end
it 'leaves token empty' do
expect { subject.perform(PersonalAccessToken, :token, :token_digest, 1, 2) }.not_to(
change { PersonalAccessToken.find(1).read_attribute(:token) }.from(nil))
end
end
end
end
spec/lib/gitlab/background_migration/encrypt_columns_spec.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::EncryptColumns, schema: 20180910115836 do
let(:model) { Gitlab::BackgroundMigration::Models::EncryptColumns::WebHook }
let(:web_hooks) { table(:web_hooks) }
let(:plaintext_attrs) do
{
'encrypted_token' => nil,
'encrypted_url' => nil,
'token' => 'secret',
'url' => 'http://example.com?access_token=secret'
}
end
let(:encrypted_attrs) do
{
'encrypted_token' => be_present,
'encrypted_url' => be_present,
'token' => nil,
'url' => nil
}
end
describe '#perform' do
it 'encrypts columns for the specified range' do
hooks = web_hooks.create([plaintext_attrs] * 5).sort_by(&:id)
# Encrypt all but the first and last rows
subject.perform(model, [:token, :url], hooks[1].id, hooks[3].id)
hooks = web_hooks.where(id: hooks.map(&:id)).order(:id)
aggregate_failures do
expect(hooks[0]).to have_attributes(plaintext_attrs)
expect(hooks[1]).to have_attributes(encrypted_attrs)
expect(hooks[2]).to have_attributes(encrypted_attrs)
expect(hooks[3]).to have_attributes(encrypted_attrs)
expect(hooks[4]).to have_attributes(plaintext_attrs)
end
end
it 'acquires an exclusive lock for the update' do
relation = double('relation', each: nil)
expect(model).to receive(:where) { relation }
expect(relation).to receive(:lock) { relation }
subject.perform(model, [:token, :url], 1, 1)
end
it 'skips already-encrypted columns' do
values = {
'encrypted_token' => 'known encrypted token',
'encrypted_url' => 'known encrypted url',
'token' => 'token',
'url' => 'url'
}
hook = web_hooks.create(values)
subject.perform(model, [:token, :url], hook.id, hook.id)
hook.reload
expect(hook).to have_attributes(values)
end
it 'reloads the model column information' do
expect(model).to receive(:reset_column_information).and_call_original
expect(model).to receive(:define_attribute_methods).and_call_original
subject.perform(model, [:token, :url], 1, 1)
end
it 'fails if a source column is not present' do
columns = model.columns.reject { |c| c.name == 'url' }
allow(model).to receive(:columns) { columns }
expect do
subject.perform(model, [:token, :url], 1, 1)
end.to raise_error(/source column: url is missing/)
end
it 'fails if a destination column is not present' do
columns = model.columns.reject { |c| c.name == 'encrypted_url' }
allow(model).to receive(:columns) { columns }
expect do
subject.perform(model, [:token, :url], 1, 1)
end.to raise_error(/destination column: encrypted_url is missing/)
end
end
end
spec/lib/gitlab/background_migration/encrypt_runners_tokens_spec.rb deleted 100644 → 0
View file @ c5cb2100
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::EncryptRunnersTokens, schema: 20181121111200 do
let(:settings) { table(:application_settings) }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:runners) { table(:ci_runners) }
context 'when migrating application settings' do
before do
settings.create!(id: 1, runners_registration_token: 'plain-text-token1')
end
it 'migrates runners registration tokens' do
migrate!(:settings, 1, 1)
encrypted_token = settings.first.runners_registration_token_encrypted
decrypted_token = ::Gitlab::CryptoHelper.aes256_gcm_decrypt(encrypted_token)
expect(decrypted_token).to eq 'plain-text-token1'
expect(settings.first.runners_registration_token).to eq 'plain-text-token1'
end
end
context 'when migrating namespaces' do
before do
namespaces.create!(id: 11, name: 'gitlab', path: 'gitlab-org', runners_token: 'my-token1')
namespaces.create!(id: 12, name: 'gitlab', path: 'gitlab-org', runners_token: 'my-token2')
namespaces.create!(id: 22, name: 'gitlab', path: 'gitlab-org', runners_token: 'my-token3')
end
it 'migrates runners registration tokens' do
migrate!(:namespace, 11, 22)
expect(namespaces.all.reload).to all(
have_attributes(runners_token: be_a(String), runners_token_encrypted: be_a(String))
)
end
end
context 'when migrating projects' do
before do
namespaces.create!(id: 11, name: 'gitlab', path: 'gitlab-org')
projects.create!(id: 111, namespace_id: 11, name: 'gitlab', path: 'gitlab-ce', runners_token: 'my-token1')
projects.create!(id: 114, namespace_id: 11, name: 'gitlab', path: 'gitlab-ce', runners_token: 'my-token2')
projects.create!(id: 116, namespace_id: 11, name: 'gitlab', path: 'gitlab-ce', runners_token: 'my-token3')
end
it 'migrates runners registration tokens' do
migrate!(:project, 111, 116)
expect(projects.all.reload).to all(
have_attributes(runners_token: be_a(String), runners_token_encrypted: be_a(String))
)
end
end
context 'when migrating runners' do
before do
runners.create!(id: 201, runner_type: 1, token: 'plain-text-token1')
runners.create!(id: 202, runner_type: 1, token: 'plain-text-token2')
runners.create!(id: 203, runner_type: 1, token: 'plain-text-token3')
end
it 'migrates runners communication tokens' do
migrate!(:runner, 201, 203)
expect(runners.all.reload).to all(
have_attributes(token: be_a(String), token_encrypted: be_a(String))
)
end
end
def migrate!(model, from, to)
subject.perform(model, from, to)
end
end
spec/lib/gitlab/background_migration/fix_cross_project_label_links_spec.rb
View file @ d43602d8
......@@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::FixCrossProjectLabelLinks, schema: 20180702120647 do
RSpec.describe Gitlab::BackgroundMigration::FixCrossProjectLabelLinks do
let(:namespaces_table) { table(:namespaces) }
let(:projects_table) { table(:projects) }
let(:issues_table) { table(:issues) }
......
spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb
View file @ d43602d8
......@@ -2,7 +2,8 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::MigrateBuildStage, schema: 20180212101928 do
RSpec.describe Gitlab::BackgroundMigration::MigrateBuildStage do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:pipelines) { table(:ci_pipelines) }
let(:stages) { table(:ci_stages) }
......@@ -22,7 +23,8 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateBuildStage, schema: 201802121
end
before do
projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce')
namespace = namespaces.create!(name: 'gitlab-org', path: 'gitlab-org')
projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id)
pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
jobs.create!(id: 1, commit_id: 1, project_id: 123,
......
spec/lib/gitlab/background_migration/migrate_legacy_artifacts_spec.rb
View file @ d43602d8
......@@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::MigrateLegacyArtifacts, schema: 20180816161409 do
RSpec.describe Gitlab::BackgroundMigration::MigrateLegacyArtifacts do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:pipelines) { table(:ci_pipelines) }
......
spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb
View file @ d43602d8
......@@ -2,35 +2,33 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::MigrateStageIndex, schema: 20180420080616 do
RSpec.describe Gitlab::BackgroundMigration::MigrateStageIndex do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:pipelines) { table(:ci_pipelines) }
let(:stages) { table(:ci_stages) }
let(:jobs) { table(:ci_builds) }
let(:namespace) { namespaces.create(name: 'gitlab-org', path: 'gitlab-org') }
let(:project) { projects.create!(namespace_id: namespace.id, name: 'gitlab', path: 'gitlab') }
let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a') }
let(:stage1) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'build') }
let(:stage2) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'test') }
before do
namespaces.create(id: 10, name: 'gitlab-org', path: 'gitlab-org')
projects.create!(id: 11, namespace_id: 10, name: 'gitlab', path: 'gitlab')
pipelines.create!(id: 12, project_id: 11, ref: 'master', sha: 'adf43c3a')
stages.create(id: 100, project_id: 11, pipeline_id: 12, name: 'build')
stages.create(id: 101, project_id: 11, pipeline_id: 12, name: 'test')
jobs.create!(id: 121, commit_id: 12, project_id: 11,
stage_idx: 2, stage_id: 100)
jobs.create!(id: 122, commit_id: 12, project_id: 11,
stage_idx: 2, stage_id: 100)
jobs.create!(id: 123, commit_id: 12, project_id: 11,
stage_idx: 10, stage_id: 100)
jobs.create!(id: 124, commit_id: 12, project_id: 11,
stage_idx: 3, stage_id: 101)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 2, stage_id: stage1.id)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 2, stage_id: stage1.id)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 10, stage_id: stage1.id)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 3, stage_id: stage2.id)
end
it 'correctly migrates stages indices' do
expect(stages.all.pluck(:position)).to all(be_nil)
described_class.new.perform(100, 101)
described_class.new.perform(stage1.id, stage2.id)
expect(stages.all.order(:id).pluck(:position)).to eq [2, 3]
end
......
spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb
View file @ d43602d8
......@@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::PopulateClusterKubernetesNamespaceTable, schema: 20181022173835 do
RSpec.describe Gitlab::BackgroundMigration::PopulateClusterKubernetesNamespaceTable do
include MigrationHelpers::ClusterHelpers
let(:migration) { described_class.new }
......
spec/lib/gitlab/background_migration/populate_untracked_uploads_dependencies/untracked_file_spec.rb
View file @ d43602d8
......@@ -2,8 +2,7 @@
require 'spec_helper'
# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
RSpec.describe Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile, schema: 20180208183958 do
RSpec.describe Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile do
include MigrationsHelpers::TrackUntrackedUploadsHelpers
let!(:appearances) { table(:appearances) }
......
spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb
View file @ d43602d8
......@@ -2,8 +2,7 @@
require 'spec_helper'
# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
RSpec.describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, schema: 20180208183958 do
RSpec.describe Gitlab::BackgroundMigration::PopulateUntrackedUploads do
include MigrationsHelpers::TrackUntrackedUploadsHelpers
subject { described_class.new }
......
spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb
View file @ d43602d8
......@@ -3,7 +3,7 @@
require 'spec_helper'
# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
RSpec.describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, schema: 20180208183958 do
RSpec.describe Gitlab::BackgroundMigration::PrepareUntrackedUploads do
include MigrationsHelpers::TrackUntrackedUploadsHelpers
let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
......
spec/lib/gitlab/background_migration/remove_restricted_todos_spec.rb
View file @ d43602d8
......@@ -2,7 +2,8 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos, schema: 20180704204006 do
RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:users) { table(:users) }
let(:todos) { table(:todos) }
......@@ -18,8 +19,9 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos, schema: 20180
users.create(id: 2, email: 'reporter@example.com', projects_limit: 10)
users.create(id: 3, email: 'guest@example.com', projects_limit: 10)
projects.create!(id: 1, name: 'project-1', path: 'project-1', visibility_level: 0, namespace_id: 1)
projects.create!(id: 2, name: 'project-2', path: 'project-2', visibility_level: 0, namespace_id: 1)
namespace = namespaces.create(name: 'gitlab-org', path: 'gitlab-org')
projects.create!(id: 1, name: 'project-1', path: 'project-1', visibility_level: 0, namespace_id: namespace.id)
projects.create!(id: 2, name: 'project-2', path: 'project-2', visibility_level: 0, namespace_id: namespace.id)
issues.create(id: 1, project_id: 1)
issues.create(id: 2, project_id: 2)
......@@ -92,7 +94,7 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos, schema: 20180
context 'when issues are restricted to project members' do
before do
project_features.create(issues_access_level: 10, project_id: 2)
project_features.create(issues_access_level: 10, pages_access_level: 10, project_id: 2)
end
it 'removes non members issue todos' do
......@@ -102,7 +104,7 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos, schema: 20180
context 'when merge requests are restricted to project members' do
before do
project_features.create(merge_requests_access_level: 10, project_id: 2)
project_features.create(merge_requests_access_level: 10, pages_access_level: 10, project_id: 2)
end
it 'removes non members issue todos' do
......@@ -112,7 +114,7 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos, schema: 20180
context 'when repository and merge requests are restricted to project members' do
before do
project_features.create(repository_access_level: 10, merge_requests_access_level: 10, project_id: 2)
project_features.create(repository_access_level: 10, merge_requests_access_level: 10, pages_access_level: 10, project_id: 2)
end
it 'removes non members commit and merge requests todos' do
......
spec/lib/gitlab/background_migration/set_confidential_note_events_on_services_spec.rb
View file @ d43602d8
......@@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnServices, schema: 20180122154930 do
RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnServices do
let(:services) { table(:services) }
describe '#perform' do
......
spec/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks_spec.rb
View file @ d43602d8
......@@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnWebhooks, schema: 20180104131052 do
RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnWebhooks do
let(:web_hooks) { table(:web_hooks) }
describe '#perform' do
......
spec/lib/quality/test_level_spec.rb
View file @ d43602d8
......@@ -28,7 +28,7 @@ RSpec.describe Quality::TestLevel do
context 'when level is migration' do
it 'returns a pattern' do
expect(subject.pattern(:migration))
.to eq("spec/{migrations}{,/**/}*_spec.rb")
.to eq("spec/{migrations,lib/gitlab/background_migration,lib/ee/gitlab/background_migration}{,/**/}*_spec.rb")
end
end
......@@ -96,7 +96,7 @@ RSpec.describe Quality::TestLevel do
context 'when level is migration' do
it 'returns a regexp' do
expect(subject.regexp(:migration))
.to eq(%r{spec/(migrations)})
.to eq(%r{spec/(migrations|lib/gitlab/background_migration|lib/ee/gitlab/background_migration)})
end
end
......
spec/migrations/active_record/schema_spec.rb
View file @ d43602d8
......@@ -25,6 +25,6 @@ RSpec.describe ActiveRecord::Schema, schema: :latest do
it 'the schema_migrations table contains all schema versions' do
versions = ActiveRecord::Base.connection.execute('SELECT version FROM schema_migrations ORDER BY version').map { |m| Integer(m['version']) }
expect(versions).to eq(all_migrations)
expect(versions).to match_array(all_migrations)
end
end
spec/lib/gitlab/background_migration/schedule_calculate_wiki_sizes_spec.rb → spec/migrations/schedule_calculate_wiki_sizes_spec.rb
View file @ d43602d8
......@@ -10,45 +10,41 @@ RSpec.describe ScheduleCalculateWikiSizes do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:project_statistics) { table(:project_statistics) }
let(:namespace) { namespaces.create!(name: 'wiki-migration', path: 'wiki-migration') }
let(:project1) { projects.create!(name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: namespace.id) }
let(:project2) { projects.create!(name: 'wiki-project-2', path: 'wiki-project-2', namespace_id: namespace.id) }
let(:project3) { projects.create!(name: 'wiki-project-3', path: 'wiki-project-3', namespace_id: namespace.id) }
context 'when missing wiki sizes exist' do
before do
namespaces.create!(id: 1, name: 'wiki-migration', path: 'wiki-migration')
projects.create!(id: 1, name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: 1)
projects.create!(id: 2, name: 'wiki-project-2', path: 'wiki-project-2', namespace_id: 1)
projects.create!(id: 3, name: 'wiki-project-3', path: 'wiki-project-3', namespace_id: 1)
project_statistics.create!(id: 1, project_id: 1, namespace_id: 1, wiki_size: 1000)
project_statistics.create!(id: 2, project_id: 2, namespace_id: 1, wiki_size: nil)
project_statistics.create!(id: 3, project_id: 3, namespace_id: 1, wiki_size: nil)
end
let!(:project_statistic1) { project_statistics.create!(id: 1, project_id: project1.id, namespace_id: namespace.id, wiki_size: 1000) }
let!(:project_statistic2) { project_statistics.create!(id: 2, project_id: project2.id, namespace_id: namespace.id, wiki_size: nil) }
let!(:project_statistic3) { project_statistics.create!(id: 3, project_id: project3.id, namespace_id: namespace.id, wiki_size: nil) }
it 'schedules a background migration' do
Sidekiq::Testing.fake! do
Timecop.freeze do
migrate!
Timecop.freeze do
migrate!
expect(migration_name).to be_scheduled_delayed_migration(5.minutes, 2, 3)
expect(BackgroundMigrationWorker.jobs.size).to eq 1
end
expect(migration_name).to be_scheduled_delayed_migration(5.minutes, project_statistic2.id, project_statistic3.id)
expect(BackgroundMigrationWorker.jobs.size).to eq 1
end
end
it 'calculates missing wiki sizes', :sidekiq_might_not_need_inline do
expect(project_statistics.find_by(id: 2).wiki_size).to be_nil
expect(project_statistics.find_by(id: 3).wiki_size).to be_nil
it 'calculates missing wiki sizes', :sidekiq_inline do
expect(project_statistic2.wiki_size).to be_nil
expect(project_statistic3.wiki_size).to be_nil
migrate!
expect(project_statistics.find_by(id: 2).wiki_size).not_to be_nil
expect(project_statistics.find_by(id: 3).wiki_size).not_to be_nil
expect(project_statistic2.reload.wiki_size).not_to be_nil
expect(project_statistic3.reload.wiki_size).not_to be_nil
end
end
context 'when missing wiki sizes do not exist' do
before do
namespaces.create!(id: 1, name: 'wiki-migration', path: 'wiki-migration')
projects.create!(id: 1, name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: 1)
project_statistics.create!(id: 1, project_id: 1, namespace_id: 1, wiki_size: 1000)
namespace = namespaces.create!(name: 'wiki-migration', path: 'wiki-migration')
project = projects.create!(name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: namespace.id)
project_statistics.create!(project_id: project.id, namespace_id: 1, wiki_size: 1000)
end
it 'does not schedule a background migration' do
......
spec/support/matchers/background_migrations_matchers.rb
View file @ d43602d8
......@@ -10,7 +10,8 @@ RSpec::Matchers.define :be_scheduled_delayed_migration do |delay, *expected|
failure_message do |migration|
"Migration `#{migration}` with args `#{expected.inspect}` " \
'not scheduled in expected time!'
"not scheduled in expected time! Expected any of `#{BackgroundMigrationWorker.jobs.map { |j| j['args'] }}` to be `#{[migration, expected]}` " \
"and any of `#{BackgroundMigrationWorker.jobs.map { |j| j['at'].to_i }}` to be `#{delay.to_i + Time.now.to_i}` (`#{delay.to_i}` + `#{Time.now.to_i}`)."
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7