Commit 90a99ff3 authored by Grzegorz Bizon's avatar Grzegorz Bizon Committed by Nick Thomas

Add pipeline stages position clean-up migration

parent f316cb9b
---
title: Fully migrate pipeline stages position
merge_request: 19369
author:
type: performance
class CleanupStagesPositionMigration < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
TMP_INDEX_NAME = 'tmp_id_stage_position_partial_null_index'.freeze
disable_ddl_transaction!
class Stages < ActiveRecord::Base
include EachBatch
self.table_name = 'ci_stages'
end
def up
disable_statement_timeout
Gitlab::BackgroundMigration.steal('MigrateStageIndex')
unless index_exists_by_name?(:ci_stages, TMP_INDEX_NAME)
add_concurrent_index(:ci_stages, :id, where: 'position IS NULL', name: TMP_INDEX_NAME)
end
migratable = <<~SQL
position IS NULL AND EXISTS (
SELECT 1 FROM ci_builds WHERE stage_id = ci_stages.id AND stage_idx IS NOT NULL
)
SQL
Stages.where(migratable).each_batch(of: 1000) do |batch|
batch.pluck(:id).each do |stage|
Gitlab::BackgroundMigration::MigrateStageIndex.new.perform(stage, stage)
end
end
remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME)
end
def down
if index_exists_by_name?(:ci_stages, TMP_INDEX_NAME)
remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME)
end
end
end
...@@ -32,7 +32,8 @@ with all their related data and be moved into a new GitLab instance. ...@@ -32,7 +32,8 @@ with all their related data and be moved into a new GitLab instance.
| GitLab version | Import/Export version | | GitLab version | Import/Export version |
| ---------------- | --------------------- | | ---------------- | --------------------- |
| 10.8 to current | 0.2.3 | | 11.1 to current | 0.2.4 |
| 10.8 | 0.2.3 |
| 10.4 | 0.2.2 | | 10.4 | 0.2.2 |
| 10.3 | 0.2.1 | | 10.3 | 0.2.1 |
| 10.0 | 0.2.0 | | 10.0 | 0.2.0 |
......
...@@ -3,7 +3,7 @@ module Gitlab ...@@ -3,7 +3,7 @@ module Gitlab
extend self extend self
# For every version update, the version history in import_export.md has to be kept up to date. # For every version update, the version history in import_export.md has to be kept up to date.
VERSION = '0.2.3'.freeze VERSION = '0.2.4'.freeze
FILENAME_LIMIT = 50 FILENAME_LIMIT = 50
def export_path(relative_path:) def export_path(relative_path:)
......
...@@ -10,15 +10,22 @@ namespace :gitlab do ...@@ -10,15 +10,22 @@ namespace :gitlab do
puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true)
end end
desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' desc 'GitLab | Bumps the Import/Export version in fixtures and project templates'
task bump_test_version: :environment do task bump_version: :environment do
Dir.mktmpdir do |tmp_dir| archives = Dir['vendor/project_templates/*.tar.gz']
system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") archives.push('spec/features/projects/import_export/test_project_export.tar.gz')
File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") archives.each do |archive|
raise ArgumentError unless File.exist?(archive)
Dir.mktmpdir do |tmp_dir|
system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null")
File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null")
end
end end
puts "Updated to #{Gitlab::ImportExport.version}" puts "Updated #{archives} to #{Gitlab::ImportExport.version}."
end end
end end
end end
...@@ -4,14 +4,14 @@ describe Gitlab::ImportExport::Importer do ...@@ -4,14 +4,14 @@ describe Gitlab::ImportExport::Importer do
let(:user) { create(:user) } let(:user) { create(:user) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" } let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared } let(:shared) { project.import_export_shared }
let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) } let(:project) { create(:project, import_source: File.join(test_path, 'test_project_export.tar.gz')) }
subject(:importer) { described_class.new(project) } subject(:importer) { described_class.new(project) }
before do before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
FileUtils.mkdir_p(shared.export_path) FileUtils.mkdir_p(shared.export_path)
FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path) FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path)
allow(subject).to receive(:remove_import_file) allow(subject).to receive(:remove_import_file)
end end
......
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20180604123514_cleanup_stages_position_migration.rb')
describe CleanupStagesPositionMigration, :migration, :sidekiq, :redis do
let(:migration) { spy('migration') }
before do
allow(Gitlab::BackgroundMigration::MigrateStageIndex)
.to receive(:new).and_return(migration)
end
context 'when there are pending background migrations' do
it 'processes pending jobs synchronously' do
Sidekiq::Testing.disable! do
BackgroundMigrationWorker
.perform_in(2.minutes, 'MigrateStageIndex', [1, 1])
BackgroundMigrationWorker
.perform_async('MigrateStageIndex', [1, 1])
migrate!
expect(migration).to have_received(:perform).with(1, 1).twice
end
end
end
context 'when there are no background migrations pending' do
it 'does nothing' do
Sidekiq::Testing.disable! do
migrate!
expect(migration).not_to have_received(:perform)
end
end
end
context 'when there are still unmigrated stages present' do
let(:stages) { table('ci_stages') }
let(:builds) { table('ci_builds') }
let!(:entities) do
%w[build test broken].map do |name|
stages.create(name: name)
end
end
before do
stages.update_all(position: nil)
builds.create(name: 'unit', stage_id: entities.first.id, stage_idx: 1, ref: 'master')
builds.create(name: 'unit', stage_id: entities.second.id, stage_idx: 1, ref: 'master')
end
it 'migrates stages sequentially for every stage' do
expect(stages.all).to all(have_attributes(position: nil))
migrate!
expect(migration).to have_received(:perform)
.with(entities.first.id, entities.first.id)
expect(migration).to have_received(:perform)
.with(entities.second.id, entities.second.id)
expect(migration).not_to have_received(:perform)
.with(entities.third.id, entities.third.id)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment