diff --git a/changelogs/unreleased/add-background-migration-to-fill-file-store.yml b/changelogs/unreleased/add-background-migration-to-fill-file-store.yml new file mode 100644 index 0000000000000000000000000000000000000000..ab6bde86fd4eb7fc8c58620810416f9b789deac0 --- /dev/null +++ b/changelogs/unreleased/add-background-migration-to-fill-file-store.yml @@ -0,0 +1,5 @@ +--- +title: Add backgound migration for filling nullfied file_store columns +merge_request: 18557 +author: +type: performance diff --git a/db/post_migrate/20180424151928_fill_file_store.rb b/db/post_migrate/20180424151928_fill_file_store.rb new file mode 100644 index 0000000000000000000000000000000000000000..b41feb233be90c51dade85ad10bd4d2161a4f8ca --- /dev/null +++ b/db/post_migrate/20180424151928_fill_file_store.rb @@ -0,0 +1,72 @@ +class FillFileStore < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class JobArtifact < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_job_artifacts' + BATCH_SIZE = 10_000 + + def self.params_for_background_migration + yield self.where(file_store: nil), 'FillFileStoreJobArtifact', 5.minutes, BATCH_SIZE + end + end + + class LfsObject < ActiveRecord::Base + include EachBatch + self.table_name = 'lfs_objects' + BATCH_SIZE = 10_000 + + def self.params_for_background_migration + yield self.where(file_store: nil), 'FillFileStoreLfsObject', 5.minutes, BATCH_SIZE + end + end + + class Upload < ActiveRecord::Base + include EachBatch + self.table_name = 'uploads' + self.inheritance_column = :_type_disabled # Disable STI + BATCH_SIZE = 10_000 + + def self.params_for_background_migration + yield self.where(store: nil), 'FillStoreUpload', 5.minutes, BATCH_SIZE + end + end + + def up + # NOTE: Schedule background migrations that fill 'NULL' value by '1'(ObjectStorage::Store::LOCAL) on `file_store`, `store` columns + # + # Here are the target columns + # - ci_job_artifacts.file_store + # - lfs_objects.file_store + # - uploads.store + + FillFileStore::JobArtifact.params_for_background_migration do |relation, class_name, delay_interval, batch_size| + queue_background_migration_jobs_by_range_at_intervals(relation, + class_name, + delay_interval, + batch_size: batch_size) + end + + FillFileStore::LfsObject.params_for_background_migration do |relation, class_name, delay_interval, batch_size| + queue_background_migration_jobs_by_range_at_intervals(relation, + class_name, + delay_interval, + batch_size: batch_size) + end + + FillFileStore::Upload.params_for_background_migration do |relation, class_name, delay_interval, batch_size| + queue_background_migration_jobs_by_range_at_intervals(relation, + class_name, + delay_interval, + batch_size: batch_size) + end + end + + def down + # noop + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..22b0ac71920f7bf43248f6067e073f95b4299d3d --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreJobArtifact + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + end + + def perform(start_id, stop_id) + FillFileStoreJobArtifact::JobArtifact + .where(file_store: nil) + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb new file mode 100644 index 0000000000000000000000000000000000000000..d0816ae3ed5c84c2778d7fef114be6aa7c2b2634 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreLfsObject + class LfsObject < ActiveRecord::Base + self.table_name = 'lfs_objects' + end + + def perform(start_id, stop_id) + FillFileStoreLfsObject::LfsObject + .where(file_store: nil) + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb new file mode 100644 index 0000000000000000000000000000000000000000..94c65459a67e1b017ef1d4542685d8d7373790b9 --- /dev/null +++ b/lib/gitlab/background_migration/fill_store_upload.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillStoreUpload + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillStoreUpload::Upload + .where(store: nil) + .where(id: (start_id..stop_id)) + .update_all(store: 1) + end + end + end +end diff --git a/spec/migrations/fill_file_store_spec.rb b/spec/migrations/fill_file_store_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..5ff7aa56ce2ac2d0f1a9db2ec7ef48cb9f9f971f --- /dev/null +++ b/spec/migrations/fill_file_store_spec.rb @@ -0,0 +1,43 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180424151928_fill_file_store') + +describe FillFileStore, :migration do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:builds) { table(:ci_builds) } + let(:job_artifacts) { table(:ci_job_artifacts) } + let(:lfs_objects) { table(:lfs_objects) } + let(:uploads) { table(:uploads) } + + before do + namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) + builds.create!(id: 1) + + ## + # Create rows that have nullfied `file_store` column + job_artifacts.create!(project_id: 123, job_id: 1, file_type: 1, file_store: nil) + lfs_objects.create!(oid: 123, size: 10, file: 'file_name', file_store: nil) + uploads.create!(size: 10, path: 'path', uploader: 'uploader', mount_point: 'file_name', store: nil) + end + + it 'correctly migrates nullified file_store/store column' do + expect(job_artifacts.where(file_store: nil).count).to eq(1) + expect(lfs_objects.where(file_store: nil).count).to eq(1) + expect(uploads.where(store: nil).count).to eq(1) + + expect(job_artifacts.where(file_store: 1).count).to eq(0) + expect(lfs_objects.where(file_store: 1).count).to eq(0) + expect(uploads.where(store: 1).count).to eq(0) + + migrate! + + expect(job_artifacts.where(file_store: nil).count).to eq(0) + expect(lfs_objects.where(file_store: nil).count).to eq(0) + expect(uploads.where(store: nil).count).to eq(0) + + expect(job_artifacts.where(file_store: 1).count).to eq(1) + expect(lfs_objects.where(file_store: 1).count).to eq(1) + expect(uploads.where(store: 1).count).to eq(1) + end +end