Merge branch 'jp-route-backfill-ns' into 'master'

Backfill route namespace_id for namespaces

See merge request gitlab-org/gitlab!78393

db/post_migrate/20220120123700_add_tmp_index_routes_id_for_namespaces.rb

+# frozen_string_literal: true
+
+class AddTmpIndexRoutesIdForNamespaces < Gitlab::Database::Migration[1.0]
+  INDEX_NAME = 'tmp_index_for_namespace_id_migration_on_routes'
+
+  disable_ddl_transaction!
+
+  def up
+    # Temporary index to be removed in 14.9
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/352353
+    add_concurrent_index :routes, :id, where: "routes.namespace_id is null and routes.source_type = 'Namespace'", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :routes, INDEX_NAME
+  end
+end

db/post_migrate/20220120123800_backfill_namespace_id_for_namespace_routes.rb

+# frozen_string_literal: true
+
+class BackfillNamespaceIdForNamespaceRoutes < Gitlab::Database::Migration[1.0]
+  MIGRATION = 'BackfillNamespaceIdForNamespaceRoute'
+  INTERVAL = 2.minutes
+  BATCH_SIZE = 1_000
+  MAX_BATCH_SIZE = 10_000
+  SUB_BATCH_SIZE = 200
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :routes,
+      :id,
+      job_interval: INTERVAL,
+      batch_size: BATCH_SIZE,
+      max_batch_size: MAX_BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    Gitlab::Database::BackgroundMigration::BatchedMigration
+      .for_configuration(MIGRATION, :routes, :id, [])
+      .delete_all
+  end
+end

db/schema_migrations/20220120123700

+9e274eae18520821dd890a11c8c6192da82a6051dce9ec2934b1365e57a10fdb
\ No newline at end of file

db/schema_migrations/20220120123800

+9d2f0b0d2cb1a5844bdca5bcb4b82fa5cc6b465fa19177f9f6ca16574128fdc8
\ No newline at end of file

db/structure.sql

@@ -28311,6 +28311,8 @@ CREATE INDEX tmp_idx_deduplicate_vulnerability_occurrences ON vulnerability_occu
 
 CREATE INDEX tmp_idx_vulnerability_occurrences_on_id_where_report_type_7_99 ON vulnerability_occurrences USING btree (id) WHERE (report_type = ANY (ARRAY[7, 99]));
 
+CREATE INDEX tmp_index_for_namespace_id_migration_on_routes ON routes USING btree (id) WHERE ((namespace_id IS NULL) AND ((source_type)::text = 'Namespace'::text));
+
 CREATE INDEX tmp_index_members_on_state ON members USING btree (state) WHERE (state = 2);
 
 CREATE INDEX tmp_index_namespaces_empty_traversal_ids_with_child_namespaces ON namespaces USING btree (id) WHERE ((parent_id IS NOT NULL) AND (traversal_ids = '{}'::integer[]));

lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfills the `routes.namespace_id` column, by copying source_id value
+    # (for groups and user namespaces source_id == namespace_id)
+    class BackfillNamespaceIdForNamespaceRoute
+      include Gitlab::Database::DynamicModelHelpers
+
+      def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms)
+        parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
+
+        parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
+          batch_metrics.time_operation(:update_all) do
+            sub_batch.update_all('namespace_id=source_id')
+          end
+
+          pause_ms = [0, pause_ms].max
+          sleep(pause_ms * 0.001)
+        end
+      end
+
+      def batch_metrics
+        @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
+      end
+
+      private
+
+      def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
+        define_batchable_model(source_table, connection: ActiveRecord::Base.connection)
+          .joins('inner join namespaces on routes.source_id = namespaces.id')
+          .where(source_key_column => start_id..stop_id)
+          .where(namespace_id: nil)
+          .where(source_type: 'Namespace')
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForNamespaceRoute, :migration, schema: 20220120123800 do
+  let(:migration) { described_class.new }
+  let(:namespaces_table) { table(:namespaces) }
+  let(:projects_table) { table(:projects) }
+  let(:routes_table) { table(:routes) }
+
+  let(:table_name) { 'routes' }
+  let(:batch_column) { :id }
+  let(:sub_batch_size) { 200 }
+  let(:pause_ms) { 0 }
+
+  let(:namespace1) { namespaces_table.create!(name: 'namespace1', path: 'namespace1', type: 'User') }
+  let(:namespace2) { namespaces_table.create!(name: 'namespace2', path: 'namespace2', type: 'Group') }
+  let(:namespace3) { namespaces_table.create!(name: 'namespace3', path: 'namespace3', type: 'Group') }
+  let(:namespace4) { namespaces_table.create!(name: 'namespace4', path: 'namespace4', type: 'Group') }
+  let(:project1) { projects_table.create!(name: 'project1', namespace_id: namespace1.id) }
+
+  subject(:perform_migration) { migration.perform(1, 10, table_name, batch_column, sub_batch_size, pause_ms) }
+
+  before do
+    routes_table.create!(id: 1, name: 'test1', path: 'test1', source_id: namespace1.id,
+                         source_type: namespace1.class.sti_name)
+    routes_table.create!(id: 2, name: 'test2', path: 'test2', source_id: namespace2.id,
+                         source_type: namespace2.class.sti_name)
+    routes_table.create!(id: 5, name: 'test3', path: 'test3', source_id: project1.id,
+                         source_type: project1.class.sti_name) # should be ignored - project route
+    routes_table.create!(id: 6, name: 'test4', path: 'test4', source_id: non_existing_record_id,
+                         source_type: namespace3.class.sti_name) # should be ignored - invalid source_id
+    routes_table.create!(id: 10, name: 'test5', path: 'test5', source_id: namespace3.id,
+                         source_type: namespace3.class.sti_name)
+    routes_table.create!(id: 11, name: 'test6', path: 'test6', source_id: namespace4.id,
+                         source_type: namespace4.class.sti_name) # should be ignored - outside the scope
+  end
+
+  it 'backfills `type` for the selected records', :aggregate_failures do
+    perform_migration
+
+    expect(routes_table.where.not(namespace_id: nil).pluck(:id)).to match_array([1, 2, 10])
+  end
+
+  it 'tracks timings of queries' do
+    expect(migration.batch_metrics.timings).to be_empty
+
+    expect { perform_migration }.to change { migration.batch_metrics.timings }
+  end
+end

spec/migrations/backfill_namespace_id_for_namespace_routes_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe BackfillNamespaceIdForNamespaceRoutes do
+  let_it_be(:migration) { described_class::MIGRATION }
+
+  describe '#up' do
+    it 'schedules background jobs for each batch of routes' do
+      migrate!
+
+      expect(migration).to have_scheduled_batched_migration(
+        table_name: :routes,
+        column_name: :id,
+        interval: described_class::INTERVAL
+      )
+    end
+  end
+
+  describe '#down' do
+    it 'deletes all batched migration records' do
+      migrate!
+      schema_migrate_down!
+
+      expect(migration).not_to have_scheduled_batched_migration
+    end
+  end
+end