Merge branch 'create-namespace-and-project-mirrors-table' into 'master'

Create ci_namespace_mirrors and ci_project_mirrors tables

See merge request gitlab-org/gitlab!75621

app/models/ci/namespace_mirror.rb

+# frozen_string_literal: true
+
+module Ci
+  # This model represents a record in a shadow table of the main database's namespaces table.
+  # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN.
+  class NamespaceMirror < ApplicationRecord
+    # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517
+  end
+end

app/models/ci/project_mirror.rb

+# frozen_string_literal: true
+
+module Ci
+  # This model represents a shadow table of the main database's projects table.
+  # It allows us to navigate the project and namespace hierarchy on the ci database.
+  class ProjectMirror < ApplicationRecord
+    # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517
+  end
+end

db/migrate/20211011140930_create_ci_namespace_mirrors.rb

+# frozen_string_literal: true
+
+class CreateCiNamespaceMirrors < Gitlab::Database::Migration[1.0]
+  TABLE_NAME = :ci_namespace_mirrors
+  INDEX_NAME = "index_gin_#{TABLE_NAME}_on_traversal_ids"
+
+  def change
+    create_table TABLE_NAME do |t|
+      t.integer :namespace_id, null: false, index: { unique: true }
+      t.integer :traversal_ids, array: true, default: [], null: false
+
+      t.index :traversal_ids, name: INDEX_NAME, using: :gin
+    end
+  end
+end

db/migrate/20211011140931_create_ci_project_mirrors.rb

+# frozen_string_literal: true
+
+class CreateCiProjectMirrors < Gitlab::Database::Migration[1.0]
+  TABLE_NAME = :ci_project_mirrors
+
+  def change
+    create_table TABLE_NAME do |t|
+      t.integer :project_id, null: false, index: { unique: true }
+      t.integer :namespace_id, null: false, index: true
+    end
+  end
+end

db/schema_migrations/20211011140930

+cdae819e8de3b5ad721014376bfd9af97a45e953e2d345daf62784f986a5eb31
\ No newline at end of file

db/schema_migrations/20211011140931

+7e51eb4443fd74da9bef4d9c1c3cc40376c311abbc05ca7871f725fada79b48a
\ No newline at end of file

db/structure.sql

@@ -11767,6 +11767,21 @@ CREATE SEQUENCE ci_minutes_additional_packs_id_seq
 
 ALTER SEQUENCE ci_minutes_additional_packs_id_seq OWNED BY ci_minutes_additional_packs.id;
 
+CREATE TABLE ci_namespace_mirrors (
+    id bigint NOT NULL,
+    namespace_id integer NOT NULL,
+    traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL
+);
+
+CREATE SEQUENCE ci_namespace_mirrors_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE ci_namespace_mirrors_id_seq OWNED BY ci_namespace_mirrors.id;
+
 CREATE TABLE ci_namespace_monthly_usages (
     id bigint NOT NULL,
     namespace_id bigint NOT NULL,
@@ -12015,6 +12030,21 @@ CREATE SEQUENCE ci_platform_metrics_id_seq
 
 ALTER SEQUENCE ci_platform_metrics_id_seq OWNED BY ci_platform_metrics.id;
 
+CREATE TABLE ci_project_mirrors (
+    id bigint NOT NULL,
+    project_id integer NOT NULL,
+    namespace_id integer NOT NULL
+);
+
+CREATE SEQUENCE ci_project_mirrors_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE ci_project_mirrors_id_seq OWNED BY ci_project_mirrors.id;
+
 CREATE TABLE ci_project_monthly_usages (
     id bigint NOT NULL,
     project_id bigint NOT NULL,
@@ -21264,6 +21294,8 @@ ALTER TABLE ONLY ci_job_variables ALTER COLUMN id SET DEFAULT nextval('ci_job_va
 
 ALTER TABLE ONLY ci_minutes_additional_packs ALTER COLUMN id SET DEFAULT nextval('ci_minutes_additional_packs_id_seq'::regclass);
 
+ALTER TABLE ONLY ci_namespace_mirrors ALTER COLUMN id SET DEFAULT nextval('ci_namespace_mirrors_id_seq'::regclass);
+
 ALTER TABLE ONLY ci_namespace_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_namespace_monthly_usages_id_seq'::regclass);
 
 ALTER TABLE ONLY ci_pending_builds ALTER COLUMN id SET DEFAULT nextval('ci_pending_builds_id_seq'::regclass);
@@ -21286,6 +21318,8 @@ ALTER TABLE ONLY ci_pipelines_config ALTER COLUMN pipeline_id SET DEFAULT nextva
 
 ALTER TABLE ONLY ci_platform_metrics ALTER COLUMN id SET DEFAULT nextval('ci_platform_metrics_id_seq'::regclass);
 
+ALTER TABLE ONLY ci_project_mirrors ALTER COLUMN id SET DEFAULT nextval('ci_project_mirrors_id_seq'::regclass);
+
 ALTER TABLE ONLY ci_project_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_project_monthly_usages_id_seq'::regclass);
 
 ALTER TABLE ONLY ci_refs ALTER COLUMN id SET DEFAULT nextval('ci_refs_id_seq'::regclass);
@@ -22727,6 +22761,9 @@ ALTER TABLE ONLY ci_job_variables
 ALTER TABLE ONLY ci_minutes_additional_packs
     ADD CONSTRAINT ci_minutes_additional_packs_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY ci_namespace_mirrors
+    ADD CONSTRAINT ci_namespace_mirrors_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY ci_namespace_monthly_usages
     ADD CONSTRAINT ci_namespace_monthly_usages_pkey PRIMARY KEY (id);
 
@@ -22760,6 +22797,9 @@ ALTER TABLE ONLY ci_pipelines
 ALTER TABLE ONLY ci_platform_metrics
     ADD CONSTRAINT ci_platform_metrics_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY ci_project_mirrors
+    ADD CONSTRAINT ci_project_mirrors_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY ci_project_monthly_usages
     ADD CONSTRAINT ci_project_monthly_usages_pkey PRIMARY KEY (id);
 
@@ -25337,6 +25377,8 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables
 
 CREATE INDEX index_ci_minutes_additional_packs_on_namespace_id_purchase_xid ON ci_minutes_additional_packs USING btree (namespace_id, purchase_xid);
 
+CREATE UNIQUE INDEX index_ci_namespace_mirrors_on_namespace_id ON ci_namespace_mirrors USING btree (namespace_id);
+
 CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date);
 
 CREATE INDEX index_ci_pending_builds_id_on_protected_partial ON ci_pending_builds USING btree (id) WHERE (protected = true);
@@ -25425,6 +25467,10 @@ CREATE INDEX index_ci_pipelines_on_user_id_and_created_at_and_source ON ci_pipel
 
 CREATE INDEX index_ci_pipelines_on_user_id_and_id_and_cancelable_status ON ci_pipelines USING btree (user_id, id) WHERE ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('waiting_for_resource'::character varying)::text, ('preparing'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text, ('scheduled'::character varying)::text]));
 
+CREATE INDEX index_ci_project_mirrors_on_namespace_id ON ci_project_mirrors USING btree (namespace_id);
+
+CREATE UNIQUE INDEX index_ci_project_mirrors_on_project_id ON ci_project_mirrors USING btree (project_id);
+
 CREATE UNIQUE INDEX index_ci_project_monthly_usages_on_project_id_and_date ON ci_project_monthly_usages USING btree (project_id, date);
 
 CREATE UNIQUE INDEX index_ci_refs_on_project_id_and_ref_path ON ci_refs USING btree (project_id, ref_path);
@@ -26001,6 +26047,8 @@ CREATE INDEX index_geo_repository_updated_events_on_source ON geo_repository_upd
 
 CREATE INDEX index_geo_reset_checksum_events_on_project_id ON geo_reset_checksum_events USING btree (project_id);
 
+CREATE INDEX index_gin_ci_namespace_mirrors_on_traversal_ids ON ci_namespace_mirrors USING gin (traversal_ids);
+
 CREATE INDEX index_gin_ci_pending_builds_on_namespace_traversal_ids ON ci_pending_builds USING gin (namespace_traversal_ids);
 
 CREATE INDEX index_gitlab_subscription_histories_on_gitlab_subscription_id ON gitlab_subscription_histories USING btree (gitlab_subscription_id);

lib/gitlab/database/gitlab_loose_foreign_keys.yml

@@ -18,3 +18,14 @@ clusters_applications_runners:
   - table: ci_runners
     column: runner_id
     on_delete: async_nullify
+ci_namespace_mirrors:
+  - table: namespaces
+    column: namespace_id
+    on_delete: async_delete
+ci_project_mirrors:
+  - table: projects
+    column: project_id
+    on_delete: async_delete
+  - table: namespaces
+    column: namespace_id
+    on_delete: async_delete

lib/gitlab/database/gitlab_schemas.yml

@@ -86,6 +86,7 @@ ci_job_token_project_scope_links: :gitlab_ci
 ci_job_variables: :gitlab_ci
 ci_minutes_additional_packs: :gitlab_ci
 ci_namespace_monthly_usages: :gitlab_ci
+ci_namespace_mirrors: :gitlab_ci
 ci_pending_builds: :gitlab_ci
 ci_pipeline_artifacts: :gitlab_ci
 ci_pipeline_chat_data: :gitlab_ci
@@ -97,6 +98,7 @@ ci_pipelines: :gitlab_ci
 ci_pipeline_variables: :gitlab_ci
 ci_platform_metrics: :gitlab_ci
 ci_project_monthly_usages: :gitlab_ci
+ci_project_mirrors: :gitlab_ci
 ci_refs: :gitlab_ci
 ci_resource_groups: :gitlab_ci
 ci_resources: :gitlab_ci

spec/models/namespace_spec.rb

@@ -2056,4 +2056,8 @@ RSpec.describe Namespace do
       it { is_expected.to be(true) }
     end
   end
+
+  it_behaves_like 'it has loose foreign keys' do
+    let(:factory_name) { :group }
+  end
 end

spec/models/project_spec.rb

@@ -7453,6 +7453,12 @@ RSpec.describe Project, factory_default: :keep do
     end
   end
 
+  it_behaves_like 'it has loose foreign keys' do
+    let(:factory_name) { :project }
+  end
+
+  private
+
   def finish_job(export_job)
     export_job.start
     export_job.finish

spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb

@@ -4,6 +4,8 @@ RSpec.shared_examples 'it has loose foreign keys' do
   let(:factory_name) { nil }
   let(:table_name) { described_class.table_name }
   let(:connection) { described_class.connection }
+  let(:fully_qualified_table_name) { "#{connection.current_schema}.#{table_name}" }
+  let(:deleted_records) { LooseForeignKeys::DeletedRecord.where(fully_qualified_table_name: fully_qualified_table_name) }
 
   it 'has at least one loose foreign key definition' do
     definitions = Gitlab::Database::LooseForeignKeys.definitions_by_table[table_name]
@@ -29,7 +31,7 @@ RSpec.shared_examples 'it has loose foreign keys' do
     # using delete to avoid cross-database modification errors when associations with dependent option are present
     model.delete
 
-    deleted_record = LooseForeignKeys::DeletedRecord.find_by(fully_qualified_table_name: "#{connection.current_schema}.#{table_name}", primary_key_value: model.id)
+    deleted_record = deleted_records.find_by(primary_key_value: model.id)
 
     expect(deleted_record).not_to be_nil
   end
@@ -37,11 +39,11 @@ RSpec.shared_examples 'it has loose foreign keys' do
   it 'cleans up record deletions' do
     model = create(factory_name) # rubocop: disable Rails/SaveBang
 
-    expect { model.delete }.to change { LooseForeignKeys::DeletedRecord.count }.by(1)
+    expect { model.delete }.to change { deleted_records.count }.by(1)
 
     LooseForeignKeys::ProcessDeletedRecordsService.new(connection: connection).execute
 
-    expect(LooseForeignKeys::DeletedRecord.status_pending.count).to be(0)
-    expect(LooseForeignKeys::DeletedRecord.status_processed.count).to be(1)
+    expect(deleted_records.status_pending.count).to be(0)
+    expect(deleted_records.status_processed.count).to be(1)
   end
 end