Commit 0acb6ca8 authored by Adam Hegyi's avatar Adam Hegyi

Merge branch 'create-namespace-and-project-mirrors-table' into 'master'

Create ci_namespace_mirrors and ci_project_mirrors tables

See merge request gitlab-org/gitlab!75621
parents 983e4443 9e1df0b6
# frozen_string_literal: true
module Ci
# This model represents a record in a shadow table of the main database's namespaces table.
# It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN.
class NamespaceMirror < ApplicationRecord
# Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517
end
end
# frozen_string_literal: true
module Ci
# This model represents a shadow table of the main database's projects table.
# It allows us to navigate the project and namespace hierarchy on the ci database.
class ProjectMirror < ApplicationRecord
# Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517
end
end
# frozen_string_literal: true
class CreateCiNamespaceMirrors < Gitlab::Database::Migration[1.0]
TABLE_NAME = :ci_namespace_mirrors
INDEX_NAME = "index_gin_#{TABLE_NAME}_on_traversal_ids"
def change
create_table TABLE_NAME do |t|
t.integer :namespace_id, null: false, index: { unique: true }
t.integer :traversal_ids, array: true, default: [], null: false
t.index :traversal_ids, name: INDEX_NAME, using: :gin
end
end
end
# frozen_string_literal: true
class CreateCiProjectMirrors < Gitlab::Database::Migration[1.0]
TABLE_NAME = :ci_project_mirrors
def change
create_table TABLE_NAME do |t|
t.integer :project_id, null: false, index: { unique: true }
t.integer :namespace_id, null: false, index: true
end
end
end
cdae819e8de3b5ad721014376bfd9af97a45e953e2d345daf62784f986a5eb31
\ No newline at end of file
7e51eb4443fd74da9bef4d9c1c3cc40376c311abbc05ca7871f725fada79b48a
\ No newline at end of file
...@@ -11767,6 +11767,21 @@ CREATE SEQUENCE ci_minutes_additional_packs_id_seq ...@@ -11767,6 +11767,21 @@ CREATE SEQUENCE ci_minutes_additional_packs_id_seq
ALTER SEQUENCE ci_minutes_additional_packs_id_seq OWNED BY ci_minutes_additional_packs.id; ALTER SEQUENCE ci_minutes_additional_packs_id_seq OWNED BY ci_minutes_additional_packs.id;
CREATE TABLE ci_namespace_mirrors (
id bigint NOT NULL,
namespace_id integer NOT NULL,
traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL
);
CREATE SEQUENCE ci_namespace_mirrors_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE ci_namespace_mirrors_id_seq OWNED BY ci_namespace_mirrors.id;
CREATE TABLE ci_namespace_monthly_usages ( CREATE TABLE ci_namespace_monthly_usages (
id bigint NOT NULL, id bigint NOT NULL,
namespace_id bigint NOT NULL, namespace_id bigint NOT NULL,
...@@ -12015,6 +12030,21 @@ CREATE SEQUENCE ci_platform_metrics_id_seq ...@@ -12015,6 +12030,21 @@ CREATE SEQUENCE ci_platform_metrics_id_seq
ALTER SEQUENCE ci_platform_metrics_id_seq OWNED BY ci_platform_metrics.id; ALTER SEQUENCE ci_platform_metrics_id_seq OWNED BY ci_platform_metrics.id;
CREATE TABLE ci_project_mirrors (
id bigint NOT NULL,
project_id integer NOT NULL,
namespace_id integer NOT NULL
);
CREATE SEQUENCE ci_project_mirrors_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE ci_project_mirrors_id_seq OWNED BY ci_project_mirrors.id;
CREATE TABLE ci_project_monthly_usages ( CREATE TABLE ci_project_monthly_usages (
id bigint NOT NULL, id bigint NOT NULL,
project_id bigint NOT NULL, project_id bigint NOT NULL,
...@@ -21264,6 +21294,8 @@ ALTER TABLE ONLY ci_job_variables ALTER COLUMN id SET DEFAULT nextval('ci_job_va ...@@ -21264,6 +21294,8 @@ ALTER TABLE ONLY ci_job_variables ALTER COLUMN id SET DEFAULT nextval('ci_job_va
ALTER TABLE ONLY ci_minutes_additional_packs ALTER COLUMN id SET DEFAULT nextval('ci_minutes_additional_packs_id_seq'::regclass); ALTER TABLE ONLY ci_minutes_additional_packs ALTER COLUMN id SET DEFAULT nextval('ci_minutes_additional_packs_id_seq'::regclass);
ALTER TABLE ONLY ci_namespace_mirrors ALTER COLUMN id SET DEFAULT nextval('ci_namespace_mirrors_id_seq'::regclass);
ALTER TABLE ONLY ci_namespace_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_namespace_monthly_usages_id_seq'::regclass); ALTER TABLE ONLY ci_namespace_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_namespace_monthly_usages_id_seq'::regclass);
ALTER TABLE ONLY ci_pending_builds ALTER COLUMN id SET DEFAULT nextval('ci_pending_builds_id_seq'::regclass); ALTER TABLE ONLY ci_pending_builds ALTER COLUMN id SET DEFAULT nextval('ci_pending_builds_id_seq'::regclass);
...@@ -21286,6 +21318,8 @@ ALTER TABLE ONLY ci_pipelines_config ALTER COLUMN pipeline_id SET DEFAULT nextva ...@@ -21286,6 +21318,8 @@ ALTER TABLE ONLY ci_pipelines_config ALTER COLUMN pipeline_id SET DEFAULT nextva
ALTER TABLE ONLY ci_platform_metrics ALTER COLUMN id SET DEFAULT nextval('ci_platform_metrics_id_seq'::regclass); ALTER TABLE ONLY ci_platform_metrics ALTER COLUMN id SET DEFAULT nextval('ci_platform_metrics_id_seq'::regclass);
ALTER TABLE ONLY ci_project_mirrors ALTER COLUMN id SET DEFAULT nextval('ci_project_mirrors_id_seq'::regclass);
ALTER TABLE ONLY ci_project_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_project_monthly_usages_id_seq'::regclass); ALTER TABLE ONLY ci_project_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_project_monthly_usages_id_seq'::regclass);
ALTER TABLE ONLY ci_refs ALTER COLUMN id SET DEFAULT nextval('ci_refs_id_seq'::regclass); ALTER TABLE ONLY ci_refs ALTER COLUMN id SET DEFAULT nextval('ci_refs_id_seq'::regclass);
...@@ -22727,6 +22761,9 @@ ALTER TABLE ONLY ci_job_variables ...@@ -22727,6 +22761,9 @@ ALTER TABLE ONLY ci_job_variables
ALTER TABLE ONLY ci_minutes_additional_packs ALTER TABLE ONLY ci_minutes_additional_packs
ADD CONSTRAINT ci_minutes_additional_packs_pkey PRIMARY KEY (id); ADD CONSTRAINT ci_minutes_additional_packs_pkey PRIMARY KEY (id);
ALTER TABLE ONLY ci_namespace_mirrors
ADD CONSTRAINT ci_namespace_mirrors_pkey PRIMARY KEY (id);
ALTER TABLE ONLY ci_namespace_monthly_usages ALTER TABLE ONLY ci_namespace_monthly_usages
ADD CONSTRAINT ci_namespace_monthly_usages_pkey PRIMARY KEY (id); ADD CONSTRAINT ci_namespace_monthly_usages_pkey PRIMARY KEY (id);
...@@ -22760,6 +22797,9 @@ ALTER TABLE ONLY ci_pipelines ...@@ -22760,6 +22797,9 @@ ALTER TABLE ONLY ci_pipelines
ALTER TABLE ONLY ci_platform_metrics ALTER TABLE ONLY ci_platform_metrics
ADD CONSTRAINT ci_platform_metrics_pkey PRIMARY KEY (id); ADD CONSTRAINT ci_platform_metrics_pkey PRIMARY KEY (id);
ALTER TABLE ONLY ci_project_mirrors
ADD CONSTRAINT ci_project_mirrors_pkey PRIMARY KEY (id);
ALTER TABLE ONLY ci_project_monthly_usages ALTER TABLE ONLY ci_project_monthly_usages
ADD CONSTRAINT ci_project_monthly_usages_pkey PRIMARY KEY (id); ADD CONSTRAINT ci_project_monthly_usages_pkey PRIMARY KEY (id);
...@@ -25337,6 +25377,8 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables ...@@ -25337,6 +25377,8 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables
CREATE INDEX index_ci_minutes_additional_packs_on_namespace_id_purchase_xid ON ci_minutes_additional_packs USING btree (namespace_id, purchase_xid); CREATE INDEX index_ci_minutes_additional_packs_on_namespace_id_purchase_xid ON ci_minutes_additional_packs USING btree (namespace_id, purchase_xid);
CREATE UNIQUE INDEX index_ci_namespace_mirrors_on_namespace_id ON ci_namespace_mirrors USING btree (namespace_id);
CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date); CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date);
CREATE INDEX index_ci_pending_builds_id_on_protected_partial ON ci_pending_builds USING btree (id) WHERE (protected = true); CREATE INDEX index_ci_pending_builds_id_on_protected_partial ON ci_pending_builds USING btree (id) WHERE (protected = true);
...@@ -25425,6 +25467,10 @@ CREATE INDEX index_ci_pipelines_on_user_id_and_created_at_and_source ON ci_pipel ...@@ -25425,6 +25467,10 @@ CREATE INDEX index_ci_pipelines_on_user_id_and_created_at_and_source ON ci_pipel
CREATE INDEX index_ci_pipelines_on_user_id_and_id_and_cancelable_status ON ci_pipelines USING btree (user_id, id) WHERE ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('waiting_for_resource'::character varying)::text, ('preparing'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text, ('scheduled'::character varying)::text])); CREATE INDEX index_ci_pipelines_on_user_id_and_id_and_cancelable_status ON ci_pipelines USING btree (user_id, id) WHERE ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('waiting_for_resource'::character varying)::text, ('preparing'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text, ('scheduled'::character varying)::text]));
CREATE INDEX index_ci_project_mirrors_on_namespace_id ON ci_project_mirrors USING btree (namespace_id);
CREATE UNIQUE INDEX index_ci_project_mirrors_on_project_id ON ci_project_mirrors USING btree (project_id);
CREATE UNIQUE INDEX index_ci_project_monthly_usages_on_project_id_and_date ON ci_project_monthly_usages USING btree (project_id, date); CREATE UNIQUE INDEX index_ci_project_monthly_usages_on_project_id_and_date ON ci_project_monthly_usages USING btree (project_id, date);
CREATE UNIQUE INDEX index_ci_refs_on_project_id_and_ref_path ON ci_refs USING btree (project_id, ref_path); CREATE UNIQUE INDEX index_ci_refs_on_project_id_and_ref_path ON ci_refs USING btree (project_id, ref_path);
...@@ -26001,6 +26047,8 @@ CREATE INDEX index_geo_repository_updated_events_on_source ON geo_repository_upd ...@@ -26001,6 +26047,8 @@ CREATE INDEX index_geo_repository_updated_events_on_source ON geo_repository_upd
CREATE INDEX index_geo_reset_checksum_events_on_project_id ON geo_reset_checksum_events USING btree (project_id); CREATE INDEX index_geo_reset_checksum_events_on_project_id ON geo_reset_checksum_events USING btree (project_id);
CREATE INDEX index_gin_ci_namespace_mirrors_on_traversal_ids ON ci_namespace_mirrors USING gin (traversal_ids);
CREATE INDEX index_gin_ci_pending_builds_on_namespace_traversal_ids ON ci_pending_builds USING gin (namespace_traversal_ids); CREATE INDEX index_gin_ci_pending_builds_on_namespace_traversal_ids ON ci_pending_builds USING gin (namespace_traversal_ids);
CREATE INDEX index_gitlab_subscription_histories_on_gitlab_subscription_id ON gitlab_subscription_histories USING btree (gitlab_subscription_id); CREATE INDEX index_gitlab_subscription_histories_on_gitlab_subscription_id ON gitlab_subscription_histories USING btree (gitlab_subscription_id);
...@@ -18,3 +18,14 @@ clusters_applications_runners: ...@@ -18,3 +18,14 @@ clusters_applications_runners:
- table: ci_runners - table: ci_runners
column: runner_id column: runner_id
on_delete: async_nullify on_delete: async_nullify
ci_namespace_mirrors:
- table: namespaces
column: namespace_id
on_delete: async_delete
ci_project_mirrors:
- table: projects
column: project_id
on_delete: async_delete
- table: namespaces
column: namespace_id
on_delete: async_delete
...@@ -86,6 +86,7 @@ ci_job_token_project_scope_links: :gitlab_ci ...@@ -86,6 +86,7 @@ ci_job_token_project_scope_links: :gitlab_ci
ci_job_variables: :gitlab_ci ci_job_variables: :gitlab_ci
ci_minutes_additional_packs: :gitlab_ci ci_minutes_additional_packs: :gitlab_ci
ci_namespace_monthly_usages: :gitlab_ci ci_namespace_monthly_usages: :gitlab_ci
ci_namespace_mirrors: :gitlab_ci
ci_pending_builds: :gitlab_ci ci_pending_builds: :gitlab_ci
ci_pipeline_artifacts: :gitlab_ci ci_pipeline_artifacts: :gitlab_ci
ci_pipeline_chat_data: :gitlab_ci ci_pipeline_chat_data: :gitlab_ci
...@@ -97,6 +98,7 @@ ci_pipelines: :gitlab_ci ...@@ -97,6 +98,7 @@ ci_pipelines: :gitlab_ci
ci_pipeline_variables: :gitlab_ci ci_pipeline_variables: :gitlab_ci
ci_platform_metrics: :gitlab_ci ci_platform_metrics: :gitlab_ci
ci_project_monthly_usages: :gitlab_ci ci_project_monthly_usages: :gitlab_ci
ci_project_mirrors: :gitlab_ci
ci_refs: :gitlab_ci ci_refs: :gitlab_ci
ci_resource_groups: :gitlab_ci ci_resource_groups: :gitlab_ci
ci_resources: :gitlab_ci ci_resources: :gitlab_ci
......
...@@ -2056,4 +2056,8 @@ RSpec.describe Namespace do ...@@ -2056,4 +2056,8 @@ RSpec.describe Namespace do
it { is_expected.to be(true) } it { is_expected.to be(true) }
end end
end end
it_behaves_like 'it has loose foreign keys' do
let(:factory_name) { :group }
end
end end
...@@ -7453,6 +7453,12 @@ RSpec.describe Project, factory_default: :keep do ...@@ -7453,6 +7453,12 @@ RSpec.describe Project, factory_default: :keep do
end end
end end
it_behaves_like 'it has loose foreign keys' do
let(:factory_name) { :project }
end
private
def finish_job(export_job) def finish_job(export_job)
export_job.start export_job.start
export_job.finish export_job.finish
......
...@@ -4,6 +4,8 @@ RSpec.shared_examples 'it has loose foreign keys' do ...@@ -4,6 +4,8 @@ RSpec.shared_examples 'it has loose foreign keys' do
let(:factory_name) { nil } let(:factory_name) { nil }
let(:table_name) { described_class.table_name } let(:table_name) { described_class.table_name }
let(:connection) { described_class.connection } let(:connection) { described_class.connection }
let(:fully_qualified_table_name) { "#{connection.current_schema}.#{table_name}" }
let(:deleted_records) { LooseForeignKeys::DeletedRecord.where(fully_qualified_table_name: fully_qualified_table_name) }
it 'has at least one loose foreign key definition' do it 'has at least one loose foreign key definition' do
definitions = Gitlab::Database::LooseForeignKeys.definitions_by_table[table_name] definitions = Gitlab::Database::LooseForeignKeys.definitions_by_table[table_name]
...@@ -29,7 +31,7 @@ RSpec.shared_examples 'it has loose foreign keys' do ...@@ -29,7 +31,7 @@ RSpec.shared_examples 'it has loose foreign keys' do
# using delete to avoid cross-database modification errors when associations with dependent option are present # using delete to avoid cross-database modification errors when associations with dependent option are present
model.delete model.delete
deleted_record = LooseForeignKeys::DeletedRecord.find_by(fully_qualified_table_name: "#{connection.current_schema}.#{table_name}", primary_key_value: model.id) deleted_record = deleted_records.find_by(primary_key_value: model.id)
expect(deleted_record).not_to be_nil expect(deleted_record).not_to be_nil
end end
...@@ -37,11 +39,11 @@ RSpec.shared_examples 'it has loose foreign keys' do ...@@ -37,11 +39,11 @@ RSpec.shared_examples 'it has loose foreign keys' do
it 'cleans up record deletions' do it 'cleans up record deletions' do
model = create(factory_name) # rubocop: disable Rails/SaveBang model = create(factory_name) # rubocop: disable Rails/SaveBang
expect { model.delete }.to change { LooseForeignKeys::DeletedRecord.count }.by(1) expect { model.delete }.to change { deleted_records.count }.by(1)
LooseForeignKeys::ProcessDeletedRecordsService.new(connection: connection).execute LooseForeignKeys::ProcessDeletedRecordsService.new(connection: connection).execute
expect(LooseForeignKeys::DeletedRecord.status_pending.count).to be(0) expect(deleted_records.status_pending.count).to be(0)
expect(LooseForeignKeys::DeletedRecord.status_processed.count).to be(1) expect(deleted_records.status_processed.count).to be(1)
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment