Merge branch 'improve-schemas-to-include-views-and-models' into 'master'

Improve `gitlab_schemas.yml` to include `gitlab_shared` and views

See merge request gitlab-org/gitlab!73643

lib/gitlab/database/gitlab_schema.rb

@@ -2,6 +2,13 @@
 
 # This module gathers information about table to schema mapping
 # to understand table affinity
+#
+# Each table / view needs to have assigned gitlab_schema. Names supported today:
+#
+# - gitlab_shared - defines a set of tables that are found on all databases (data accessed is dependent on connection)
+# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given database
+#
+
 module Gitlab
   module Database
     module GitlabSchema

lib/gitlab/database/gitlab_schemas.yml

@@ -32,6 +32,7 @@ approval_project_rules_users: :gitlab_main
 approvals: :gitlab_main
 approver_groups: :gitlab_main
 approvers: :gitlab_main
+ar_internal_metadata: :gitlab_shared
 atlassian_identities: :gitlab_main
 audit_events_external_audit_event_destinations: :gitlab_main
 audit_events: :gitlab_main
@@ -171,7 +172,7 @@ design_management_designs: :gitlab_main
 design_management_designs_versions: :gitlab_main
 design_management_versions: :gitlab_main
 design_user_mentions: :gitlab_main
-detached_partitions: :gitlab_main
+detached_partitions: :gitlab_shared
 diff_note_positions: :gitlab_main
 dora_daily_metrics: :gitlab_main
 draft_notes: :gitlab_main
@@ -373,8 +374,13 @@ personal_access_tokens: :gitlab_main
 plan_limits: :gitlab_main
 plans: :gitlab_main
 pool_repositories: :gitlab_main
-postgres_async_indexes: :gitlab_main
-postgres_reindex_actions: :gitlab_main
+postgres_async_indexes: :gitlab_shared
+postgres_foreign_keys: :gitlab_shared
+postgres_index_bloat_estimates: :gitlab_shared
+postgres_indexes: :gitlab_shared
+postgres_partitioned_tables: :gitlab_shared
+postgres_partitions: :gitlab_shared
+postgres_reindex_actions: :gitlab_shared
 product_analytics_events_experimental: :gitlab_main
 programming_languages: :gitlab_main
 project_access_tokens: :gitlab_main
@@ -438,6 +444,7 @@ reviews: :gitlab_main
 routes: :gitlab_main
 saml_group_links: :gitlab_main
 saml_providers: :gitlab_main
+schema_migrations: :gitlab_shared
 scim_identities: :gitlab_main
 scim_oauth_access_tokens: :gitlab_main
 security_findings: :gitlab_main

spec/lib/gitlab/database/gitlab_schema_spec.rb

@@ -2,28 +2,37 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Database::GitlabSchema do
-  it 'matches all the tables in the database', :aggregate_failures do
-    # These tables do not need a gitlab_schema
-    excluded_tables = %w(ar_internal_metadata schema_migrations)
+  describe '.tables_to_schema' do
+    subject { described_class.tables_to_schema }
 
-    all_tables_in_database = ApplicationRecord.connection.tables
+    it 'all tables have assigned a known gitlab_schema' do
+      is_expected.to all(
+        match([be_a(String), be_in([:gitlab_shared, :gitlab_main, :gitlab_ci])])
+      )
+    end
 
-    all_tables_with_gitlab_schema = described_class.tables_to_schema.keys
+    # This being run across different databases indirectly also tests
+    # a general consistency of structure across databases
+    Gitlab::Database.database_base_models.each do |db_config_name, db_class|
+      let(:db_data_sources) { db_class.connection.data_sources }
 
-    missing = []
-    all_tables_in_database.each do |table_in_database|
-      next if table_in_database.in?(excluded_tables)
+      context "for #{db_config_name} using #{db_class}" do
+        it 'new data sources are added' do
+          missing_tables = db_data_sources.to_set - subject.keys
 
-      missing << table_in_database unless all_tables_with_gitlab_schema.include?(table_in_database)
-    end
+          expect(missing_tables).to be_empty, \
+            "Missing table(s) #{missing_tables.to_a} not found in #{described_class}.tables_to_schema. " \
+            "Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml."
+        end
 
-    extras = []
-    all_tables_with_gitlab_schema.each do |table_with_gitlab_schema|
-      extras << table_with_gitlab_schema unless all_tables_in_database.include?(table_with_gitlab_schema)
-    end
+        it 'non-existing data sources are removed' do
+          extra_tables = subject.keys.to_set - db_data_sources
 
-    expect(missing).to be_empty, "Missing table(s) #{missing} not found in #{described_class}.tables_to_schema. Any new tables must be added to spec/support/database/gitlab_schemas.yml ."
-
-    expect(extras).to be_empty, "Extra table(s) #{extras} found in #{described_class}.tables_to_schema. Any removed or renamed tables must be removed from spec/support/database/gitlab_schemas.yml ."
+          expect(extra_tables).to be_empty, \
+            "Extra table(s) #{extra_tables.to_a} found in #{described_class}.tables_to_schema. " \
+            "Any removed or renamed tables must be removed from lib/gitlab/database/gitlab_schemas.yml."
+        end
+      end
+    end
   end
 end