Commit e18c12e7 authored by Sean McGivern's avatar Sean McGivern

Merge branch 'validate-that-foreign-keys-are-created-ee' into 'master'

EE: Validate that foreign keys are created

Closes gitlab-ce#50875

See merge request gitlab-org/gitlab-ee!8232
parents 86036385 186f8c53
---
title: Validate foreign keys being created and indexed for column with _id
merge_request: 22808
author:
type: performance
# frozen_string_literal: true
class AddMissingIndexesForForeignKeys < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index(:application_settings, :usage_stats_set_by_user_id)
add_concurrent_index(:ci_pipeline_schedules, :owner_id)
add_concurrent_index(:ci_trigger_requests, :trigger_id)
add_concurrent_index(:ci_triggers, :owner_id)
add_concurrent_index(:clusters_applications_helm, :cluster_id, unique: true)
add_concurrent_index(:clusters_applications_ingress, :cluster_id, unique: true)
add_concurrent_index(:clusters_applications_jupyter, :cluster_id, unique: true)
add_concurrent_index(:clusters_applications_jupyter, :oauth_application_id)
add_concurrent_index(:clusters_applications_knative, :cluster_id, unique: true)
add_concurrent_index(:clusters_applications_prometheus, :cluster_id, unique: true)
add_concurrent_index(:fork_network_members, :forked_from_project_id)
add_concurrent_index(:internal_ids, :namespace_id)
add_concurrent_index(:internal_ids, :project_id)
add_concurrent_index(:issues, :closed_by_id)
add_concurrent_index(:label_priorities, :label_id)
add_concurrent_index(:merge_request_metrics, :merged_by_id)
add_concurrent_index(:merge_request_metrics, :latest_closed_by_id)
add_concurrent_index(:oauth_openid_requests, :access_grant_id)
add_concurrent_index(:project_deploy_tokens, :deploy_token_id)
add_concurrent_index(:protected_tag_create_access_levels, :group_id)
add_concurrent_index(:subscriptions, :project_id)
add_concurrent_index(:user_statuses, :user_id)
add_concurrent_index(:users, :accepted_term_id)
end
def down
# MySQL requires index for FK,
# thus removal of indexes does fail
return if Gitlab::Database.mysql?
remove_concurrent_index(:application_settings, :usage_stats_set_by_user_id)
remove_concurrent_index(:ci_pipeline_schedules, :owner_id)
remove_concurrent_index(:ci_trigger_requests, :trigger_id)
remove_concurrent_index(:ci_triggers, :owner_id)
remove_concurrent_index(:clusters_applications_helm, :cluster_id, unique: true)
remove_concurrent_index(:clusters_applications_ingress, :cluster_id, unique: true)
remove_concurrent_index(:clusters_applications_jupyter, :cluster_id, unique: true)
remove_concurrent_index(:clusters_applications_jupyter, :oauth_application_id)
remove_concurrent_index(:clusters_applications_knative, :cluster_id, unique: true)
remove_concurrent_index(:clusters_applications_prometheus, :cluster_id, unique: true)
remove_concurrent_index(:fork_network_members, :forked_from_project_id)
remove_concurrent_index(:internal_ids, :namespace_id)
remove_concurrent_index(:internal_ids, :project_id)
remove_concurrent_index(:issues, :closed_by_id)
remove_concurrent_index(:label_priorities, :label_id)
remove_concurrent_index(:merge_request_metrics, :merged_by_id)
remove_concurrent_index(:merge_request_metrics, :latest_closed_by_id)
remove_concurrent_index(:oauth_openid_requests, :access_grant_id)
remove_concurrent_index(:project_deploy_tokens, :deploy_token_id)
remove_concurrent_index(:protected_tag_create_access_levels, :group_id)
remove_concurrent_index(:subscriptions, :project_id)
remove_concurrent_index(:user_statuses, :user_id)
remove_concurrent_index(:users, :accepted_term_id)
end
end
......@@ -212,6 +212,9 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "diff_max_patch_bytes", default: 102400, null: false
t.integer "archive_builds_in_seconds"
t.string "commit_email_hostname"
t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id", using: :btree
t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id", using: :btree
t.index ["usage_stats_set_by_user_id"], name: "index_application_settings_on_usage_stats_set_by_user_id", using: :btree
end
create_table "approvals", force: :cascade do |t|
......@@ -279,6 +282,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
create_table "board_assignees", force: :cascade do |t|
t.integer "board_id", null: false
t.integer "assignee_id", null: false
t.index ["assignee_id"], name: "index_board_assignees_on_assignee_id", using: :btree
t.index ["board_id", "assignee_id"], name: "index_board_assignees_on_board_id_and_assignee_id", unique: true, using: :btree
end
......@@ -298,6 +302,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "board_id", null: false
t.integer "label_id", null: false
t.index ["board_id", "label_id"], name: "index_board_labels_on_board_id_and_label_id", unique: true, using: :btree
t.index ["label_id"], name: "index_board_labels_on_label_id", using: :btree
end
create_table "board_project_recent_visits", id: :bigserial, force: :cascade do |t|
......@@ -506,6 +511,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "pipeline_id", null: false
t.integer "chat_name_id", null: false
t.text "response_url", null: false
t.index ["chat_name_id"], name: "index_ci_pipeline_chat_data_on_chat_name_id", using: :btree
t.index ["pipeline_id"], name: "index_ci_pipeline_chat_data_on_pipeline_id", unique: true, using: :btree
end
......@@ -533,6 +539,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.datetime "created_at"
t.datetime "updated_at"
t.index ["next_run_at", "active"], name: "index_ci_pipeline_schedules_on_next_run_at_and_active", using: :btree
t.index ["owner_id"], name: "index_ci_pipeline_schedules_on_owner_id", using: :btree
t.index ["project_id"], name: "index_ci_pipeline_schedules_on_project_id", using: :btree
end
......@@ -658,6 +665,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.datetime "updated_at"
t.integer "commit_id"
t.index ["commit_id"], name: "index_ci_trigger_requests_on_commit_id", using: :btree
t.index ["trigger_id"], name: "index_ci_trigger_requests_on_trigger_id", using: :btree
end
create_table "ci_triggers", force: :cascade do |t|
......@@ -668,6 +676,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "owner_id"
t.string "description"
t.string "ref"
t.index ["owner_id"], name: "index_ci_triggers_on_owner_id", using: :btree
t.index ["project_id"], name: "index_ci_triggers_on_project_id", using: :btree
end
......@@ -757,6 +766,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.text "encrypted_ca_key"
t.text "encrypted_ca_key_iv"
t.text "ca_cert"
t.index ["cluster_id"], name: "index_clusters_applications_helm_on_cluster_id", unique: true, using: :btree
end
create_table "clusters_applications_ingress", force: :cascade do |t|
......@@ -769,6 +779,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.string "cluster_ip"
t.text "status_reason"
t.string "external_ip"
t.index ["cluster_id"], name: "index_clusters_applications_ingress_on_cluster_id", unique: true, using: :btree
end
create_table "clusters_applications_jupyter", force: :cascade do |t|
......@@ -780,6 +791,8 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.text "status_reason"
t.index ["cluster_id"], name: "index_clusters_applications_jupyter_on_cluster_id", unique: true, using: :btree
t.index ["oauth_application_id"], name: "index_clusters_applications_jupyter_on_oauth_application_id", using: :btree
end
create_table "clusters_applications_knative", force: :cascade do |t|
......@@ -790,6 +803,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.string "version", null: false
t.string "hostname"
t.text "status_reason"
t.index ["cluster_id"], name: "index_clusters_applications_knative_on_cluster_id", unique: true, using: :btree
end
create_table "clusters_applications_prometheus", force: :cascade do |t|
......@@ -800,6 +814,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.datetime_with_timezone "last_update_started_at"
t.index ["cluster_id"], name: "index_clusters_applications_prometheus_on_cluster_id", unique: true, using: :btree
end
create_table "clusters_applications_runners", force: :cascade do |t|
......@@ -1051,6 +1066,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "project_id", null: false
t.integer "forked_from_project_id"
t.index ["fork_network_id"], name: "index_fork_network_members_on_fork_network_id", using: :btree
t.index ["forked_from_project_id"], name: "index_fork_network_members_on_forked_from_project_id", using: :btree
t.index ["project_id"], name: "index_fork_network_members_on_project_id", unique: true, using: :btree
end
......@@ -1139,6 +1155,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.datetime "updated_at", null: false
t.index ["geo_node_id", "namespace_id"], name: "index_geo_node_namespace_links_on_geo_node_id_and_namespace_id", unique: true, using: :btree
t.index ["geo_node_id"], name: "index_geo_node_namespace_links_on_geo_node_id", using: :btree
t.index ["namespace_id"], name: "index_geo_node_namespace_links_on_namespace_id", using: :btree
end
create_table "geo_node_statuses", force: :cascade do |t|
......@@ -1362,6 +1379,8 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "usage", null: false
t.integer "last_value", null: false
t.integer "namespace_id"
t.index ["namespace_id"], name: "index_internal_ids_on_namespace_id", using: :btree
t.index ["project_id"], name: "index_internal_ids_on_project_id", using: :btree
t.index ["usage", "namespace_id"], name: "index_internal_ids_on_usage_and_namespace_id", unique: true, where: "(namespace_id IS NOT NULL)", using: :btree
t.index ["usage", "project_id"], name: "index_internal_ids_on_usage_and_project_id", unique: true, where: "(project_id IS NOT NULL)", using: :btree
end
......@@ -1421,6 +1440,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.datetime_with_timezone "closed_at"
t.integer "closed_by_id"
t.index ["author_id"], name: "index_issues_on_author_id", using: :btree
t.index ["closed_by_id"], name: "index_issues_on_closed_by_id", using: :btree
t.index ["confidential"], name: "index_issues_on_confidential", using: :btree
t.index ["description"], name: "index_issues_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
t.index ["milestone_id"], name: "index_issues_on_milestone_id", using: :btree
......@@ -1466,6 +1486,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "priority", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["label_id"], name: "index_label_priorities_on_label_id", using: :btree
t.index ["priority"], name: "index_label_priorities_on_priority", using: :btree
t.index ["project_id", "label_id"], name: "index_label_priorities_on_project_id_and_label_id", unique: true, using: :btree
end
......@@ -1630,7 +1651,9 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "latest_closed_by_id"
t.datetime_with_timezone "latest_closed_at"
t.index ["first_deployed_to_production_at"], name: "index_merge_request_metrics_on_first_deployed_to_production_at", using: :btree
t.index ["latest_closed_by_id"], name: "index_merge_request_metrics_on_latest_closed_by_id", using: :btree
t.index ["merge_request_id"], name: "index_merge_request_metrics", using: :btree
t.index ["merged_by_id"], name: "index_merge_request_metrics_on_merged_by_id", using: :btree
t.index ["pipeline_id"], name: "index_merge_request_metrics_on_pipeline_id", using: :btree
end
......@@ -1763,6 +1786,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "file_template_project_id"
t.string "saml_discovery_token"
t.index ["created_at"], name: "index_namespaces_on_created_at", using: :btree
t.index ["file_template_project_id"], name: "index_namespaces_on_file_template_project_id", using: :btree
t.index ["ldap_sync_last_successful_update_at"], name: "index_namespaces_on_ldap_sync_last_successful_update_at", using: :btree
t.index ["ldap_sync_last_update_at"], name: "index_namespaces_on_ldap_sync_last_update_at", using: :btree
t.index ["name", "parent_id"], name: "index_namespaces_on_name_and_parent_id", unique: true, using: :btree
......@@ -1898,6 +1922,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
create_table "oauth_openid_requests", force: :cascade do |t|
t.integer "access_grant_id", null: false
t.string "nonce", null: false
t.index ["access_grant_id"], name: "index_oauth_openid_requests_on_access_grant_id", using: :btree
end
create_table "operations_feature_flags", id: :bigserial, force: :cascade do |t|
......@@ -2289,6 +2314,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.datetime "updated_at", null: false
t.integer "user_id"
t.integer "group_id"
t.index ["group_id"], name: "index_protected_branch_merge_access_levels_on_group_id", using: :btree
t.index ["protected_branch_id"], name: "index_protected_branch_merge_access", using: :btree
t.index ["user_id"], name: "index_protected_branch_merge_access_levels_on_user_id", using: :btree
end
......@@ -2300,6 +2326,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.datetime "updated_at", null: false
t.integer "user_id"
t.integer "group_id"
t.index ["group_id"], name: "index_protected_branch_push_access_levels_on_group_id", using: :btree
t.index ["protected_branch_id"], name: "index_protected_branch_push_access", using: :btree
t.index ["user_id"], name: "index_protected_branch_push_access_levels_on_user_id", using: :btree
end
......@@ -2350,6 +2377,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "group_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["group_id"], name: "index_protected_tag_create_access_levels_on_group_id", using: :btree
t.index ["protected_tag_id"], name: "index_protected_tag_create_access", using: :btree
t.index ["user_id"], name: "index_protected_tag_create_access_levels_on_user_id", using: :btree
end
......@@ -2581,6 +2609,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "software_license_id", null: false
t.integer "approval_status", default: 0, null: false
t.index ["project_id", "software_license_id"], name: "index_software_license_policies_unique_per_project", unique: true, using: :btree
t.index ["software_license_id"], name: "index_software_license_policies_on_software_license_id", using: :btree
end
create_table "software_licenses", force: :cascade do |t|
......@@ -2610,6 +2639,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.datetime "created_at"
t.datetime "updated_at"
t.integer "project_id"
t.index ["project_id"], name: "index_subscriptions_on_project_id", using: :btree
t.index ["subscribable_id", "subscribable_type", "user_id", "project_id"], name: "index_subscriptions_on_subscribable_and_user_id_and_project_id", unique: true, using: :btree
end
......@@ -2774,6 +2804,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.string "emoji", default: "speech_balloon", null: false
t.string "message", limit: 100
t.string "message_html"
t.index ["user_id"], name: "index_user_statuses_on_user_id", using: :btree
end
create_table "user_synced_attributes_metadata", force: :cascade do |t|
......@@ -2863,6 +2894,7 @@ ActiveRecord::Schema.define(version: 20181107054254) do
t.integer "roadmap_layout", limit: 2
t.boolean "include_private_contributions"
t.string "commit_email"
t.index ["accepted_term_id"], name: "index_users_on_accepted_term_id", using: :btree
t.index ["admin"], name: "index_users_on_admin", using: :btree
t.index ["confirmation_token"], name: "index_users_on_confirmation_token", unique: true, using: :btree
t.index ["created_at"], name: "index_users_on_created_at", using: :btree
......
......@@ -187,12 +187,7 @@ end
When adding a foreign-key constraint to either an existing or new
column remember to also add a index on the column.
This is _required_ if the foreign-key constraint specifies
`ON DELETE CASCADE` or `ON DELETE SET NULL` behavior. On a cascading
delete, the [corresponding record needs to be retrieved using an
index](https://www.cybertec-postgresql.com/en/postgresql-indexes-and-foreign-keys/)
(otherwise, we'd need to scan the whole table) for subsequent update or
deletion.
This is _required_ for all foreign-keys.
Here's an example where we add a new column with a foreign key
constraint. Note it includes `index: true` to create an index for it.
......
# frozen_string_literal: true
class AddMissingIndexesForForeignKeysEE < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index(:application_settings, :file_template_project_id)
add_concurrent_index(:application_settings, :custom_project_templates_group_id)
add_concurrent_index(:board_assignees, :assignee_id)
add_concurrent_index(:board_labels, :label_id)
add_concurrent_index(:ci_pipeline_chat_data, :chat_name_id)
add_concurrent_index(:geo_node_namespace_links, :namespace_id)
add_concurrent_index(:namespaces, :file_template_project_id)
add_concurrent_index(:protected_branch_merge_access_levels, :group_id)
add_concurrent_index(:protected_branch_push_access_levels, :group_id)
add_concurrent_index(:software_license_policies, :software_license_id)
end
def down
# MySQL requires index for FK,
# thus removal of indexes does fail
return if Gitlab::Database.mysql?
remove_concurrent_index(:application_settings, :file_template_project_id)
remove_concurrent_index(:application_settings, :custom_project_templates_group_id)
remove_concurrent_index(:board_assignees, :assignee_id)
remove_concurrent_index(:board_labels, :label_id)
remove_concurrent_index(:ci_pipeline_chat_data, :chat_name_id)
remove_concurrent_index(:geo_node_namespace_links, :namespace_id)
remove_concurrent_index(:namespaces, :file_template_project_id)
remove_concurrent_index(:protected_branch_merge_access_levels, :group_id)
remove_concurrent_index(:protected_branch_push_access_levels, :group_id)
remove_concurrent_index(:software_license_policies, :software_license_id)
end
end
# frozen_string_literal: true
module EE
module DB
module SchemaSupport
extend ActiveSupport::Concern
prepended do
EE_IGNORED_FK_COLUMNS = {
application_settings: %w[slack_app_id snowplow_site_id],
approvals: %w[user_id],
approver_groups: %w[target_id],
approvers: %w[target_id user_id],
boards: %w[milestone_id],
draft_notes: %w[discussion_id],
epics: %w[updated_by_id last_edited_by_id start_date_sourcing_milestone_id due_date_sourcing_milestone_id],
geo_event_log: %w[hashed_storage_attachments_event_id],
geo_job_artifact_deleted_events: %w[job_artifact_id],
geo_lfs_object_deleted_events: %w[lfs_object_id],
geo_node_statuses: %w[last_event_id cursor_last_event_id],
geo_nodes: %w[oauth_application_id],
geo_repository_deleted_events: %w[project_id],
geo_upload_deleted_events: %w[upload_id model_id],
ldap_group_links: %w[group_id],
projects: %w[mirror_user_id],
slack_integrations: %w[team_id user_id],
users: %w[email_opted_in_source_id],
vulnerability_identifiers: %w[external_id],
vulnerability_scanners: %w[external_id],
web_hooks: %w[group_id]
}.with_indifferent_access.freeze
end
def ignored_fk_columns(column)
super + EE_IGNORED_FK_COLUMNS.fetch(column, [])
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('ee', 'spec', 'db', 'schema_support')
describe 'Database schema' do
prepend ::EE::DB::SchemaSupport
let(:connection) { ActiveRecord::Base.connection }
let(:tables) { connection.tables }
# Use if you are certain that this column should not have a foreign key
# EE: edit the ee/spec/db/schema_support.rb
IGNORED_FK_COLUMNS = {
abuse_reports: %w[reporter_id user_id],
application_settings: %w[performance_bar_allowed_group_id],
audit_events: %w[author_id entity_id],
award_emoji: %w[awardable_id user_id],
chat_names: %w[chat_id service_id team_id user_id],
chat_teams: %w[team_id],
ci_builds: %w[erased_by_id runner_id trigger_request_id user_id],
ci_pipelines: %w[user_id],
ci_runner_projects: %w[runner_id],
ci_trigger_requests: %w[commit_id],
cluster_providers_gcp: %w[gcp_project_id operation_id],
deploy_keys_projects: %w[deploy_key_id],
deployments: %w[deployable_id environment_id user_id],
emails: %w[user_id],
events: %w[target_id],
forked_project_links: %w[forked_from_project_id],
identities: %w[user_id],
issues: %w[last_edited_by_id],
keys: %w[user_id],
label_links: %w[target_id],
lfs_objects_projects: %w[lfs_object_id project_id],
members: %w[source_id created_by_id],
merge_requests: %w[last_edited_by_id],
namespaces: %w[owner_id parent_id],
notes: %w[author_id commit_id noteable_id updated_by_id resolved_by_id discussion_id],
notification_settings: %w[source_id],
oauth_access_grants: %w[resource_owner_id application_id],
oauth_access_tokens: %w[resource_owner_id application_id],
oauth_applications: %w[owner_id],
project_group_links: %w[group_id],
project_statistics: %w[namespace_id],
projects: %w[creator_id namespace_id ci_id],
redirect_routes: %w[source_id],
repository_languages: %w[programming_language_id],
routes: %w[source_id],
sent_notifications: %w[project_id noteable_id recipient_id commit_id in_reply_to_discussion_id],
snippets: %w[author_id],
spam_logs: %w[user_id],
subscriptions: %w[user_id subscribable_id],
taggings: %w[tag_id taggable_id tagger_id],
timelogs: %w[user_id],
todos: %w[target_id commit_id],
uploads: %w[model_id],
user_agent_details: %w[subject_id],
users: %w[color_scheme_id created_by_id theme_id],
users_star_projects: %w[user_id],
web_hooks: %w[service_id]
}.with_indifferent_access.freeze
context 'for table' do
ActiveRecord::Base.connection.tables.sort.each do |table|
describe table do
let(:indexes) { connection.indexes(table) }
let(:columns) { connection.columns(table) }
let(:foreign_keys) { connection.foreign_keys(table) }
context 'all foreign keys' do
# for index to be effective, the FK constraint has to be at first place
it 'are indexed' do
first_indexed_column = indexes.map(&:columns).map(&:first)
foreign_keys_columns = foreign_keys.map(&:column)
expect(first_indexed_column.uniq).to include(*foreign_keys_columns)
end
end
context 'columns ending with _id' do
let(:column_names) { columns.map(&:name) }
let(:column_names_with_id) { column_names.select { |column_name| column_name.ends_with?('_id') } }
let(:foreign_keys_columns) { foreign_keys.map(&:column) }
let(:ignored_columns) { ignored_fk_columns(table) }
it 'do have the foreign keys' do
expect(column_names_with_id - ignored_columns).to contain_exactly(*foreign_keys_columns)
end
end
end
end
end
private
def ignored_fk_columns(column)
IGNORED_FK_COLUMNS.fetch(column, [])
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment