Merge branch 'philipcunningham-drop-dast-fks-for-ci-sharding-336195' into 'master'

Add loose foreign key preparation for DAST [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!66889

db/post_migrate/20211027043206_track_ci_pipeline_deletions.rb

+# frozen_string_literal: true
+
+class TrackCiPipelineDeletions < Gitlab::Database::Migration[1.0]
+  include Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers
+
+  enable_lock_retries!
+
+  def up
+    track_record_deletions(:ci_pipelines)
+  end
+
+  def down
+    untrack_record_deletions(:ci_pipelines)
+  end
+end

db/post_migrate/20211027043229_track_ci_build_deletions.rb

+# frozen_string_literal: true
+
+class TrackCiBuildDeletions < Gitlab::Database::Migration[1.0]
+  include Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers
+
+  enable_lock_retries!
+
+  def up
+    track_record_deletions(:ci_builds)
+  end
+
+  def down
+    untrack_record_deletions(:ci_builds)
+  end
+end

db/schema_migrations/20211027043206

+543feeedace6596d63207738829dcd62249a9f048a08928fbe4131ec69058322
\ No newline at end of file

db/schema_migrations/20211027043229

+2bceb12bdb90052cc8c1aedbd52c11cb8125471e1b59de3d75ef476fc64851c9
\ No newline at end of file

db/structure.sql

@@ -28723,6 +28723,10 @@ ALTER INDEX product_analytics_events_experimental_pkey ATTACH PARTITION gitlab_p
 
 CREATE TRIGGER chat_names_loose_fk_trigger AFTER DELETE ON chat_names REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records();
 
+CREATE TRIGGER ci_builds_loose_fk_trigger AFTER DELETE ON ci_builds REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records();
+
+CREATE TRIGGER ci_pipelines_loose_fk_trigger AFTER DELETE ON ci_pipelines REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records();
+
 CREATE TRIGGER ci_runners_loose_fk_trigger AFTER DELETE ON ci_runners REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records();
 
 CREATE TRIGGER trigger_delete_project_namespace_on_project_delete AFTER DELETE ON projects FOR EACH ROW WHEN ((old.project_namespace_id IS NOT NULL)) EXECUTE FUNCTION delete_associated_project_namespace();

lib/gitlab/database/gitlab_loose_foreign_keys.yml

@@ -2,6 +2,17 @@ chat_names:
   - to_table: ci_pipeline_chat_data
     column: chat_name_id
     on_delete: async_delete
+ci_builds:
+  - to_table: dast_site_profiles_builds
+    column: ci_build_id
+    on_delete: async_delete
+  - to_table: dast_scanner_profiles_builds
+    column: ci_build_id
+    on_delete: async_delete
+ci_pipelines:
+  - to_table: dast_profiles_pipelines
+    column: ci_pipeline_id
+    on_delete: async_delete
 ci_runners:
   - to_table: clusters_applications_runners
     column: runner_id

spec/models/ci/build_spec.rb

@@ -5403,4 +5403,8 @@ RSpec.describe Ci::Build do
       expect(subject).to be true
     end
   end
+
+  it_behaves_like 'it has loose foreign keys' do
+    let(:factory_name) { :ci_build }
+  end
 end

spec/models/ci/pipeline_spec.rb

@@ -4611,4 +4611,8 @@ RSpec.describe Ci::Pipeline, :mailer, factory_default: :keep do
       expect(pipeline.authorized_cluster_agents).to contain_exactly(agent) # cached
     end
   end
+
+  it_behaves_like 'it has loose foreign keys' do
+    let(:factory_name) { :ci_pipeline }
+  end
 end

spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb

@@ -25,7 +25,9 @@ RSpec.shared_examples 'it has loose foreign keys' do
 
   it 'records record deletions' do
     model = create(factory_name) # rubocop: disable Rails/SaveBang
-    model.destroy!
+
+    # using delete to avoid cross-database modification errors when associations with dependent option are present
+    model.delete
 
     deleted_record = LooseForeignKeys::DeletedRecord.find_by(fully_qualified_table_name: "#{connection.current_schema}.#{table_name}", primary_key_value: model.id)
 
@@ -35,7 +37,7 @@ RSpec.shared_examples 'it has loose foreign keys' do
   it 'cleans up record deletions' do
     model = create(factory_name) # rubocop: disable Rails/SaveBang
 
-    expect { model.destroy! }.to change { LooseForeignKeys::DeletedRecord.count }.by(1)
+    expect { model.delete }.to change { LooseForeignKeys::DeletedRecord.count }.by(1)
 
     LooseForeignKeys::ProcessDeletedRecordsService.new(connection: connection).execute