Add web_hook_logs partitioning migration

- Create a new partitioned table web_hook_logs_part_0c5294f417
  with a nearly identical schema as web_hook_logs
- Create all the monthly partitions that cover
  existing web_hook_logs data
- Install trigger that keeps tables in sync
- Add WebHookLogPartitioned model and register it for
  dynamic partition creation
- Schedule background migration jobs to copy data from web_hook_logs
  to the partitioned table
- Add explicit primary key definition to WebHookLog
  This will facilitate the smooth transition when we swap the tables

app/models/hooks/web_hook_log.rb

@@ -6,6 +6,8 @@ class WebHookLog < ApplicationRecord
   include DeleteWithLimit
   include CreatedAtFilterable
 
+  self.primary_key = :id
+
   belongs_to :web_hook
 
   serialize :request_headers, Hash # rubocop:disable Cop/ActiveRecordSerialize

app/models/hooks/web_hook_log_partitioned.rb

+# frozen_string_literal: true
+
+# This model is not yet intended to be used.
+# It is in a transitioning phase while we are partitioning
+# the web_hook_logs table on the database-side.
+# Please refer to https://gitlab.com/groups/gitlab-org/-/epics/5558
+# for details.
+# rubocop:disable Gitlab/NamespacedClass: This is a temporary class with no relevant namespace
+#  WebHook, WebHookLog and all hooks are defined outside of a namespace
+class WebHookLogPartitioned < ApplicationRecord
+  include PartitionedTable
+
+  self.table_name = 'web_hook_logs_part_0c5294f417'
+  self.primary_key = :id
+
+  partitioned_by :created_at, strategy: :monthly
+end

changelogs/unreleased/323674-initiate-web-hook-logs-partitioning.yml

+---
+title: Add web_hook_logs partitioning migration
+merge_request: 55938
+author:
+type: other

config/initializers/postgres_partitioning.rb

@@ -4,6 +4,7 @@
 # (even with eager loading disabled).
 
 Gitlab::Database::Partitioning::PartitionCreator.register(AuditEvent)
+Gitlab::Database::Partitioning::PartitionCreator.register(WebHookLogPartitioned)
 
 begin
   Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP']

db/migrate/20210306121300_partition_web_hook_logs.rb

+# frozen_string_literal: true
+
+class PartitionWebHookLogs < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+  include Gitlab::Database::PartitioningMigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    partition_table_by_date :web_hook_logs, :created_at
+  end
+
+  def down
+    drop_partitioned_table_for :web_hook_logs
+  end
+end

db/post_migrate/20210306121310_backfill_partitioned_web_hook_logs.rb

+# frozen_string_literal: true
+
+class BackfillPartitionedWebHookLogs < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::PartitioningMigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    enqueue_partitioning_data_migration :web_hook_logs
+  end
+
+  def down
+    cleanup_partitioning_data_migration :web_hook_logs
+  end
+end

db/schema_migrations/20210306121300

+44d53ac15c5e54c2f1c825286155dec643b82573184026caaf08288512168aef
\ No newline at end of file

db/schema_migrations/20210306121310

+90072e3dee4517061ff9e08decda7fecb9cc9b38a56345c09685e3cce48a8b66
\ No newline at end of file

db/structure.sql

@@ -41,6 +41,62 @@ RETURN NULL;
 END
 $$;
 
+CREATE FUNCTION table_sync_function_29bc99d6db() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+IF (TG_OP = 'DELETE') THEN
+  DELETE FROM web_hook_logs_part_0c5294f417 where id = OLD.id;
+ELSIF (TG_OP = 'UPDATE') THEN
+  UPDATE web_hook_logs_part_0c5294f417
+  SET web_hook_id = NEW.web_hook_id,
+    trigger = NEW.trigger,
+    url = NEW.url,
+    request_headers = NEW.request_headers,
+    request_data = NEW.request_data,
+    response_headers = NEW.response_headers,
+    response_body = NEW.response_body,
+    response_status = NEW.response_status,
+    execution_duration = NEW.execution_duration,
+    internal_error_message = NEW.internal_error_message,
+    updated_at = NEW.updated_at,
+    created_at = NEW.created_at
+  WHERE web_hook_logs_part_0c5294f417.id = NEW.id;
+ELSIF (TG_OP = 'INSERT') THEN
+  INSERT INTO web_hook_logs_part_0c5294f417 (id,
+    web_hook_id,
+    trigger,
+    url,
+    request_headers,
+    request_data,
+    response_headers,
+    response_body,
+    response_status,
+    execution_duration,
+    internal_error_message,
+    updated_at,
+    created_at)
+  VALUES (NEW.id,
+    NEW.web_hook_id,
+    NEW.trigger,
+    NEW.url,
+    NEW.request_headers,
+    NEW.request_data,
+    NEW.response_headers,
+    NEW.response_body,
+    NEW.response_status,
+    NEW.execution_duration,
+    NEW.internal_error_message,
+    NEW.updated_at,
+    NEW.created_at);
+END IF;
+RETURN NULL;
+
+END
+$$;
+
+COMMENT ON FUNCTION table_sync_function_29bc99d6db() IS 'Partitioning migration: table sync for web_hook_logs table';
+
 CREATE FUNCTION table_sync_function_2be879775d() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -114,6 +170,23 @@ CREATE TABLE audit_events (
 )
 PARTITION BY RANGE (created_at);
 
+CREATE TABLE web_hook_logs_part_0c5294f417 (
+    id bigint NOT NULL,
+    web_hook_id integer NOT NULL,
+    trigger character varying,
+    url character varying,
+    request_headers text,
+    request_data text,
+    response_headers text,
+    response_body text,
+    response_status character varying,
+    execution_duration double precision,
+    internal_error_message character varying,
+    updated_at timestamp without time zone NOT NULL,
+    created_at timestamp without time zone NOT NULL
+)
+PARTITION BY RANGE (created_at);
+
 CREATE TABLE product_analytics_events_experimental (
     id bigint NOT NULL,
     project_id integer NOT NULL,
@@ -21182,6 +21255,9 @@ ALTER TABLE ONLY vulnerability_statistics
 ALTER TABLE ONLY vulnerability_user_mentions
     ADD CONSTRAINT vulnerability_user_mentions_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY web_hook_logs_part_0c5294f417
+    ADD CONSTRAINT web_hook_logs_part_0c5294f417_pkey PRIMARY KEY (id, created_at);
+
 ALTER TABLE ONLY web_hook_logs
     ADD CONSTRAINT web_hook_logs_pkey PRIMARY KEY (id);
 
@@ -24243,6 +24319,8 @@ ALTER INDEX product_analytics_events_experimental_pkey ATTACH PARTITION gitlab_p
 
 ALTER INDEX product_analytics_events_experimental_pkey ATTACH PARTITION gitlab_partitions_static.product_analytics_events_experimental_63_pkey;
 
+CREATE TRIGGER table_sync_trigger_b99eb6998c AFTER INSERT OR DELETE OR UPDATE ON web_hook_logs FOR EACH ROW EXECUTE PROCEDURE table_sync_function_29bc99d6db();
+
 CREATE TRIGGER table_sync_trigger_ee39a25f9d AFTER INSERT OR DELETE OR UPDATE ON audit_events FOR EACH ROW EXECUTE PROCEDURE table_sync_function_2be879775d();
 
 CREATE TRIGGER trigger_has_external_issue_tracker_on_delete AFTER DELETE ON services FOR EACH ROW WHEN ((((old.category)::text = 'issue_tracker'::text) AND (old.active = true) AND (old.project_id IS NOT NULL))) EXECUTE PROCEDURE set_has_external_issue_tracker();

lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb

@@ -9,7 +9,7 @@ module Gitlab
         include ::Gitlab::Database::MigrationHelpers
         include ::Gitlab::Database::Migrations::BackgroundMigrationHelpers
 
-        ALLOWED_TABLES = %w[audit_events].freeze
+        ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze
         ERROR_SCOPE = 'table partitioning'
 
         MIGRATION_CLASS_NAME = "::#{module_parent_name}::BackfillPartitionedTable"

spec/db/schema_spec.rb

@@ -87,7 +87,8 @@ RSpec.describe 'Database schema' do
     users_star_projects: %w[user_id],
     vulnerability_identifiers: %w[external_id],
     vulnerability_scanners: %w[external_id],
-    web_hooks: %w[group_id]
+    web_hooks: %w[group_id],
+    web_hook_logs_part_0c5294f417: %w[web_hook_id]
   }.with_indifferent_access.freeze
 
   context 'for table' do