Commit 9c9f25c1 authored by Yannis Roussos's avatar Yannis Roussos

Merge branch '323674-initiate-web-hook-logs-partitioning' into 'master'

Initiate the web_hook_logs partitioning

See merge request gitlab-org/gitlab!55938
parents 918509b3 11cf38bb
......@@ -6,6 +6,8 @@ class WebHookLog < ApplicationRecord
include DeleteWithLimit
include CreatedAtFilterable
self.primary_key = :id
belongs_to :web_hook
serialize :request_headers, Hash # rubocop:disable Cop/ActiveRecordSerialize
......
# frozen_string_literal: true
# This model is not yet intended to be used.
# It is in a transitioning phase while we are partitioning
# the web_hook_logs table on the database-side.
# Please refer to https://gitlab.com/groups/gitlab-org/-/epics/5558
# for details.
# rubocop:disable Gitlab/NamespacedClass: This is a temporary class with no relevant namespace
# WebHook, WebHookLog and all hooks are defined outside of a namespace
class WebHookLogPartitioned < ApplicationRecord
include PartitionedTable
self.table_name = 'web_hook_logs_part_0c5294f417'
self.primary_key = :id
partitioned_by :created_at, strategy: :monthly
end
---
title: Add web_hook_logs partitioning migration
merge_request: 55938
author:
type: other
......@@ -4,6 +4,7 @@
# (even with eager loading disabled).
Gitlab::Database::Partitioning::PartitionCreator.register(AuditEvent)
Gitlab::Database::Partitioning::PartitionCreator.register(WebHookLogPartitioned)
begin
Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP']
......
# frozen_string_literal: true
class PartitionWebHookLogs < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
include Gitlab::Database::PartitioningMigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
disable_ddl_transaction!
def up
partition_table_by_date :web_hook_logs, :created_at
end
def down
drop_partitioned_table_for :web_hook_logs
end
end
# frozen_string_literal: true
class BackfillPartitionedWebHookLogs < ActiveRecord::Migration[6.0]
include Gitlab::Database::PartitioningMigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
enqueue_partitioning_data_migration :web_hook_logs
end
def down
cleanup_partitioning_data_migration :web_hook_logs
end
end
44d53ac15c5e54c2f1c825286155dec643b82573184026caaf08288512168aef
\ No newline at end of file
90072e3dee4517061ff9e08decda7fecb9cc9b38a56345c09685e3cce48a8b66
\ No newline at end of file
......@@ -41,6 +41,62 @@ RETURN NULL;
END
$$;
CREATE FUNCTION table_sync_function_29bc99d6db() RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
IF (TG_OP = 'DELETE') THEN
DELETE FROM web_hook_logs_part_0c5294f417 where id = OLD.id;
ELSIF (TG_OP = 'UPDATE') THEN
UPDATE web_hook_logs_part_0c5294f417
SET web_hook_id = NEW.web_hook_id,
trigger = NEW.trigger,
url = NEW.url,
request_headers = NEW.request_headers,
request_data = NEW.request_data,
response_headers = NEW.response_headers,
response_body = NEW.response_body,
response_status = NEW.response_status,
execution_duration = NEW.execution_duration,
internal_error_message = NEW.internal_error_message,
updated_at = NEW.updated_at,
created_at = NEW.created_at
WHERE web_hook_logs_part_0c5294f417.id = NEW.id;
ELSIF (TG_OP = 'INSERT') THEN
INSERT INTO web_hook_logs_part_0c5294f417 (id,
web_hook_id,
trigger,
url,
request_headers,
request_data,
response_headers,
response_body,
response_status,
execution_duration,
internal_error_message,
updated_at,
created_at)
VALUES (NEW.id,
NEW.web_hook_id,
NEW.trigger,
NEW.url,
NEW.request_headers,
NEW.request_data,
NEW.response_headers,
NEW.response_body,
NEW.response_status,
NEW.execution_duration,
NEW.internal_error_message,
NEW.updated_at,
NEW.created_at);
END IF;
RETURN NULL;
END
$$;
COMMENT ON FUNCTION table_sync_function_29bc99d6db() IS 'Partitioning migration: table sync for web_hook_logs table';
CREATE FUNCTION table_sync_function_2be879775d() RETURNS trigger
LANGUAGE plpgsql
AS $$
......@@ -114,6 +170,23 @@ CREATE TABLE audit_events (
)
PARTITION BY RANGE (created_at);
CREATE TABLE web_hook_logs_part_0c5294f417 (
id bigint NOT NULL,
web_hook_id integer NOT NULL,
trigger character varying,
url character varying,
request_headers text,
request_data text,
response_headers text,
response_body text,
response_status character varying,
execution_duration double precision,
internal_error_message character varying,
updated_at timestamp without time zone NOT NULL,
created_at timestamp without time zone NOT NULL
)
PARTITION BY RANGE (created_at);
CREATE TABLE product_analytics_events_experimental (
id bigint NOT NULL,
project_id integer NOT NULL,
......@@ -21254,6 +21327,9 @@ ALTER TABLE ONLY vulnerability_statistics
ALTER TABLE ONLY vulnerability_user_mentions
ADD CONSTRAINT vulnerability_user_mentions_pkey PRIMARY KEY (id);
ALTER TABLE ONLY web_hook_logs_part_0c5294f417
ADD CONSTRAINT web_hook_logs_part_0c5294f417_pkey PRIMARY KEY (id, created_at);
ALTER TABLE ONLY web_hook_logs
ADD CONSTRAINT web_hook_logs_pkey PRIMARY KEY (id);
......@@ -24319,6 +24395,8 @@ ALTER INDEX product_analytics_events_experimental_pkey ATTACH PARTITION gitlab_p
ALTER INDEX product_analytics_events_experimental_pkey ATTACH PARTITION gitlab_partitions_static.product_analytics_events_experimental_63_pkey;
CREATE TRIGGER table_sync_trigger_b99eb6998c AFTER INSERT OR DELETE OR UPDATE ON web_hook_logs FOR EACH ROW EXECUTE PROCEDURE table_sync_function_29bc99d6db();
CREATE TRIGGER table_sync_trigger_ee39a25f9d AFTER INSERT OR DELETE OR UPDATE ON audit_events FOR EACH ROW EXECUTE PROCEDURE table_sync_function_2be879775d();
CREATE TRIGGER trigger_has_external_issue_tracker_on_delete AFTER DELETE ON services FOR EACH ROW WHEN ((((old.category)::text = 'issue_tracker'::text) AND (old.active = true) AND (old.project_id IS NOT NULL))) EXECUTE PROCEDURE set_has_external_issue_tracker();
......@@ -9,7 +9,7 @@ module Gitlab
include ::Gitlab::Database::MigrationHelpers
include ::Gitlab::Database::Migrations::BackgroundMigrationHelpers
ALLOWED_TABLES = %w[audit_events].freeze
ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze
ERROR_SCOPE = 'table partitioning'
MIGRATION_CLASS_NAME = "::#{module_parent_name}::BackfillPartitionedTable"
......
......@@ -87,7 +87,8 @@ RSpec.describe 'Database schema' do
users_star_projects: %w[user_id],
vulnerability_identifiers: %w[external_id],
vulnerability_scanners: %w[external_id],
web_hooks: %w[group_id]
web_hooks: %w[group_id],
web_hook_logs_part_0c5294f417: %w[web_hook_id]
}.with_indifferent_access.freeze
context 'for table' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment