Commit 9baace02 authored by Adam Hegyi's avatar Adam Hegyi

Merge branch 'ab/hash-partitioning' into 'master'

Add support for hash partitioning

See merge request gitlab-org/gitlab!35268
parents c3b26d1f 1a34ebd9
---
title: Create schema for static partitions
merge_request: 35268
author:
type: other
# frozen_string_literal: true
class CreateStaticPartitionsSchema < ActiveRecord::Migration[6.0]
include Gitlab::Database::SchemaHelpers
DOWNTIME = false
def up
execute 'CREATE SCHEMA gitlab_partitions_static'
create_comment(:schema, :gitlab_partitions_static, <<~EOS.strip)
Schema to hold static partitions, e.g. for hash partitioning
EOS
end
def down
execute 'DROP SCHEMA gitlab_partitions_static'
end
end
......@@ -4,6 +4,10 @@ CREATE SCHEMA gitlab_partitions_dynamic;
COMMENT ON SCHEMA gitlab_partitions_dynamic IS 'Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.';
CREATE SCHEMA gitlab_partitions_static;
COMMENT ON SCHEMA gitlab_partitions_static IS 'Schema to hold static partitions, e.g. for hash partitioning';
CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
CREATE TABLE public.abuse_reports (
......@@ -14164,6 +14168,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200623000148
20200623000320
20200623121135
20200623170000
20200624075411
\.
......@@ -22,12 +22,15 @@ module Gitlab
MIN_SCHEMA_VERSION = 20190506135400
MIN_SCHEMA_GITLAB_VERSION = '11.11.0'
# Schema we store dynamically managed partitions in
# Schema we store dynamically managed partitions in (e.g. for time partitioning)
DYNAMIC_PARTITIONS_SCHEMA = :gitlab_partitions_dynamic
# Schema we store static partitions in (e.g. for hash partitioning)
STATIC_PARTITIONS_SCHEMA = :gitlab_partitions_static
# This is an extensive list of postgres schemas owned by GitLab
# It does not include the default public schema
EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA].freeze
EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze
define_histogram :gitlab_database_transaction_seconds do
docstring "Time spent in database transactions, in seconds"
......
......@@ -70,6 +70,23 @@ module Gitlab
drop_table(part_table_name)
end
def create_hash_partitions(table_name, number_of_partitions)
transaction do
(0..number_of_partitions - 1).each do |partition|
decimals = Math.log10(number_of_partitions).ceil
suffix = "%0#{decimals}d" % partition
partition_name = "#{table_name}_#{suffix}"
schema = Gitlab::Database::STATIC_PARTITIONS_SCHEMA
execute(<<~SQL)
CREATE TABLE #{schema}.#{partition_name}
PARTITION OF #{table_name}
FOR VALUES WITH (MODULUS #{number_of_partitions}, REMAINDER #{partition});
SQL
end
end
end
private
def assert_table_is_allowed(table_name)
......
......@@ -10,6 +10,7 @@ module RuboCop
ALLOWED_MIGRATION_METHODS = %i[
create_table
create_hash_partitions
drop_table
add_foreign_key
remove_foreign_key
......
......@@ -320,6 +320,38 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end
end
describe '#create_hash_partitions' do
before do
connection.execute(<<~SQL)
CREATE TABLE #{partitioned_table}
(id serial not null, some_id integer not null, PRIMARY KEY (id, some_id))
PARTITION BY HASH (some_id);
SQL
end
it 'creates partitions for the full hash space (8 partitions)' do
partitions = 8
migration.create_hash_partitions(partitioned_table, partitions)
(0..partitions - 1).each do |partition|
partition_name = "#{partitioned_table}_#{"%01d" % partition}"
expect_hash_partition_of(partition_name, partitioned_table, partitions, partition)
end
end
it 'creates partitions for the full hash space (16 partitions)' do
partitions = 16
migration.create_hash_partitions(partitioned_table, partitions)
(0..partitions - 1).each do |partition|
partition_name = "#{partitioned_table}_#{"%02d" % partition}"
expect_hash_partition_of(partition_name, partitioned_table, partitions, partition)
end
end
end
def filter_columns_by_name(columns, names)
columns.reject { |c| names.include?(c.name) }
end
......
......@@ -31,6 +31,14 @@ module PartitioningHelpers
expect_total_partitions(table_name, partitions.size, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)
end
def expect_hash_partition_of(partition_name, table_name, modulus, remainder)
definition = find_partition_definition(partition_name, schema: Gitlab::Database::STATIC_PARTITIONS_SCHEMA)
expect(definition).not_to be_nil
expect(definition['base_table']).to eq(table_name.to_s)
expect(definition['condition']).to eq("FOR VALUES WITH (modulus #{modulus}, remainder #{remainder})")
end
private
def find_partitioned_columns(table)
......@@ -55,7 +63,7 @@ module PartitioningHelpers
SQL
end
def find_partition_definition(partition, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)
def find_partition_definition(partition, schema: )
connection.select_one(<<~SQL)
select
parent_class.relname as base_table,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment