Merge branch 'ab/hash-partitioning' into 'master'

Add support for hash partitioning

See merge request gitlab-org/gitlab!35268

changelogs/unreleased/ab-hash-partitioning.yml

+---
+title: Create schema for static partitions
+merge_request: 35268
+author:
+type: other

db/migrate/20200623170000_create_static_partitions_schema.rb

+# frozen_string_literal: true
+
+class CreateStaticPartitionsSchema < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::SchemaHelpers
+
+  DOWNTIME = false
+
+  def up
+    execute 'CREATE SCHEMA gitlab_partitions_static'
+
+    create_comment(:schema, :gitlab_partitions_static, <<~EOS.strip)
+      Schema to hold static partitions, e.g. for hash partitioning
+    EOS
+  end
+
+  def down
+    execute 'DROP SCHEMA gitlab_partitions_static'
+  end
+end

db/structure.sql

@@ -4,6 +4,10 @@ CREATE SCHEMA gitlab_partitions_dynamic;
 
 COMMENT ON SCHEMA gitlab_partitions_dynamic IS 'Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.';
 
+CREATE SCHEMA gitlab_partitions_static;
+
+COMMENT ON SCHEMA gitlab_partitions_static IS 'Schema to hold static partitions, e.g. for hash partitioning';
+
 CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
 
 CREATE TABLE public.abuse_reports (
@@ -14164,6 +14168,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200623000148
 20200623000320
 20200623121135
+20200623170000
 20200624075411
 \.
 

lib/gitlab/database.rb

@@ -22,12 +22,15 @@ module Gitlab
     MIN_SCHEMA_VERSION = 20190506135400
     MIN_SCHEMA_GITLAB_VERSION = '11.11.0'
 
-    # Schema we store dynamically managed partitions in
+    # Schema we store dynamically managed partitions in (e.g. for time partitioning)
     DYNAMIC_PARTITIONS_SCHEMA = :gitlab_partitions_dynamic
 
+    # Schema we store static partitions in (e.g. for hash partitioning)
+    STATIC_PARTITIONS_SCHEMA = :gitlab_partitions_static
+
     # This is an extensive list of postgres schemas owned by GitLab
     # It does not include the default public schema
-    EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA].freeze
+    EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze
 
     define_histogram :gitlab_database_transaction_seconds do
       docstring "Time spent in database transactions, in seconds"

lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb

@@ -70,6 +70,23 @@ module Gitlab
           drop_table(part_table_name)
         end
 
+        def create_hash_partitions(table_name, number_of_partitions)
+          transaction do
+            (0..number_of_partitions - 1).each do |partition|
+              decimals = Math.log10(number_of_partitions).ceil
+              suffix = "%0#{decimals}d" % partition
+              partition_name = "#{table_name}_#{suffix}"
+              schema = Gitlab::Database::STATIC_PARTITIONS_SCHEMA
+
+              execute(<<~SQL)
+                CREATE TABLE #{schema}.#{partition_name}
+                PARTITION OF #{table_name}
+                FOR VALUES WITH (MODULUS #{number_of_partitions}, REMAINDER #{partition});
+              SQL
+            end
+          end
+        end
+
         private
 
         def assert_table_is_allowed(table_name)

rubocop/cop/migration/with_lock_retries_disallowed_method.rb

@@ -10,6 +10,7 @@ module RuboCop
 
         ALLOWED_MIGRATION_METHODS = %i[
           create_table
+          create_hash_partitions
           drop_table
           add_foreign_key
           remove_foreign_key

spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb

@@ -320,6 +320,38 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
     end
   end
 
+  describe '#create_hash_partitions' do
+    before do
+      connection.execute(<<~SQL)
+        CREATE TABLE #{partitioned_table}
+          (id serial not null, some_id integer not null, PRIMARY KEY (id, some_id))
+          PARTITION BY HASH (some_id);
+      SQL
+    end
+
+    it 'creates partitions for the full hash space (8 partitions)' do
+      partitions = 8
+
+      migration.create_hash_partitions(partitioned_table, partitions)
+
+      (0..partitions - 1).each do |partition|
+        partition_name = "#{partitioned_table}_#{"%01d" % partition}"
+        expect_hash_partition_of(partition_name, partitioned_table, partitions, partition)
+      end
+    end
+
+    it 'creates partitions for the full hash space (16 partitions)' do
+      partitions = 16
+
+      migration.create_hash_partitions(partitioned_table, partitions)
+
+      (0..partitions - 1).each do |partition|
+        partition_name = "#{partitioned_table}_#{"%02d" % partition}"
+        expect_hash_partition_of(partition_name, partitioned_table, partitions, partition)
+      end
+    end
+  end
+
   def filter_columns_by_name(columns, names)
     columns.reject { |c| names.include?(c.name) }
   end

spec/support/helpers/partitioning_helpers.rb

@@ -31,6 +31,14 @@ module PartitioningHelpers
     expect_total_partitions(table_name, partitions.size, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)
   end
 
+  def expect_hash_partition_of(partition_name, table_name, modulus, remainder)
+    definition = find_partition_definition(partition_name, schema: Gitlab::Database::STATIC_PARTITIONS_SCHEMA)
+
+    expect(definition).not_to be_nil
+    expect(definition['base_table']).to eq(table_name.to_s)
+    expect(definition['condition']).to eq("FOR VALUES WITH (modulus #{modulus}, remainder #{remainder})")
+  end
+
   private
 
   def find_partitioned_columns(table)
@@ -55,7 +63,7 @@ module PartitioningHelpers
     SQL
   end
 
-  def find_partition_definition(partition, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)
+  def find_partition_definition(partition, schema: )
     connection.select_one(<<~SQL)
       select
         parent_class.relname as base_table,