Merge branch 'ci_db' into 'master'

Setup new CI DB (decomposition)

See merge request gitlab-org/gitlab!64289

app/models/ci/application_record.rb

+# frozen_string_literal: true
+
+module Ci
+  # TODO: https://gitlab.com/groups/gitlab-org/-/epics/6168
+  #
+  # Do not use this yet outside of `ci_instance_variables`.
+  # This class is part of a migration to move all CI classes to a new separate database.
+  # Initially we are only going to be moving the `Ci::InstanceVariable` model and it will be duplicated in the main and CI tables
+  # Do not extend this class in any other models.
+  class ApplicationRecord < ::ApplicationRecord
+    self.abstract_class = true
+
+    if Gitlab::Database.has_config?(:ci)
+      connects_to database: { writing: :ci, reading: :ci }
+    end
+  end
+end

app/models/ci/instance_variable.rb

 # frozen_string_literal: true
 
 module Ci
-  class InstanceVariable < ApplicationRecord
+  class InstanceVariable < ::Ci::ApplicationRecord
     extend Gitlab::Ci::Model
     extend Gitlab::ProcessMemoryCache::Helper
     include Ci::NewHasVariable

config/initializers/0_post_deployment_migrations.rb → config/initializers/0_migration_paths_additional.rb

 # frozen_string_literal: true
 
+# Because we use Gitlab::Database, which in turn uses prepend_mod_with,
+# we need this intializer to be after config/initializers/0_inject_enterprise_edition_module.rb.
+
 # Post deployment migrations are included by default. This file must be loaded
 # before other initializers as Rails may otherwise memoize a list of migrations
 # excluding the post deployment migrations.

config/initializers/check_decomposition_database_config.rb

+# frozen_string_literal: true
+
+ci_db_config = Gitlab::Application.config.database_configuration[Rails.env]["ci"]
+
+if ci_db_config.present?
+  raise "migrations_paths setting for ci database must be `db/ci_migrate`" unless ci_db_config["migrations_paths"] == 'db/ci_migrate'
+end

db/ci_migrate/20210617101848_create_ci_instance_variables_on_ci.rb

+# frozen_string_literal: true
+
+class CreateCiInstanceVariablesOnCi < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:ci_instance_variables)
+      create_table :ci_instance_variables do |t|
+        t.integer :variable_type, null: false, limit: 2, default: 1
+        t.boolean :masked, default: false, allow_null: false
+        t.boolean :protected, default: false, allow_null: false
+        t.text :key, null: false
+        t.text :encrypted_value
+        t.text :encrypted_value_iv
+
+        t.index [:key], name: 'index_ci_instance_variables_on_key', unique: true, using: :btree
+      end
+    end
+
+    add_text_limit(:ci_instance_variables, :key, 255)
+    # Use constraint_name generated from db/migrate/20200625193358_increase_size_on_instance_level_variable_values.rb
+    add_text_limit(:ci_instance_variables, :encrypted_value, 13_579, constraint_name: 'check_956afd70f1')
+    add_text_limit(:ci_instance_variables, :encrypted_value_iv, 255)
+  end
+
+  def down
+    drop_table :ci_instance_variables
+  end
+end

db/ci_structure.sql

+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
+SET check_function_bodies = false;
+SET xmloption = content;
+SET client_min_messages = warning;
+SET row_security = off;
+
+SET default_tablespace = '';
+
+SET default_table_access_method = heap;
+
+--
+-- Name: ar_internal_metadata; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.ar_internal_metadata (
+    key character varying NOT NULL,
+    value character varying,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL
+);
+
+
+--
+-- Name: ci_instance_variables; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.ci_instance_variables (
+    id bigint NOT NULL,
+    variable_type smallint DEFAULT 1 NOT NULL,
+    masked boolean DEFAULT false,
+    protected boolean DEFAULT false,
+    key text NOT NULL,
+    encrypted_value text,
+    encrypted_value_iv text,
+    CONSTRAINT check_07a45a5bcb CHECK ((char_length(encrypted_value_iv) <= 255)),
+    CONSTRAINT check_5aede12208 CHECK ((char_length(key) <= 255)),
+    CONSTRAINT check_956afd70f1 CHECK ((char_length(encrypted_value) <= 13579))
+);
+
+
+--
+-- Name: ci_instance_variables_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.ci_instance_variables_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: ci_instance_variables_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.ci_instance_variables_id_seq OWNED BY public.ci_instance_variables.id;
+
+
+--
+-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.schema_migrations (
+    version character varying NOT NULL
+);
+
+
+--
+-- Name: ci_instance_variables id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.ci_instance_variables ALTER COLUMN id SET DEFAULT nextval('public.ci_instance_variables_id_seq'::regclass);
+
+
+--
+-- Name: ar_internal_metadata ar_internal_metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.ar_internal_metadata
+    ADD CONSTRAINT ar_internal_metadata_pkey PRIMARY KEY (key);
+
+
+--
+-- Name: ci_instance_variables ci_instance_variables_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.ci_instance_variables
+    ADD CONSTRAINT ci_instance_variables_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.schema_migrations
+    ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version);
+
+
+--
+-- Name: index_ci_instance_variables_on_key; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX index_ci_instance_variables_on_key ON public.ci_instance_variables USING btree (key);
+
+
+--
+-- PostgreSQL database dump complete
+--
+
+SET search_path TO "$user", public;
+
+INSERT INTO "schema_migrations" (version) VALUES
+('20210617101848');
+
+

doc/development/database/multiple_databases.md

+---
+stage: Enablement
+group: Sharding
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
+---
+
+# Multiple Databases
+
+In order to scale GitLab, the GitLab application database
+will be [decomposed into multiple
+databases](https://gitlab.com/groups/gitlab-org/-/epics/6168).
+
+## CI Database
+
+Support for configuring the GitLab Rails application to use a distinct
+database for CI tables was added in [GitLab
+14.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64289). This
+feature is still under development, and is not ready for production use.
+
+By default, GitLab is configured to use only one main database. To
+opt-in to use a main database, and CI database, modify the
+`config/database.yml` file to have a `primary` and a `ci` database
+configurations. For example, given a `config/database.yml` like below:
+
+```yaml
+development:
+  adapter: postgresql
+  encoding: unicode
+  database: gitlabhq_development
+  host: /path/to/gdk/postgresql
+  pool: 10
+  prepared_statements: false
+  variables:
+    statement_timeout: 120s
+
+test: &test
+  adapter: postgresql
+  encoding: unicode
+  database: gitlabhq_test
+  host: /path/to/gdk/postgresql
+  pool: 10
+  prepared_statements: false
+  variables:
+    statement_timeout: 120s
+```
+
+Edit the `config/database.yml` to look like this:
+
+```yaml
+development:
+  primary:
+    adapter: postgresql
+    encoding: unicode
+    database: gitlabhq_development
+    host: /path/to/gdk/postgresql
+    pool: 10
+    prepared_statements: false
+    variables:
+      statement_timeout: 120s
+  ci:
+    adapter: postgresql
+    encoding: unicode
+    database: gitlabhq_development_ci
+    migrations_paths: db/ci_migrate
+    host: /path/to/gdk/postgresql
+    pool: 10
+    prepared_statements: false
+    variables:
+      statement_timeout: 120s
+
+test: &test
+  primary:
+    adapter: postgresql
+    encoding: unicode
+    database: gitlabhq_test
+    host: /path/to/gdk/postgresql
+    pool: 10
+    prepared_statements: false
+    variables:
+      statement_timeout: 120s
+  ci:
+    adapter: postgresql
+    encoding: unicode
+    database: gitlabhq_test_ci
+    migrations_paths: db/ci_migrate
+    host: /path/to/gdk/postgresql
+    pool: 10
+    prepared_statements: false
+    variables:
+      statement_timeout: 120s
+```
+
+Note that we use `primary` in the `config/database.yml` to refer to the main
+database. This is to match the default name Rails has.
+
+### Migrations
+
+Any migrations that affect `Ci::ApplicationRecord` models
+and their tables must be placed in two directories for now:
+
+- `db/migrate`
+- `db/ci_migrate`
+
+We aim to keep the schema for both tables the same across both databases.

lib/gitlab/database.rb

@@ -2,6 +2,8 @@
 
 module Gitlab
   module Database
+    CI_DATABASE_NAME = 'ci'
+
     # This constant is used when renaming tables concurrently.
     # If you plan to rename a table using the `rename_table_safely` method, add your table here one milestone before the rename.
     # Example:
@@ -68,6 +70,14 @@ module Gitlab
       end
     end
 
+    def self.has_config?(database_name)
+      Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s)
+    end
+
+    def self.ci_database?(name)
+      name == CI_DATABASE_NAME
+    end
+
     def self.username
       config['username'] || ENV['USER']
     end

lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb

@@ -7,6 +7,8 @@ module Gitlab
         extend ActiveSupport::Concern
 
         def dump_schema_information # :nodoc:
+          return super unless ActiveRecord::Base.configurations.primary?(pool.db_config.name)
+
           versions = schema_migration.all_versions
           Gitlab::Database::SchemaVersionFiles.touch_all(versions) if versions.any?
 

lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb

@@ -6,9 +6,14 @@ module Gitlab
       module LoadSchemaVersionsMixin
         extend ActiveSupport::Concern
 
-        def structure_load(*args)
-          super(*args)
+        def structure_load(...)
+          result = super(...)
+
+          if ActiveRecord::Base.configurations.primary?(connection.pool.db_config.name)
             Gitlab::Database::SchemaVersionFiles.load_all
+          else
+            result
+          end
         end
       end
     end

spec/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin_spec.rb

@@ -4,20 +4,38 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do
   let(:schema_migration) { double('schema_migration', all_versions: versions) }
+  let(:db_name) { 'primary' }
+  let(:versions) { %w(5 2 1000 200 4 93 2) }
+
+  let(:instance_class) do
+    klass = Class.new do
+      def dump_schema_information
+        original_dump_schema_information
+      end
+
+      def original_dump_schema_information
+      end
+    end
+
+    klass.prepend(described_class)
 
-  let(:instance) do
-    Object.new.extend(described_class)
+    klass
   end
 
+  let(:instance) { instance_class.new }
+
   before do
     allow(instance).to receive(:schema_migration).and_return(schema_migration)
+
+    # pool is from ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+    allow(instance).to receive_message_chain(:pool, :db_config, :name).and_return(db_name)
   end
 
+  context 'when database name is primary' do
     context 'when version files exist' do
-    let(:versions) { %w(5 2 1000 200 4 93 2) }
-
       it 'touches version files' do
         expect(Gitlab::Database::SchemaVersionFiles).to receive(:touch_all).with(versions)
+        expect(instance).not_to receive(:original_dump_schema_information)
 
         instance.dump_schema_information
       end
@@ -28,6 +46,19 @@ RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do
 
       it 'does not touch version files' do
         expect(Gitlab::Database::SchemaVersionFiles).not_to receive(:touch_all)
+        expect(instance).not_to receive(:original_dump_schema_information)
+
+        instance.dump_schema_information
+      end
+    end
+  end
+
+  context 'when database name is ci' do
+    let(:db_name) { 'ci' }
+
+    it 'does not touch version files' do
+      expect(Gitlab::Database::SchemaVersionFiles).not_to receive(:touch_all)
+      expect(instance).to receive(:original_dump_schema_information)
 
       instance.dump_schema_information
     end

spec/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Database::PostgresqlDatabaseTasks::LoadSchemaVersionsMixin do
+  let(:db_name) { 'primary' }
+
+  let(:instance_class) do
+    klass = Class.new do
+      def structure_load
+        original_structure_load
+      end
+
+      def original_structure_load
+      end
+    end
+
+    klass.prepend(described_class)
+
+    klass
+  end
+
+  let(:instance) { instance_class.new }
+
+  before do
+    # connection is available in ActiveRecord::Tasks::PostgreSQLDatabaseTasks
+    allow(instance).to receive_message_chain(:connection, :pool, :db_config, :name).and_return(db_name)
+  end
+
+  context 'when database is primary' do
+    it 'loads version files' do
+      expect(Gitlab::Database::SchemaVersionFiles).to receive(:load_all)
+      expect(instance).to receive(:original_structure_load)
+
+      instance.structure_load
+    end
+  end
+
+  context 'when the database is ci' do
+    let(:db_name) { 'ci' }
+
+    it 'does not load version files' do
+      expect(Gitlab::Database::SchemaVersionFiles).not_to receive(:load_all)
+      expect(instance).to receive(:original_structure_load)
+
+      instance.structure_load
+    end
+  end
+end

spec/lib/gitlab/database_spec.rb

@@ -41,6 +41,45 @@ RSpec.describe Gitlab::Database do
     end
   end
 
+  describe '.has_config?' do
+    context 'two tier database config' do
+      before do
+        allow(Gitlab::Application).to receive_message_chain(:config, :database_configuration, :[]).with(Rails.env)
+          .and_return({ "adapter" => "postgresql", "database" => "gitlabhq_test" })
+      end
+
+      it 'returns false for primary' do
+        expect(described_class.has_config?(:primary)).to eq(false)
+      end
+
+      it 'returns false for ci' do
+        expect(described_class.has_config?(:ci)).to eq(false)
+      end
+    end
+
+    context 'three tier database config' do
+      before do
+        allow(Gitlab::Application).to receive_message_chain(:config, :database_configuration, :[]).with(Rails.env)
+          .and_return({
+            "primary" => { "adapter" => "postgresql", "database" => "gitlabhq_test" },
+            "ci" => { "adapter" => "postgresql", "database" => "gitlabhq_test_ci" }
+          })
+      end
+
+      it 'returns true for primary' do
+        expect(described_class.has_config?(:primary)).to eq(true)
+      end
+
+      it 'returns true for ci' do
+        expect(described_class.has_config?(:ci)).to eq(true)
+      end
+
+      it 'returns false for non-existent' do
+        expect(described_class.has_config?(:nonexistent)).to eq(false)
+      end
+    end
+  end
+
   describe '.adapter_name' do
     it 'returns the name of the adapter' do
       expect(described_class.adapter_name).to be_an_instance_of(String)

spec/migrations/active_record/schema_spec.rb

@@ -7,10 +7,10 @@ require 'spec_helper'
 
 RSpec.describe ActiveRecord::Schema, schema: :latest do
   let(:all_migrations) do
-    migrations_paths = %w[db/migrate db/post_migrate]
-      .map { |path| Rails.root.join(*path, '*') }
+    migrations_directories = %w[db/migrate db/post_migrate].map { |path| Rails.root.join(path).to_s }
+    migrations_paths = migrations_directories.map { |path| File.join(path, '*') }
 
-    migrations = Dir[*migrations_paths]
+    migrations = Dir[*migrations_paths] - migrations_directories
     migrations.map { |migration| File.basename(migration).split('_').first.to_i }.sort
   end