Merge branch 'run-migrations-tests-in-dedicated-jobs' into 'master'

Run migrations tests in dedicated jobs

Closes #37417 and #36715

See merge request gitlab-org/gitlab!20845

.gitlab/ci/rails.gitlab-ci.yml

@@ -92,13 +92,21 @@ setup-test-env:
     - .use-pg10
     - .only-master
 
+rspec migration pg9:
+  extends: .rspec-base-pg9
+  parallel: 4
+
+rspec migration pg9-foss:
+  extends: .rspec-base-pg9-foss
+  parallel: 4
+
 rspec unit pg9:
   extends: .rspec-base-pg9
-  parallel: 24
+  parallel: 20
 
 rspec unit pg9-foss:
   extends: .rspec-base-pg9-foss
-  parallel: 24
+  parallel: 20
 
 rspec integration pg9:
   extends: .rspec-base-pg9
@@ -140,9 +148,13 @@ rspec system pg10:
     - .only-ee
     - .use-pg10-ee
 
+rspec-ee migration pg9:
+  extends: .rspec-ee-base-pg9
+  parallel: 2
+
 rspec-ee unit pg9:
   extends: .rspec-ee-base-pg9
-  parallel: 7
+  parallel: 5
 
 rspec-ee integration pg9:
   extends: .rspec-ee-base-pg9
@@ -152,11 +164,17 @@ rspec-ee system pg9:
   extends: .rspec-ee-base-pg9
   parallel: 5
 
+rspec-ee migration pg10:
+  extends:
+    - .rspec-ee-base-pg10
+    - .only-master
+  parallel: 2
+
 rspec-ee unit pg10:
   extends:
     - .rspec-ee-base-pg10
     - .only-master
-  parallel: 7
+  parallel: 5
 
 rspec-ee integration pg10:
   extends:

lib/quality/test_level.rb

@@ -36,6 +36,10 @@ module Quality
         workers
         elastic_integration
       ],
+      migration: %w[
+        migrations
+        lib/gitlab/background_migration
+      ],
       integration: %w[
         controllers
         mailers
@@ -62,6 +66,10 @@ module Quality
 
     def level_for(file_path)
       case file_path
+      # Detect migration first since some background migration tests are under
+      # spec/lib/gitlab/background_migration and tests under spec/lib are unit by default
+      when regexp(:migration)
+        :migration
       when regexp(:unit)
         :unit
       when regexp(:integration)

spec/lib/quality/test_level_spec.rb

@@ -25,6 +25,13 @@ RSpec.describe Quality::TestLevel do
       end
     end
 
+    context 'when level is migration' do
+      it 'returns a pattern' do
+        expect(subject.pattern(:migration))
+          .to eq("spec/{migrations,lib/gitlab/background_migration}{,/**/}*_spec.rb")
+      end
+    end
+
     context 'when level is integration' do
       it 'returns a pattern' do
         expect(subject.pattern(:integration))
@@ -79,6 +86,13 @@ RSpec.describe Quality::TestLevel do
       end
     end
 
+    context 'when level is migration' do
+      it 'returns a regexp' do
+        expect(subject.regexp(:migration))
+          .to eq(%r{spec/(migrations|lib/gitlab/background_migration)})
+      end
+    end
+
     context 'when level is integration' do
       it 'returns a regexp' do
         expect(subject.regexp(:integration))
@@ -116,6 +130,18 @@ RSpec.describe Quality::TestLevel do
       expect(subject.level_for('spec/models/abuse_report_spec.rb')).to eq(:unit)
     end
 
+    it 'returns the correct level for a migration test' do
+      expect(subject.level_for('spec/migrations/add_default_and_free_plans_spec.rb')).to eq(:migration)
+    end
+
+    it 'returns the correct level for a background_migration test' do
+      expect(subject.level_for('spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb')).to eq(:migration)
+    end
+
+    it 'returns the correct level for a geo migration test' do
+      expect(described_class.new('ee/').level_for('ee/spec/migrations/geo/migrate_ci_job_artifacts_to_separate_registry_spec.rb')).to eq(:migration)
+    end
+
     it 'returns the correct level for an integration test' do
       expect(subject.level_for('spec/mailers/abuse_report_mailer_spec.rb')).to eq(:integration)
     end

spec/support/database_cleaner.rb

@@ -15,6 +15,39 @@ RSpec.configure do |config|
     delete_from_all_tables!
   end
 
+  config.append_after(:context, :migration) do
+    delete_from_all_tables!
+
+    # Postgres maximum number of columns in a table is 1600 (https://github.com/postgres/postgres/blob/de41869b64d57160f58852eab20a27f248188135/src/include/access/htup_details.h#L23-L47).
+    # And since:
+    # "The DROP COLUMN form does not physically remove the column, but simply makes
+    # it invisible to SQL operations. Subsequent insert and update operations in the
+    # table will store a null value for the column. Thus, dropping a column is quick
+    # but it will not immediately reduce the on-disk size of your table, as the space
+    # occupied by the dropped column is not reclaimed.
+    # The space will be reclaimed over time as existing rows are updated."
+    # according to https://www.postgresql.org/docs/current/sql-altertable.html.
+    # We drop and recreate the database if any table has more than 1200 columns, just to be safe.
+    max_allowed_columns = 1200
+    tables_with_more_than_allowed_columns =
+      ApplicationRecord.connection.execute("SELECT attrelid::regclass::text AS table, COUNT(*) AS column_count FROM pg_attribute GROUP BY attrelid HAVING COUNT(*) > #{max_allowed_columns}")
+
+    if tables_with_more_than_allowed_columns.any?
+      tables_with_more_than_allowed_columns.each do |result|
+        puts "The #{result['table']} table has #{result['column_count']} columns."
+      end
+      puts "Recreating the database"
+      start = Gitlab::Metrics::System.monotonic_time
+
+      ActiveRecord::Tasks::DatabaseTasks.drop_current
+      ActiveRecord::Tasks::DatabaseTasks.create_current
+      ActiveRecord::Tasks::DatabaseTasks.load_schema_current
+      ActiveRecord::Tasks::DatabaseTasks.migrate
+
+      puts "Database re-creation done in #{Gitlab::Metrics::System.monotonic_time - start}"
+    end
+  end
+
   config.around(:each, :delete) do |example|
     self.class.use_transactional_tests = false