Merge branch 'allow-to-quarantine-tests' into 'master'

Allow to quarantine tests

See merge request gitlab-org/gitlab-ce!24369

.gitlab-ci.yml

@@ -499,6 +499,22 @@ rspec-mysql:
   <<: *rspec-metadata-mysql
   parallel: 50
 
+.rspec-quarantine: &rspec-quarantine
+  script:
+    - export CACHE_CLASSES=true
+    - scripts/gitaly-test-spawn
+    - bin/rspec --color --format documentation --tag quarantine spec/
+
+rspec-pg-quarantine:
+  <<: *rspec-metadata-pg
+  <<: *rspec-quarantine
+  allow_failure: true
+
+rspec-mysql-quarantine:
+  <<: *rspec-metadata-mysql
+  <<: *rspec-quarantine
+  allow_failure: true
+
 static-analysis:
   <<: *dedicated-no-docs-no-db-pull-cache-job
   dependencies:

doc/development/testing_guide/flaky_tests.md

@@ -5,6 +5,18 @@
 It's a test that sometimes fails, but if you retry it enough times, it passes,
 eventually.
 
+## Quarantined tests
+
+Tests can be put in quarantine by assigning `:quarantine` metadata. This means
+they will be skipped unless run with `--tag quarantine`. This can be used for
+tests that are expected to fail while a fix is in progress (similar to how
+[`skip` or `pending`](https://relishapp.com/rspec/rspec-core/v/3-8/docs/pending-and-skipped-examples)
+ can be used).
+
+```
+bin/rspec --tag quarantine
+```
+
 ## Automatic retries and flaky tests detection
 
 On our CI, we use [rspec-retry] to automatically retry a failing example a few

spec/features/boards/boards_spec.rb

@@ -97,7 +97,7 @@ describe 'Issue Boards', :js do
       expect(find('.board:nth-child(4)')).to have_selector('.board-card')
     end
 
-    it 'shows description tooltip on list title' do
+    it 'shows description tooltip on list title', :quarantine do
       page.within('.board:nth-child(2)') do
         expect(find('.board-title span.has-tooltip')[:title]).to eq('Test')
       end
@@ -411,7 +411,7 @@ describe 'Issue Boards', :js do
         wait_for_empty_boards((2..4))
       end
 
-      it 'filters by label with space after reload' do
+      it 'filters by label with space after reload', :quarantine do
         set_filter("label", "\"#{accepting.title}")
         click_filter_link(accepting.title)
         submit_filter
@@ -477,7 +477,7 @@ describe 'Issue Boards', :js do
         end
       end
 
-      it 'filters by multiple labels' do
+      it 'filters by multiple labels', :quarantine do
         set_filter("label", testing.title)
         click_filter_link(testing.title)
 

spec/spec_helper.rb

@@ -22,7 +22,7 @@ if rspec_profiling_is_configured && (!ENV.key?('CI') || branch_can_be_profiled)
   require 'rspec_profiling/rspec'
 end
 
-if ENV['CI'] && !ENV['NO_KNAPSACK']
+if ENV['CI'] && ENV['KNAPSACK_GENERATE_REPORT'] && !ENV['NO_KNAPSACK']
   require 'knapsack'
   Knapsack::Adapters::RSpecAdapter.bind
 end
@@ -132,6 +132,11 @@ RSpec.configure do |config|
     Gitlab::ReleaseBlogPost.instance.instance_variable_set(:@url, 'https://about.gitlab.com')
   end
 
+  config.before(:example, :quarantine) do
+    # Skip tests in quarantine unless we explicitly focus on them.
+    skip('In quarantine') unless config.inclusion_filter[:quarantine]
+  end
+
   config.before(:example, :request_store) do
     RequestStore.begin!
   end