Commit b570f53d authored by Mark Lapierre's avatar Mark Lapierre

Merge branch 'qa-team-tasks-92-test-failure-retries' into 'master'

Retry failing tests

Closes gitlab-org/quality/team-tasks#92

See merge request gitlab-org/gitlab-ce!25391
parents 8a16c07d d54cb37d
......@@ -8,3 +8,4 @@ gem 'rspec', '~> 3.7'
gem 'selenium-webdriver', '~> 3.12'
gem 'airborne', '~> 0.2.13'
gem 'nokogiri', '~> 1.10.1'
gem 'rspec-retry', '~> 0.6.1'
......@@ -76,6 +76,8 @@ GEM
rspec-mocks (3.7.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.7.0)
rspec-retry (0.6.1)
rspec-core (> 3.3)
rspec-support (3.7.0)
rubyzip (1.2.2)
selenium-webdriver (3.141.0)
......@@ -101,6 +103,7 @@ DEPENDENCIES
pry-byebug (~> 3.5.1)
rake (~> 12.3.0)
rspec (~> 3.7)
rspec-retry (~> 0.6.1)
selenium-webdriver (~> 3.12)
BUNDLED WITH
......
require_relative '../qa'
require 'rspec/retry'
%w[helpers shared_examples].each do |d|
Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f }
......@@ -31,6 +32,17 @@ RSpec.configure do |config|
config.profile_examples = 10
config.order = :random
Kernel.srand config.seed
# show retry status in spec process
config.verbose_retry = true
# show exception that triggers a retry if verbose_retry is set to true
config.display_try_failure_messages = true
config.around do |example|
retry_times = example.metadata.keys.include?(:quarantine) ? 1 : 3
example.run_with_retry retry: retry_times
end
end
# Skip tests in quarantine unless we explicitly focus on them.
......
......@@ -28,6 +28,22 @@ describe 'rspec config tests' do
end
end
let(:group_2) do
RSpec.describe do
before(:all) do
@expectations = [1, 2, 3]
end
example 'not in quarantine' do
expect(@expectations.shift).to be(3)
end
example 'in quarantine', :quarantine do
expect(@expectations.shift).to be(3)
end
end
end
context 'with no tags focussed' do
before do
group.run
......@@ -301,4 +317,39 @@ describe 'rspec config tests' do
end
end
end
context 'rspec retry' do
context 'in an untagged context' do
before do
group_2.run
end
it 'should run example :retry times' do
examples = group_2.descendant_filtered_examples
ex = examples.find { |e| e.description == 'not in quarantine' }
expect(ex.execution_result.status).to eq(:passed)
end
end
context 'with :quarantine focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = :quarantine
end
group_2.run
end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
end
end
it 'should run example once only' do
examples = group_2.descendant_filtered_examples
ex = examples.find { |e| e.description == 'in quarantine' }
expect(ex.execution_result.status).to eq(:failed)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment