Add context selector to exclude tests from jobs

This module allows tests to be skipped (excluded) based on the job in
which the test runs. It's complimentary to the `:only` tag.

qa/qa.rb

@@ -577,7 +577,9 @@ module QA
     autoload :LoopRunner, 'qa/specs/loop_runner'
 
     module Helpers
+      autoload :ContextSelector, 'qa/specs/helpers/context_selector'
       autoload :Quarantine, 'qa/specs/helpers/quarantine'
+      autoload :RSpec, 'qa/specs/helpers/rspec'
     end
   end
 

qa/qa/runtime/env.rb

@@ -2,6 +2,7 @@
 
 require 'gitlab/qa'
 require 'uri'
+require 'active_support/core_ext/object/blank'
 
 module QA
   module Runtime
@@ -24,48 +25,6 @@ module QA
         SUPPORTED_FEATURES
       end
 
-      def context_matches?(*options)
-        return false unless Runtime::Scenario.attributes[:gitlab_address]
-
-        opts = {}
-        opts[:domain] = '.+'
-        opts[:tld] = '.com'
-
-        uri = URI(Runtime::Scenario.gitlab_address)
-
-        options.each do |option|
-          opts[:domain] = 'gitlab' if option == :production
-
-          if option.is_a?(Hash) && !option[:pipeline].nil? && !ci_project_name.nil?
-            return pipeline_matches?(option[:pipeline])
-
-          elsif option.is_a?(Hash) && !option[:subdomain].nil?
-            opts.merge!(option)
-
-            opts[:subdomain] = case option[:subdomain]
-                               when Array
-                                 "(#{option[:subdomain].join("|")})."
-                               when Regexp
-                                 option[:subdomain]
-                               else
-                                 "(#{option[:subdomain]})."
-                               end
-          end
-        end
-
-        uri.host.match?(/^#{opts[:subdomain]}#{opts[:domain]}#{opts[:tld]}$/)
-      end
-
-      alias_method :dot_com?, :context_matches?
-
-      def pipeline_matches?(pipeline_to_run_in)
-        Array(pipeline_to_run_in).any? { |pipeline| pipeline.to_s.casecmp?(pipeline_from_project_name) }
-      end
-
-      def pipeline_from_project_name
-        ci_project_name.to_s.start_with?('gitlab-qa') ? Runtime::Env.default_branch : ci_project_name
-      end
-
       def additional_repository_storage
         ENV['QA_ADDITIONAL_REPOSITORY_STORAGE']
       end
@@ -82,6 +41,10 @@ module QA
         ENV['CI_JOB_URL']
       end
 
+      def ci_job_name
+        ENV['CI_JOB_NAME']
+      end
+
       def ci_project_name
         ENV['CI_PROJECT_NAME']
       end

qa/qa/service/praefect_manager.rb

@@ -181,7 +181,7 @@ module QA
       end
 
       def verify_storage_move(source_storage, destination_storage, repo_type: :project)
-        return if QA::Runtime::Env.dot_com?
+        return if Specs::Helpers::ContextSelector.dot_com?
 
         repo_path = verify_storage_move_from_gitaly(source_storage[:name], repo_type: repo_type)
 

qa/qa/specs/features/ee/browser_ui/1_manage/group/restrict_by_ip_address_spec.rb

@@ -3,7 +3,7 @@ require 'securerandom'
 
 module QA
   RSpec.describe 'Manage' do
-    describe 'Group access', :requires_admin, :skip_live_env do
+    describe 'Group access', :requires_admin, :skip_live_env, exclude: { job: 'review-qa-*' } do
       include Runtime::IPAddress
 
       before(:all) do
@@ -30,7 +30,7 @@ module QA
 
         @api_client = Runtime::API::Client.new(:gitlab, user: @user)
 
-        enable_plan_on_group(@sandbox_group.path, "Gold") if Runtime::Env.dot_com?
+        enable_plan_on_group(@sandbox_group.path, "Gold") if Specs::Helpers::ContextSelector.dot_com?
       end
 
       after(:all) do

qa/qa/specs/helpers/context_selector.rb

+# frozen_string_literal: true
+
+require 'rspec/core'
+
+module QA
+  module Specs
+    module Helpers
+      module ContextSelector
+        extend self
+
+        def configure_rspec
+          ::RSpec.configure do |config|
+            config.before do |example|
+              if example.metadata.key?(:only)
+                skip('Test is not compatible with this environment or pipeline') unless ContextSelector.context_matches?(example.metadata[:only])
+              elsif example.metadata.key?(:exclude)
+                skip('Test is excluded in this job') if ContextSelector.exclude?(example.metadata[:exclude])
+              end
+            end
+          end
+        end
+
+        def exclude?(*options)
+          return false unless Runtime::Env.ci_job_name.present?
+
+          context_matches?(*options)
+        end
+
+        def context_matches?(*options)
+          return false unless Runtime::Scenario.attributes[:gitlab_address]
+
+          opts = {}
+          opts[:domain] = '.+'
+          opts[:tld] = '.com'
+
+          uri = URI(Runtime::Scenario.gitlab_address)
+
+          options.each do |option|
+            opts[:domain] = 'gitlab' if option == :production
+
+            next unless option.is_a?(Hash)
+
+            if option[:pipeline].present? && Runtime::Env.ci_project_name.present?
+              return pipeline_matches?(option[:pipeline])
+
+            elsif option[:job].present?
+              return job_matches?(option[:job])
+
+            elsif option[:subdomain].present?
+              opts.merge!(option)
+
+              opts[:subdomain] = case option[:subdomain]
+                                 when Array
+                                   "(#{option[:subdomain].join("|")})."
+                                 when Regexp
+                                   option[:subdomain]
+                                 else
+                                   "(#{option[:subdomain]})."
+                                 end
+            end
+          end
+
+          uri.host.match?(/^#{opts[:subdomain]}#{opts[:domain]}#{opts[:tld]}$/)
+        end
+
+        alias_method :dot_com?, :context_matches?
+
+        def job_matches?(job_patterns)
+          Array(job_patterns).any? do |job|
+            pattern = job.is_a?(Regexp) ? job : Regexp.new(job)
+            pattern = Regexp.new(pattern.source, pattern.options | Regexp::IGNORECASE)
+            pattern =~ Runtime::Env.ci_job_name
+          end
+        end
+
+        def pipeline_matches?(pipeline_to_run_in)
+          Array(pipeline_to_run_in).any? { |pipeline| pipeline.to_s.casecmp?(pipeline_from_project_name(Runtime::Env.ci_project_name)) }
+        end
+
+        def pipeline_from_project_name(project_name)
+          project_name.to_s.start_with?('gitlab-qa') ? Runtime::Env.default_branch : project_name
+        end
+      end
+    end
+  end
+end

qa/qa/specs/helpers/quarantine.rb

@@ -6,22 +6,18 @@ module QA
   module Specs
     module Helpers
       module Quarantine
-        include RSpec::Core::Pending
+        include ::RSpec::Core::Pending
 
         extend self
 
         def configure_rspec
-          RSpec.configure do |config|
+          ::RSpec.configure do |config|
             config.before(:context, :quarantine) do
               Quarantine.skip_or_run_quarantined_contexts(config.inclusion_filter.rules, self.class)
             end
 
             config.before do |example|
               Quarantine.skip_or_run_quarantined_tests_or_contexts(config.inclusion_filter.rules, example)
-
-              if example.metadata.key?(:only)
-                skip('Test is not compatible with this environment or pipeline') unless Runtime::Env.context_matches?(example.metadata[:only])
-              end
             end
           end
         end
@@ -55,7 +51,7 @@ module QA
               if quarantine_tag.is_a?(Hash) && quarantine_tag&.key?(:only)
                 # If the :quarantine hash contains :only, we respect that.
                 # For instance `quarantine: { only: { subdomain: :staging } }` will only quarantine the test when it runs against staging.
-                return unless Runtime::Env.context_matches?(quarantine_tag[:only])
+                return unless ContextSelector.context_matches?(quarantine_tag[:only])
               end
 
               skip(quarantine_message(quarantine_tag))

qa/qa/specs/helpers/rspec.rb

+# frozen_string_literal: true
+
+require 'rspec/core'
+
+module QA
+  module Specs
+    module Helpers
+      module RSpec
+        # We need a reporter for internal tests that's different from the reporter for
+        # external tests otherwise the results will be mixed up. We don't care about
+        # most reporting, but we do want to know if a test fails
+        class RaiseOnFailuresReporter < ::RSpec::Core::NullReporter
+          def self.example_failed(example)
+            raise example.exception
+          end
+        end
+
+        # We use an example group wrapper to prevent the state of internal tests
+        # expanding into the global state
+        # See: https://github.com/rspec/rspec-core/issues/2603
+        def describe_successfully(*args, &describe_body)
+          example_group = RSpec.describe(*args, &describe_body)
+          ran_successfully = example_group.run RaiseOnFailuresReporter
+          expect(ran_successfully).to eq true
+          example_group
+        end
+      end
+    end
+  end
+end

qa/qa/specs/runner.rb

@@ -44,7 +44,7 @@ module QA
 
         tags_for_rspec.push(%w[--tag ~skip_signup_disabled]) if QA::Runtime::Env.signup_disabled?
 
-        tags_for_rspec.push(%w[--tag ~skip_live_env]) if QA::Runtime::Env.dot_com?
+        tags_for_rspec.push(%w[--tag ~skip_live_env]) if QA::Specs::Helpers::ContextSelector.dot_com?
 
         QA::Runtime::Env.supported_features.each_key do |key|
           tags_for_rspec.push(%W[--tag ~requires_#{key}]) unless QA::Runtime::Env.can_test? key

qa/spec/runtime/env_spec.rb

@@ -341,56 +341,4 @@ RSpec.describe QA::Runtime::Env do
       end
     end
   end
-
-  describe '.context_matches?' do
-    it 'returns true when url has .com' do
-      QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
-
-      expect(described_class.dot_com?).to be_truthy
-    end
-
-    it 'returns false when url does not have .com' do
-      QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.test")
-
-      expect(described_class.dot_com?).to be_falsey
-    end
-
-    context 'with arguments' do
-      it 'returns true when :subdomain is set' do
-        QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
-
-        expect(described_class.dot_com?(subdomain: :staging)).to be_truthy
-      end
-
-      it 'matches multiple subdomains' do
-        QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
-
-        expect(described_class.context_matches?(subdomain: [:release, :staging])).to be_truthy
-        expect(described_class.context_matches?(:production, subdomain: [:release, :staging])).to be_truthy
-      end
-
-      it 'matches :production' do
-        QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.com/")
-
-        expect(described_class.context_matches?(:production)).to be_truthy
-      end
-
-      it 'doesnt match with mismatching switches' do
-        QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.test')
-
-        aggregate_failures do
-          expect(described_class.context_matches?(tld: '.net')).to be_falsey
-          expect(described_class.context_matches?(:production)).to be_falsey
-          expect(described_class.context_matches?(subdomain: [:staging])).to be_falsey
-          expect(described_class.context_matches?(domain: 'example')).to be_falsey
-        end
-      end
-    end
-
-    it 'returns false for mismatching' do
-      QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
-
-      expect(described_class.context_matches?(:production)).to be_falsey
-    end
-  end
 end

qa/spec/spec_helper.rb

@@ -22,6 +22,7 @@ RSpec.configure do |config|
   config.include ::Matchers
 
   QA::Specs::Helpers::Quarantine.configure_rspec
+  QA::Specs::Helpers::ContextSelector.configure_rspec
 
   config.before do |example|
     QA::Runtime::Logger.debug("\nStarting test: #{example.full_description}\n")

qa/spec/specs/helpers/quarantine_spec.rb

@@ -2,25 +2,6 @@
 
 require 'rspec/core/sandbox'
 
-# We need a reporter for internal tests that's different from the reporter for
-# external tests otherwise the results will be mixed up. We don't care about
-# most reporting, but we do want to know if a test fails
-class RaiseOnFailuresReporter < RSpec::Core::NullReporter
-  def self.example_failed(example)
-    raise example.exception
-  end
-end
-
-# We use an example group wrapper to prevent the state of internal tests
-# expanding into the global state
-# See: https://github.com/rspec/rspec-core/issues/2603
-def describe_successfully(*args, &describe_body)
-  example_group = RSpec.describe(*args, &describe_body)
-  ran_successfully = example_group.run RaiseOnFailuresReporter
-  expect(ran_successfully).to eq true
-  example_group
-end
-
 RSpec.configure do |c|
   c.around do |ex|
     RSpec::Core::Sandbox.sandboxed do |config|
@@ -38,6 +19,7 @@ end
 
 RSpec.describe QA::Specs::Helpers::Quarantine do
   include Helpers::StubENV
+  include QA::Specs::Helpers::RSpec
 
   describe '.skip_or_run_quarantined_contexts' do
     context 'with no tag focused' do
@@ -336,159 +318,4 @@ RSpec.describe QA::Specs::Helpers::Quarantine do
       end
     end
   end
-
-  describe 'running against specific environments or pipelines' do
-    before do
-      QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com')
-      described_class.configure_rspec
-    end
-
-    describe 'description and context blocks' do
-      context 'with environment set' do
-        it 'can apply to contexts or descriptions' do
-          group = describe_successfully 'Runs in staging', only: { subdomain: :staging } do
-            it('runs in staging') {}
-          end
-
-          expect(group.examples[0].execution_result.status).to eq(:passed)
-        end
-      end
-
-      context 'with different environment set' do
-        before do
-          QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com')
-          described_class.configure_rspec
-        end
-
-        it 'does not run against production' do
-          group = describe_successfully 'Runs in staging', :something, only: { subdomain: :staging } do
-            it('runs in staging') {}
-          end
-
-          expect(group.examples[0].execution_result.status).to eq(:pending)
-        end
-      end
-    end
-
-    it 'runs only in staging' do
-      group = describe_successfully do
-        it('runs in staging', only: { subdomain: :staging }) {}
-        it('doesnt run in staging', only: :production) {}
-        it('runs in staging also', only: { subdomain: %i[release staging] }) {}
-        it('runs in any env') {}
-      end
-
-      expect(group.examples[0].execution_result.status).to eq(:passed)
-      expect(group.examples[1].execution_result.status).to eq(:pending)
-      expect(group.examples[2].execution_result.status).to eq(:passed)
-      expect(group.examples[3].execution_result.status).to eq(:passed)
-    end
-
-    context 'custom env' do
-      before do
-        QA::Runtime::Scenario.define(:gitlab_address, 'https://release.gitlab.net')
-      end
-
-      it 'runs on a custom environment' do
-        group = describe_successfully do
-          it('runs on release gitlab net', only: { tld: '.net', subdomain: :release, domain: 'gitlab' }) {}
-          it('does not run on release', only: :production) {}
-        end
-
-        expect(group.examples.first.execution_result.status).to eq(:passed)
-        expect(group.examples.last.execution_result.status).to eq(:pending)
-      end
-    end
-
-    context 'production' do
-      before do
-        QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com/')
-      end
-
-      it 'runs on production' do
-        group = describe_successfully do
-          it('runs on prod', only: :production) {}
-          it('does not run in prod', only: { subdomain: :staging }) {}
-          it('runs in prod and staging', only: { subdomain: /(staging.)?/, domain: 'gitlab' }) {}
-        end
-
-        expect(group.examples[0].execution_result.status).to eq(:passed)
-        expect(group.examples[1].execution_result.status).to eq(:pending)
-        expect(group.examples[2].execution_result.status).to eq(:passed)
-      end
-    end
-
-    it 'outputs a message for invalid environments' do
-      group = describe_successfully do
-        it('will skip', only: :production) {}
-      end
-
-      expect(group.examples.first.execution_result.pending_message).to match(/[Tt]est.*not compatible.*environment/)
-    end
-
-    context 'with pipeline constraints' do
-      context 'without CI_PROJECT_NAME set' do
-        before do
-          stub_env('CI_PROJECT_NAME', nil)
-          described_class.configure_rspec
-        end
-
-        it 'runs on any pipeline' do
-          group = describe_successfully do
-            it('runs given a single named pipeline', only: { pipeline: :nightly }) {}
-            it('runs given an array of pipelines', only: { pipeline: [:canary, :not_nightly] }) {}
-          end
-
-          aggregate_failures do
-            expect(group.examples[0].execution_result.status).to eq(:passed)
-            expect(group.examples[1].execution_result.status).to eq(:passed)
-          end
-        end
-      end
-
-      context 'when a pipeline triggered from the default branch runs in gitlab-qa' do
-        before do
-          stub_env('CI_PROJECT_NAME', 'gitlab-qa')
-          described_class.configure_rspec
-        end
-
-        it 'runs on default branch pipelines' do
-          group = describe_successfully do
-            it('runs on master pipeline given a single pipeline', only: { pipeline: :master }) {}
-            it('runs in master given an array of pipelines', only: { pipeline: [:canary, :master] }) {}
-            it('does not run in non-default pipelines', only: { pipeline: [:nightly, :not_nightly, :not_master] }) {}
-          end
-
-          aggregate_failures do
-            expect(group.examples[0].execution_result.status).to eq(:passed)
-            expect(group.examples[1].execution_result.status).to eq(:passed)
-            expect(group.examples[2].execution_result.status).to eq(:pending)
-          end
-        end
-      end
-
-      context 'with CI_PROJECT_NAME set' do
-        before do
-          stub_env('CI_PROJECT_NAME', 'NIGHTLY')
-          described_class.configure_rspec
-        end
-
-        it 'runs on designated pipeline' do
-          group = describe_successfully do
-            it('runs on nightly', only: { pipeline: :nightly }) {}
-            it('does not run in not_nightly', only: { pipeline: :not_nightly }) {}
-            it('runs on nightly given an array', only: { pipeline: [:canary, :nightly] }) {}
-            it('does not run in not_nightly given an array', only: { pipeline: [:not_nightly, :canary] }) {}
-          end
-
-          aggregate_failures do
-            expect(group.examples[0].execution_result.status).to eq(:passed)
-            expect(group.examples[1].execution_result.status).to eq(:pending)
-            expect(group.examples[2].execution_result.status).to eq(:passed)
-            expect(group.examples[3].execution_result.status).to eq(:pending)
-          end
-        end
-      end
-    end
-  end
 end