Add ability to disable feature flag from command line

Also add unit tests for disable-feature command line option

qa/README.md

@@ -178,11 +178,13 @@ another test has `:ldap` and `:quarantine` metadata. If the tests are run with
 `--tag smoke --tag quarantine`, only the first test will run. The test with
 `:ldap` will not run even though it also has `:quarantine`.
 
-### Running tests with a feature flag enabled
+### Running tests with a feature flag enabled or disabled
 
-Tests can be run with with a feature flag enabled by using the command-line
-option `--enable-feature FEATURE_FLAG`. For example, to enable the feature flag
-that enforces Gitaly request limits, you would use the command:
+Tests can be run with with a feature flag enabled or disabled by using the command-line
+option `--enable-feature FEATURE_FLAG` or `--disable-feature FEATURE_FLAG`.
+
+For example, to enable the feature flag that enforces Gitaly request limits,
+you would use the command:
 
 ```
 bundle exec bin/qa Test::Instance::All http://localhost:3000 --enable-feature gitaly_enforce_requests_limits
@@ -193,9 +195,20 @@ feature flag ([via the API](https://docs.gitlab.com/ee/api/features.html)), run
 all the tests in the `Test::Instance::All` scenario, and then disable the
 feature flag again.
 
+Similarly, to disable the feature flag that enforces Gitaly request limits,
+you would use the command:
+
+```
+bundle exec bin/qa Test::Instance::All http://localhost:3000 --disable-feature gitaly_enforce_requests_limits
+```
+This will instruct the QA framework to disable the `gitaly_enforce_requests_limits`
+feature flag ([via the API](https://docs.gitlab.com/ee/api/features.html)) if not already disabled,
+run all the tests in the `Test::Instance::All` scenario, and then enable the
+feature flag again if it was enabled earlier.
+
 Note: the QA framework doesn't currently allow you to easily toggle a feature
 flag during a single test, [as you can in unit tests](https://docs.gitlab.com/ee/development/feature_flags.html#specs),
 but [that capability is planned](https://gitlab.com/gitlab-org/quality/team-tasks/issues/77).
 
-Note also that the `--` separator isn't used because `--enable-feature` is a QA
-framework option, not an `rspec` option.
+Note also that the `--` separator isn't used because `--enable-feature` and `--disable-feature`
+are QA framework options, not `rspec` options.

qa/qa/scenario/shared_attributes.rb

@@ -7,6 +7,7 @@ module QA
 
       attribute :gitlab_address, '--address URL', 'Address of the instance to test'
       attribute :enable_feature, '--enable-feature FEATURE_FLAG', 'Enable a feature before running tests'
+      attribute :disable_feature, '--disable-feature FEATURE_FLAG', 'Disable a feature before running tests'
       attribute :parallel, '--parallel', 'Execute tests in parallel'
       attribute :loop, '--loop', 'Execute test repeatedly'
     end

qa/qa/scenario/template.rb

@@ -30,6 +30,8 @@ module QA
 
         Runtime::Feature.enable(options[:enable_feature]) if options.key?(:enable_feature)
 
+        Runtime::Feature.disable(options[:disable_feature]) if options.key?(:disable_feature) && (@feature_enabled = Runtime::Feature.enabled?(options[:disable_feature]))
+
         Specs::Runner.perform do |specs|
           specs.tty = true
           specs.tags = self.class.focus
@@ -37,6 +39,7 @@ module QA
         end
       ensure
         Runtime::Feature.disable(options[:enable_feature]) if options.key?(:enable_feature)
+        Runtime::Feature.enable(options[:disable_feature]) if options.key?(:disable_feature) && @feature_enabled
       end
 
       def extract_option(name, options, args)

qa/spec/scenario/template_spec.rb

@@ -17,6 +17,24 @@ describe QA::Scenario::Template do
     expect(feature).to have_received(:enable).with('a-feature')
   end
 
+  it 'allows a feature to be disabled' do
+    allow(QA::Runtime::Feature).to receive(:enabled?)
+                                     .with('another-feature').and_return(true)
+
+    subject.perform({ disable_feature: 'another-feature' })
+
+    expect(feature).to have_received(:disable).with('another-feature')
+  end
+
+  it 'does not disable a feature if already disabled' do
+    allow(QA::Runtime::Feature).to receive(:enabled?)
+                                     .with('another-feature').and_return(false)
+
+    subject.perform({ disable_feature: 'another-feature' })
+
+    expect(feature).not_to have_received(:disable).with('another-feature')
+  end
+
   it 'ensures an enabled feature is disabled afterwards' do
     allow(QA::Specs::Runner).to receive(:perform).and_raise('failed test')
 
@@ -25,4 +43,28 @@ describe QA::Scenario::Template do
     expect(feature).to have_received(:enable).with('a-feature')
     expect(feature).to have_received(:disable).with('a-feature')
   end
+
+  it 'ensures a disabled feature is enabled afterwards' do
+    allow(QA::Specs::Runner).to receive(:perform).and_raise('failed test')
+
+    allow(QA::Runtime::Feature).to receive(:enabled?)
+                                     .with('another-feature').and_return(true)
+
+    expect { subject.perform({ disable_feature: 'another-feature' }) }.to raise_error('failed test')
+
+    expect(feature).to have_received(:disable).with('another-feature')
+    expect(feature).to have_received(:enable).with('another-feature')
+  end
+
+  it 'ensures a disabled feature is not enabled afterwards if it was disabled earlier' do
+    allow(QA::Specs::Runner).to receive(:perform).and_raise('failed test')
+
+    allow(QA::Runtime::Feature).to receive(:enabled?)
+                                     .with('another-feature').and_return(false)
+
+    expect { subject.perform({ disable_feature: 'another-feature' }) }.to raise_error('failed test')
+
+    expect(feature).not_to have_received(:disable).with('another-feature')
+    expect(feature).not_to have_received(:enable).with('another-feature')
+  end
 end