Merge branch 'jj-add-gitlab-experiment' into 'master'

Add the gitlab-experiment gem, with configuration

See merge request gitlab-org/gitlab!45840

Gemfile

@@ -479,6 +479,7 @@ gem 'flipper', '~> 0.17.1'
 gem 'flipper-active_record', '~> 0.17.1'
 gem 'flipper-active_support_cache_store', '~> 0.17.1'
 gem 'unleash', '~> 0.1.5'
+gem 'gitlab-experiment', '~> 0.4.2'
 
 # Structured logging
 gem 'lograge', '~> 0.5'

Gemfile.lock

@@ -427,6 +427,9 @@ GEM
     github-markup (1.7.0)
     gitlab-chronic (0.10.5)
       numerizer (~> 0.2)
+    gitlab-experiment (0.4.2)
+      activesupport (>= 3.0)
+      scientist (~> 1.5, >= 1.5.0)
     gitlab-fog-azure-rm (1.0.0)
       azure-storage-blob (~> 2.0)
       azure-storage-common (~> 2.0)
@@ -1087,6 +1090,7 @@ GEM
     sawyer (0.8.2)
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
+    scientist (1.5.0)
     scss_lint (0.59.0)
       sass (~> 3.5, >= 3.5.5)
     securecompare (1.0.0)
@@ -1353,6 +1357,7 @@ DEPENDENCIES
   gitaly (~> 13.7.0.pre.rc1)
   github-markup (~> 1.7.0)
   gitlab-chronic (~> 0.10.5)
+  gitlab-experiment (~> 0.4.2)
   gitlab-fog-azure-rm (~> 1.0)
   gitlab-labkit (= 0.13.3)
   gitlab-license (~> 1.0)

app/controllers/projects/issues_controller.rb

@@ -59,6 +59,10 @@ class Projects::IssuesController < Projects::ApplicationController
 
   around_action :allow_gitaly_ref_name_caching, only: [:discussions]
 
+  before_action :run_null_hypothesis_experiment,
+                only: [:index, :new, :create],
+                if: -> { Feature.enabled?(:gitlab_experiments) }
+
   respond_to :html
 
   alias_method :designs, :show
@@ -390,6 +394,14 @@ class Projects::IssuesController < Projects::ApplicationController
     action_name == 'service_desk'
   end
 
+  def run_null_hypothesis_experiment
+    experiment(:null_hypothesis, project: project) do |e|
+      e.use { } # define the control
+      e.try { } # define the candidate
+      e.track(action_name) # track the action so we can build a funnel
+    end
+  end
+
   # Overridden in EE
   def create_vulnerability_issue_link(issue); end
 end

changelogs/unreleased/jj-add-gitlab-experiment.yml

+---
+title: Add the gitlab-experiment gem, with configuration
+merge_request: 45840
+author:
+type: added

config/feature_flags/development/gitlab_experiments.yml

+---
+name: gitlab_experiments
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45840
+rollout_issue_url: 
+milestone: '13.7'
+type: development
+group: group::adoption
+default_enabled: false

config/feature_flags/experiment/null_hypothesis.yml

+---
+name: null_hypothesis
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45840
+rollout_issue_url:
+type: experiment
+group: group::adoption
+default_enabled: false

config/initializers/gitlab_experiment.rb

+# frozen_string_literal: true
+
+Gitlab::Experiment.configure do |config|
+  # Logic this project uses to resolve a variant for a given experiment.
+  #
+  # This can return an instance of any object that responds to `name`, or can
+  # return a variant name as a symbol or string.
+  #
+  # This block will be executed within the scope of the experiment instance,
+  # so can easily access experiment methods, like getting the name or context.
+  config.variant_resolver = lambda do |requested_variant|
+    # Return the variant if one was requested in code:
+    break requested_variant if requested_variant.present?
+
+    # Use Feature interface to determine the variant by passing the experiment,
+    # which responds to `flipper_id` and `session_id` to accommodate adapters.
+    variant_names.first if Feature.enabled?(name, self, type: :experiment)
+  end
+
+  # Tracking behavior can be implemented to link an event to an experiment.
+  #
+  # Similar to the variant_resolver, this is called within the scope of the
+  # experiment instance and so can access any methods on the experiment,
+  # such as name and signature.
+  config.tracking_behavior = lambda do |event, args|
+    Gitlab::Tracking.event(name, event.to_s, **args.merge(
+      context: (args[:context] || []) << SnowplowTracker::SelfDescribingJson.new(
+        'iglu:com.gitlab/gitlab_experiment/jsonschema/0-3-0', signature
+      )
+    ))
+  end
+
+  # Called at the end of every experiment run, with the result.
+  #
+  # You may want to track that you've assigned a variant to a given context,
+  # or push the experiment into the client or publish results elsewhere, like
+  # into redis or postgres. Also called within the scope of the experiment
+  # instance.
+  config.publishing_behavior = lambda do |result|
+    # Track the event using our own configured tracking logic.
+    track(:assignment)
+
+    # Push the experiment knowledge into the front end. The signature contains
+    # the context key, and the variant that has been determined.
+    Gon.push({ experiment: { name => signature } }, true)
+  end
+end

lib/feature/shared.rb

@@ -23,7 +23,7 @@ class Feature
         example: <<-EOS
           Feature.enabled?(:my_feature_flag, project)
           Feature.enabled?(:my_feature_flag, project, type: :development)
-          push_frontend_feature_flag?(:my_feature_flag, project)
+          push_frontend_feature_flag(:my_feature_flag, project)
         EOS
       },
       ops: {
@@ -33,8 +33,8 @@ class Feature
         ee_only: false,
         default_enabled: false,
         example: <<-EOS
-          Feature.enabled?(:my_ops_flag, type: ops)
-          push_frontend_feature_flag?(:my_ops_flag, project, type: :ops)
+          Feature.enabled?(:my_ops_flag, type: :ops)
+          push_frontend_feature_flag(:my_ops_flag, project, type: :ops)
         EOS
       },
       licensed: {
@@ -48,6 +48,16 @@ class Feature
           project.feature_available?(:my_licensed_feature)
           namespace.feature_available?(:my_licensed_feature)
         EOS
+      },
+      experiment: {
+        description: 'Short lived, used specifically to run A/B/n experiments.',
+        optional: true,
+        rollout_issue: true,
+        ee_only: true,
+        default_enabled: false,
+        example: <<-EOS
+          experiment(:my_experiment, project: project, actor: current_user) { ...variant code... }
+        EOS
       }
     }.freeze
 

spec/controllers/projects/issues_controller_spec.rb

@@ -62,6 +62,56 @@ RSpec.describe Projects::IssuesController do
           expect(response).to have_gitlab_http_status(:moved_permanently)
         end
       end
+
+      describe 'the null hypothesis experiment', :snowplow do
+        it 'defines the expected before actions' do
+          expect(controller).to use_before_action(:run_null_hypothesis_experiment)
+        end
+
+        context 'when rolled out to 100%' do
+          it 'assigns the candidate experience and tracks the event' do
+            get :index, params: { namespace_id: project.namespace, project_id: project }
+
+            expect_snowplow_event(
+              category: 'null_hypothesis',
+              action: 'index',
+              context: [{
+                schema: 'iglu:com.gitlab/gitlab_experiment/jsonschema/0-3-0',
+                data: { variant: 'candidate', experiment: 'null_hypothesis', key: anything }
+              }]
+            )
+          end
+        end
+
+        context 'when not rolled out' do
+          before do
+            stub_feature_flags(null_hypothesis: false)
+          end
+
+          it 'assigns the control experience and tracks the event' do
+            get :index, params: { namespace_id: project.namespace, project_id: project }
+
+            expect_snowplow_event(
+              category: 'null_hypothesis',
+              action: 'index',
+              context: [{
+                schema: 'iglu:com.gitlab/gitlab_experiment/jsonschema/0-3-0',
+                data: { variant: 'control', experiment: 'null_hypothesis', key: anything }
+              }]
+            )
+          end
+        end
+
+        context 'when gitlab_experiments is disabled' do
+          it 'does not run the experiment at all' do
+            stub_feature_flags(gitlab_experiments: false)
+
+            expect(controller).not_to receive(:run_null_hypothesis_experiment)
+
+            get :index, params: { namespace_id: project.namespace, project_id: project }
+          end
+        end
+      end
     end
 
     context 'internal issue tracker' do

spec/support/helpers/snowplow_helpers.rb

@@ -31,7 +31,31 @@ module SnowplowHelpers
   #       )
   #     end
   #   end
-  def expect_snowplow_event(category:, action:, **kwargs)
+  #
+  # Passing context:
+  #
+  #   Simply provide a hash that has the schema and data expected.
+  #
+  #   expect_snowplow_event(
+  #     category: 'Experiment',
+  #     action: 'created',
+  #     context: [
+  #       {
+  #         schema: 'iglu:com.gitlab/.../0-3-0',
+  #         data: { key: 'value' }
+  #       }
+  #     ]
+  #   )
+  def expect_snowplow_event(category:, action:, context: nil, **kwargs)
+    if context
+      kwargs[:context] = []
+      context.each do |c|
+        expect(SnowplowTracker::SelfDescribingJson).to have_received(:new)
+          .with(c[:schema], c[:data]).at_least(:once)
+        kwargs[:context] << an_instance_of(SnowplowTracker::SelfDescribingJson)
+      end
+    end
+
     expect(Gitlab::Tracking).to have_received(:event) # rubocop:disable RSpec/ExpectGitlabTracking
       .with(category, action, **kwargs).at_least(:once)
   end

spec/support/snowplow.rb

@@ -17,6 +17,7 @@ RSpec.configure do |config|
 
     stub_application_setting(snowplow_enabled: true)
 
+    allow(SnowplowTracker::SelfDescribingJson).to receive(:new).and_call_original
     allow(Gitlab::Tracking).to receive(:event).and_call_original # rubocop:disable RSpec/ExpectGitlabTracking
   end