Commit 3cdc8904 authored by Nicolas Dular's avatar Nicolas Dular

Introduce experiment subjects

This adds the possilibity to run experiments for subjects instead of
always using the cookie.
parent 3de560ca
......@@ -109,15 +109,6 @@ class InvitesController < ApplicationController
end
def track_invitation_reminders_experiment(action)
return unless Gitlab::Experimentation.enabled?(:invitation_reminders)
property = Gitlab::Experimentation.enabled_for_attribute?(:invitation_reminders, member.invite_email) ? 'experimental_group' : 'control_group'
Gitlab::Tracking.event(
Gitlab::Experimentation.experiment(:invitation_reminders).tracking_category,
action,
property: property,
label: Digest::MD5.hexdigest(member.to_global_id.to_s)
)
track_experiment_event(:invitation_reminders, action, subject: member)
end
end
......@@ -64,11 +64,11 @@ module Emails
layout: 'unknown_user_mailer'
)
if Gitlab::Experimentation.enabled?(:invitation_reminders)
if Gitlab::Experimentation.active?(:invitation_reminders)
Gitlab::Tracking.event(
Gitlab::Experimentation.experiment(:invitation_reminders).tracking_category,
Gitlab::Experimentation.get_experiment(:invitation_reminders).tracking_category,
'sent',
property: Gitlab::Experimentation.enabled_for_attribute?(:invitation_reminders, member.invite_email) ? 'experimental_group' : 'control_group',
property: Gitlab::Experimentation.in_experiment_group?(:invitation_reminders, subject: member.invite_email) ? 'experimental_group' : 'control_group',
label: Digest::MD5.hexdigest(member.to_global_id.to_s)
)
end
......
......@@ -25,7 +25,7 @@ module Members
private
def experiment_enabled?
Gitlab::Experimentation.enabled_for_attribute?(:invitation_reminders, invitation.invite_email)
Gitlab::Experimentation.in_experiment_group?(:invitation_reminders, subject: invitation.invite_email)
end
def days_after_invitation_sent
......
......@@ -8,7 +8,7 @@ class MemberInvitationReminderEmailsWorker # rubocop:disable Scalability/Idempot
urgency :low
def perform
return unless Gitlab::Experimentation.enabled?(:invitation_reminders)
return unless Gitlab::Experimentation.active?(:invitation_reminders)
Member.not_accepted_invitations.not_expired.last_ten_days_excluding_today.find_in_batches do |invitations|
invitations.each do |invitation|
......
......@@ -56,59 +56,90 @@ addressed.
1. Use the experiment in the code.
Experiments can be performed on a `subject`. The `subject` that gets provided needs to respond to `to_global_id` or `to_s`.
The resulting string is bucketed and assigned to either the control or the experimental group. It's therefore necessary to always provide the same `subject` for an experiment to have the same experience.
- Use this standard for the experiment in a controller:
```ruby
class RegistrationController < ApplicationController
Experiment run for a user:
```ruby
class ProjectController < ApplicationController
def show
# experiment_enabled?(:experiment_key) is also available in views and helpers
if experiment_enabled?(:signup_flow, subject: current_user)
# render the experiment
else
# render the original version
end
end
end
```
or experiment run for a namespace:
```ruby
if experiment_enabled?(:signup_flow, subject: namespace)
# experiment code
else
# control code
end
```
When no subject is given, it falls back to a cookie that gets set and is consistent until
the cookie gets deleted.
```ruby
class RegistrationController < ApplicationController
def show
# falls back to a cookie
if experiment_enabled?(:signup_flow)
# render the experiment
else
# render the original version
end
end
end
```
end
```
- Make the experiment available to the frontend in a controller:
```ruby
before_action do
push_frontend_experiment(:signup_flow)
end
```
```ruby
before_action do
push_frontend_experiment(:signup_flow, subject: current_user)
end
```
The above checks whether the experiment is enabled and push the result to the frontend.
The above checks whether the experiment is enabled and pushes the result to the frontend.
You can check the state of the feature flag in JavaScript:
You can check the state of the feature flag in JavaScript:
```javascript
import { isExperimentEnabled } from '~/experimentation';
```javascript
import { isExperimentEnabled } from '~/experimentation';
if ( isExperimentEnabled('signupFlow') ) {
// ...
}
```
if ( isExperimentEnabled('signupFlow') ) {
// ...
}
```
- It is also possible to run an experiment outside of the controller scope, for example in a worker:
```ruby
class SomeWorker
def perform
# Check if the experiment is enabled at all (the percentage_of_time_value > 0)
return unless Gitlab::Experimentation.enabled?(:experiment_key)
# Since we cannot access cookies in a worker, we need to bucket models based on a unique, unchanging attribute instead.
# Use the following method to check if the experiment is enabled for a certain attribute, for example a username or email address:
if Gitlab::Experimentation.enabled_for_attribute?(:experiment_key, some_attribute)
# execute experimental code
else
# execute control code
end
end
end
```
```ruby
class SomeWorker
def perform
# Check if the experiment is active at all (the percentage_of_time_value > 0)
return unless Gitlab::Experimentation.active?(:experiment_key)
# Since we cannot access cookies in a worker, we need to bucket models based on a unique, unchanging attribute instead.
# It is therefore necessery to always provide the same subject.
if Gitlab::Experimentation.in_experiment_group?(:experiment_key, subject: user)
# execute experimental code
else
# execute control code
end
end
end
```
### Implement the tracking events
......@@ -122,7 +153,7 @@ The framework provides the following helper method that is available in controll
```ruby
before_action do
track_experiment_event(:signup_flow, 'action', 'value')
track_experiment_event(:signup_flow, 'action', 'value', subject: current_user)
end
```
......@@ -132,7 +163,7 @@ Which can be tested as follows:
context 'when the experiment is active and the user is in the experimental group' do
before do
stub_experiment(signup_flow: true)
stub_experiment_for_user(signup_flow: true)
stub_experiment_for_subject(signup_flow: true)
end
it 'tracks an event', :snowplow do
......@@ -155,8 +186,8 @@ The framework provides the following helper method that is available in controll
```ruby
before_action do
push_frontend_experiment(:signup_flow)
frontend_experimentation_tracking_data(:signup_flow, 'action', 'value')
push_frontend_experiment(:signup_flow, subject: current_user)
frontend_experimentation_tracking_data(:signup_flow, 'action', 'value', subject: current_user)
end
```
......@@ -255,7 +286,7 @@ Along with the tracking of backend and frontend events and the [recording of exp
- **Experimental experience:** Show an in-product nudge to see if it causes more people to sign up for trials.
- **Conversion event:** The user starts a trial.
The `record_experiment_conversion_event` helper method is available to all controllers, and enables us to easily record the conversion event for the current user, regardless of whether they are in the control or experimental group:
The `record_experiment_conversion_event` helper method is available to all controllers. It enables us to record the conversion event for the current user, regardless of whether they are in the control or experimental group:
```ruby
before_action do
......@@ -296,7 +327,7 @@ context 'when the experiment is active' do
context 'when the user is in the experimental group' do
before do
stub_experiment_for_user(signup_flow: true)
stub_experiment_for_subject(signup_flow: true)
end
it { is_expected.to do_experimental_thing }
......@@ -304,7 +335,7 @@ context 'when the experiment is active' do
context 'when the user is in the control group' do
before do
stub_experiment_for_user(signup_flow: false)
stub_experiment_for_subject(signup_flow: false)
end
it { is_expected.to do_control_thing }
......
......@@ -32,7 +32,7 @@ module BillingPlansHelper
end
def experiment_tracking_data_for_button_click(button_label)
return {} unless Gitlab::Experimentation.enabled?(:contact_sales_btn_in_app)
return {} unless Gitlab::Experimentation.active?(:contact_sales_btn_in_app)
{
track: {
......
......@@ -12,7 +12,7 @@ module API
end
get do
experiments = Gitlab::Experimentation::EXPERIMENTS.keys.map do |experiment_key|
{ key: experiment_key, enabled: Gitlab::Experimentation.enabled?(experiment_key) }
{ key: experiment_key, enabled: Gitlab::Experimentation.active?(experiment_key) }
end
present experiments, with: EE::API::Entities::Experiment, current_user: current_user
......
......@@ -16,7 +16,7 @@ RSpec.describe Registrations::GroupsController do
context 'with an authenticated user' do
before do
sign_in(user)
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
end
it { is_expected.to have_gitlab_http_status(:ok) }
......@@ -37,7 +37,7 @@ RSpec.describe Registrations::GroupsController do
context 'with the experiment not enabled for user' do
before do
stub_experiment_for_user(onboarding_issues: false)
stub_experiment_for_subject(onboarding_issues: false)
end
it { is_expected.to have_gitlab_http_status(:not_found) }
......@@ -58,7 +58,7 @@ RSpec.describe Registrations::GroupsController do
context 'with an authenticated user' do
before do
sign_in(user)
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
end
it 'creates a group' do
......@@ -122,7 +122,7 @@ RSpec.describe Registrations::GroupsController do
context 'with the experiment not enabled for user' do
before do
stub_experiment_for_user(onboarding_issues: false)
stub_experiment_for_subject(onboarding_issues: false)
end
it { is_expected.to have_gitlab_http_status(:not_found) }
......
......@@ -17,7 +17,7 @@ RSpec.describe Registrations::ProjectsController do
context 'with an authenticated user' do
before do
sign_in(user)
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
end
it { is_expected.to have_gitlab_http_status(:not_found) }
......@@ -39,7 +39,7 @@ RSpec.describe Registrations::ProjectsController do
context 'with the experiment not enabled for user' do
before do
stub_experiment_for_user(onboarding_issues: false)
stub_experiment_for_subject(onboarding_issues: false)
end
it { is_expected.to have_gitlab_http_status(:not_found) }
......@@ -61,7 +61,7 @@ RSpec.describe Registrations::ProjectsController do
before do
namespace.add_owner(user)
sign_in(user)
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
end
it 'creates a new project, a "Learn GitLab" project, sets a cookie and redirects to the experience level page' do
......@@ -88,7 +88,7 @@ RSpec.describe Registrations::ProjectsController do
context 'with the experiment not enabled for user' do
before do
stub_experiment_for_user(onboarding_issues: false)
stub_experiment_for_subject(onboarding_issues: false)
end
it { is_expected.to have_gitlab_http_status(:not_found) }
......
......@@ -76,7 +76,7 @@ RSpec.describe Registrations::WelcomeController do
context 'when part of the onboarding issues experiment' do
before do
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
end
it { is_expected.to redirect_to new_users_sign_up_group_path }
......@@ -123,7 +123,7 @@ RSpec.describe Registrations::WelcomeController do
sign_in(user)
allow(::Gitlab).to receive(:com?).and_return(on_gitlab_com)
stub_experiment(onboarding_issues: experiment_enabled)
stub_experiment_for_user(onboarding_issues: experiment_enabled_for_user)
stub_experiment_for_subject(onboarding_issues: experiment_enabled_for_user)
allow(controller.helpers).to receive(:in_subscription_flow?).and_return(in_subscription_flow)
allow(controller.helpers).to receive(:in_invitation_flow?).and_return(in_invitation_flow)
allow(controller.helpers).to receive(:in_oauth_flow?).and_return(in_oauth_flow)
......
......@@ -135,7 +135,7 @@ RSpec.describe TrialsController do
context 'when the group-only trials experiment is active' do
before do
stub_experiment(group_only_trials: true)
stub_experiment_for_user(group_only_trials: user_is_in_experiment?)
stub_experiment_for_subject(group_only_trials: user_is_in_experiment?)
end
def expected_group_type
......
......@@ -18,7 +18,7 @@ RSpec.describe 'Billing plan pages', :feature do
end
before do
stub_experiment_for_user(contact_sales_btn_in_app: true)
stub_experiment_for_subject(contact_sales_btn_in_app: true)
stub_full_request("#{EE::SUBSCRIPTIONS_URL}/gitlab_plans?plan=#{plan.name}")
.to_return(status: 200, body: plans_data.to_json)
stub_application_setting(check_namespace_plan: true)
......
......@@ -5,7 +5,7 @@ require 'spec_helper'
RSpec.describe 'User sees new onboarding flow', :js do
before do
stub_const('Gitlab::QueryLimiting::Transaction::THRESHOLD', 200)
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
allow(Gitlab).to receive(:com?).and_return(true)
gitlab_sign_in(:user)
visit users_sign_up_welcome_path
......
......@@ -17,7 +17,7 @@ RSpec.describe 'Welcome screen', :js do
allow_any_instance_of(EE::WelcomeHelper).to receive(:in_invitation_flow?).and_return(in_invitation_flow)
allow_any_instance_of(EE::WelcomeHelper).to receive(:in_subscription_flow?).and_return(in_subscription_flow)
allow_any_instance_of(EE::WelcomeHelper).to receive(:in_trial_flow?).and_return(in_trial_flow)
stub_experiment_for_user(onboarding_issues: part_of_onboarding_issues_experiment)
stub_experiment_for_subject(onboarding_issues: part_of_onboarding_issues_experiment)
visit users_sign_up_welcome_path
end
......
......@@ -84,23 +84,49 @@ module Gitlab
}.freeze
class << self
def experiment(key)
Gitlab::Experimentation::Experiment.new(key, **EXPERIMENTS[key])
def get_experiment(experiment_key)
return unless EXPERIMENTS.key?(experiment_key)
::Gitlab::Experimentation::Experiment.new(experiment_key, **EXPERIMENTS[experiment_key])
end
def enabled?(experiment_key)
return false unless EXPERIMENTS.key?(experiment_key)
def active?(experiment_key)
experiment = get_experiment(experiment_key)
return false unless experiment
experiment(experiment_key).enabled?
experiment.active?
end
def enabled_for_attribute?(experiment_key, attribute)
index = Digest::SHA1.hexdigest(attribute).hex % 100
enabled_for_value?(experiment_key, index)
def in_experiment_group?(experiment_key, subject:)
return false if subject.blank?
return false unless active?(experiment_key)
experiment = get_experiment(experiment_key)
return false unless experiment
experiment.enabled_for_index?(index_for_subject(experiment, subject))
end
private
def index_for_subject(experiment, subject)
index = if experiment.use_backwards_compatible_subject_index
Digest::SHA1.hexdigest(subject_id(subject)).hex
else
Zlib.crc32("#{experiment.key}#{subject_id(subject)}")
end
index % 100
end
def enabled_for_value?(experiment_key, value)
enabled?(experiment_key) && experiment(experiment_key).enabled_for_index?(value)
def subject_id(subject)
if subject.respond_to?(:to_global_id)
subject.to_global_id.to_s
elsif subject.respond_to?(:to_s)
subject.to_s
else
raise ArgumentError.new('Subject must respond to `to_global_id` or `to_s`')
end
end
end
end
......
......@@ -3,7 +3,7 @@
require 'zlib'
# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent.
# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method
# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name, subject: nil)` method
# to controllers and views. It returns true when the experiment is enabled and the user is selected as part
# of the experimental group.
#
......@@ -28,55 +28,56 @@ module Gitlab
}
end
def push_frontend_experiment(experiment_key)
def push_frontend_experiment(experiment_key, subject: nil)
var_name = experiment_key.to_s.camelize(:lower)
enabled = experiment_enabled?(experiment_key)
enabled = experiment_enabled?(experiment_key, subject: subject)
gon.push({ experiments: { var_name => enabled } }, true)
end
def experiment_enabled?(experiment_key)
def experiment_enabled?(experiment_key, subject: nil)
return true if forced_enabled?(experiment_key)
return false if dnt_enabled?
return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index(experiment_key))
return true if forced_enabled?(experiment_key)
subject ||= fallback_experimentation_subject_index(experiment_key)
false
Experimentation.in_experiment_group?(experiment_key, subject: subject)
end
def track_experiment_event(experiment_key, action, value = nil)
def track_experiment_event(experiment_key, action, value = nil, subject: nil)
return if dnt_enabled?
track_experiment_event_for(experiment_key, action, value) do |tracking_data|
track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data|
::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data)
end
end
def frontend_experimentation_tracking_data(experiment_key, action, value = nil)
def frontend_experimentation_tracking_data(experiment_key, action, value = nil, subject: nil)
return if dnt_enabled?
track_experiment_event_for(experiment_key, action, value) do |tracking_data|
track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data|
gon.push(tracking_data: tracking_data)
end
end
def record_experiment_user(experiment_key)
return if dnt_enabled?
return unless Experimentation.enabled?(experiment_key) && current_user
return unless Experimentation.active?(experiment_key) && current_user
::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user)
::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: current_user), current_user)
end
def record_experiment_conversion_event(experiment_key)
return if dnt_enabled?
return unless current_user
return unless Experimentation.enabled?(experiment_key)
return unless Experimentation.active?(experiment_key)
::Experiment.record_conversion_event(experiment_key, current_user)
end
def experiment_tracking_category_and_group(experiment_key)
"#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group')}"
def experiment_tracking_category_and_group(experiment_key, subject: nil)
"#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group', subject: subject)}"
end
private
......@@ -89,40 +90,41 @@ module Gitlab
cookies.signed[:experimentation_subject_id]
end
def experimentation_subject_index(experiment_key)
def fallback_experimentation_subject_index(experiment_key)
return if experimentation_subject_id.blank?
if Experimentation.experiment(experiment_key).use_backwards_compatible_subject_index
experimentation_subject_id.delete('-').hex % 100
if Experimentation.get_experiment(experiment_key).use_backwards_compatible_subject_index
experimentation_subject_id.delete('-')
else
Zlib.crc32("#{experiment_key}#{experimentation_subject_id}") % 100
experimentation_subject_id
end
end
def track_experiment_event_for(experiment_key, action, value)
return unless Experimentation.enabled?(experiment_key)
def track_experiment_event_for(experiment_key, action, value, subject: nil)
return unless Experimentation.active?(experiment_key)
yield experimentation_tracking_data(experiment_key, action, value)
yield experimentation_tracking_data(experiment_key, action, value, subject: subject)
end
def experimentation_tracking_data(experiment_key, action, value)
def experimentation_tracking_data(experiment_key, action, value, subject: nil)
{
category: tracking_category(experiment_key),
action: action,
property: tracking_group(experiment_key, "_group"),
label: experimentation_subject_id,
property: tracking_group(experiment_key, "_group", subject: subject),
label: tracking_label(subject),
value: value
}.compact
end
def tracking_category(experiment_key)
Experimentation.experiment(experiment_key).tracking_category
Experimentation.get_experiment(experiment_key).tracking_category
end
def tracking_group(experiment_key, suffix = nil)
return unless Experimentation.enabled?(experiment_key)
def tracking_group(experiment_key, suffix = nil, subject: nil)
return unless Experimentation.active?(experiment_key)
group = experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL
subject ||= fallback_experimentation_subject_index(experiment_key)
group = experiment_enabled?(experiment_key, subject: subject) ? GROUP_EXPERIMENTAL : GROUP_CONTROL
suffix ? "#{group}#{suffix}" : group
end
......@@ -130,6 +132,16 @@ module Gitlab
def forced_enabled?(experiment_key)
params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s
end
def tracking_label(subject)
return experimentation_subject_id if subject.blank?
if subject.respond_to?(:to_global_id)
Digest::MD5.hexdigest(subject.to_global_id.to_s)
else
Digest::MD5.hexdigest(subject.to_s)
end
end
end
end
end
......@@ -3,16 +3,17 @@
module Gitlab
module Experimentation
class Experiment
attr_reader :tracking_category, :use_backwards_compatible_subject_index
attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index
def initialize(key, **params)
@key = key
@tracking_category = params[:tracking_category]
@use_backwards_compatible_subject_index = params[:use_backwards_compatible_subject_index]
@experiment_percentage = Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet
end
def enabled?
def active?
::Gitlab.dev_env_or_com? && experiment_percentage > 0
end
......
......@@ -333,7 +333,7 @@ RSpec.describe GroupsController, factory_default: :keep do
context 'and the user is part of the control group' do
before do
stub_experiment_for_user(onboarding_issues: false)
stub_experiment_for_subject(onboarding_issues: false)
end
it 'tracks the event with the "created_namespace" action with the "control_group" property', :snowplow do
......@@ -350,7 +350,7 @@ RSpec.describe GroupsController, factory_default: :keep do
context 'and the user is part of the experimental group' do
before do
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
end
it 'tracks the event with the "created_namespace" action with the "experimental_group" property', :snowplow do
......
......@@ -25,7 +25,7 @@ RSpec.describe InvitesController, :snowplow do
shared_examples "tracks the 'accepted' event for the invitation reminders experiment" do
before do
stub_experiment(invitation_reminders: true)
allow(Gitlab::Experimentation).to receive(:enabled_for_attribute?).with(:invitation_reminders, member.invite_email).and_return(experimental_group)
stub_experiment_for_subject(invitation_reminders: experimental_group)
end
context 'when in the control group' do
......
......@@ -18,7 +18,7 @@ RSpec.describe Projects::JobsController, :clean_gitlab_redis_shared_state do
describe 'pushing tracking_data to Gon' do
before do
stub_experiment(jobs_empty_state: experiment_active)
stub_experiment_for_user(jobs_empty_state: in_experiment_group)
stub_experiment_for_subject(jobs_empty_state: in_experiment_group)
get_index
end
......
......@@ -19,7 +19,7 @@ RSpec.describe Registrations::ExperienceLevelsController do
context 'with an authenticated user' do
before do
sign_in(user)
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
end
it { is_expected.to have_gitlab_http_status(:ok) }
......@@ -28,7 +28,7 @@ RSpec.describe Registrations::ExperienceLevelsController do
context 'when not part of the onboarding issues experiment' do
before do
stub_experiment_for_user(onboarding_issues: false)
stub_experiment_for_subject(onboarding_issues: false)
end
it { is_expected.to have_gitlab_http_status(:not_found) }
......@@ -47,12 +47,12 @@ RSpec.describe Registrations::ExperienceLevelsController do
context 'with an authenticated user' do
before do
sign_in(user)
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
end
context 'when not part of the onboarding issues experiment' do
before do
stub_experiment_for_user(onboarding_issues: false)
stub_experiment_for_subject(onboarding_issues: false)
end
it { is_expected.to have_gitlab_http_status(:not_found) }
......@@ -90,7 +90,7 @@ RSpec.describe Registrations::ExperienceLevelsController do
let(:issues_board) { build(:board, id: 123, project: project) }
before do
stub_experiment_for_user(
stub_experiment_for_subject(
onboarding_issues: true,
default_to_issues_board: default_to_issues_board_xp?
)
......
......@@ -125,7 +125,7 @@ RSpec.describe RootController do
context 'when experiment is enabled' do
before do
stub_experiment_for_user(customize_homepage: true)
stub_experiment_for_subject(customize_homepage: true)
end
it 'renders the default dashboard' do
......
......@@ -28,7 +28,7 @@ RSpec.describe 'Jobs', :clean_gitlab_redis_shared_state do
context 'with no jobs' do
before do
stub_experiment(jobs_empty_state: experiment_active)
stub_experiment_for_user(jobs_empty_state: in_experiment_group)
stub_experiment_for_subject(jobs_empty_state: in_experiment_group)
visit project_jobs_path(project)
end
......
......@@ -9,7 +9,7 @@ RSpec.describe 'Experience level screen' do
before do
group.add_owner(user)
gitlab_sign_in(user)
stub_experiment_for_user(onboarding_issues: true)
stub_experiment_for_subject(onboarding_issues: true)
visit users_sign_up_experience_level_path(namespace_path: group.to_param)
end
......
......@@ -75,29 +75,24 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
describe '#push_frontend_experiment' do
it 'pushes an experiment to the frontend' do
gon = instance_double('gon')
experiments = { experiments: { 'myExperiment' => true } }
stub_experiment_for_user(my_experiment: true)
stub_experiment_for_subject(my_experiment: true)
allow(controller).to receive(:gon).and_return(gon)
expect(gon).to receive(:push).with(experiments, true)
expect(gon).to receive(:push).with({ experiments: { 'myExperiment' => true } }, true)
controller.push_frontend_experiment(:my_experiment)
end
end
describe '#experiment_enabled?' do
def check_experiment(exp_key = :test_experiment)
controller.experiment_enabled?(exp_key)
def check_experiment(exp_key = :test_experiment, subject = nil)
controller.experiment_enabled?(exp_key, subject: subject)
end
subject { check_experiment }
context 'cookie is not present' do
it 'calls Gitlab::Experimentation.enabled_for_value? with the name of the experiment and an experimentation_subject_index of nil' do
expect(Gitlab::Experimentation).to receive(:enabled_for_value?).with(:test_experiment, nil)
check_experiment
end
it { is_expected.to eq(false) }
end
context 'cookie is present' do
......@@ -109,37 +104,56 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
end
where(:experiment_key, :index_value) do
:test_experiment | 40 # Zlib.crc32('test_experimentabcd-1234') % 100 = 40
:backwards_compatible_test_experiment | 76 # 'abcd1234'.hex % 100 = 76
:test_experiment | 'abcd-1234'
:backwards_compatible_test_experiment | 'abcd1234'
end
with_them do
it 'calls Gitlab::Experimentation.enabled_for_value? with the name of the experiment and the calculated experimentation_subject_index based on the uuid' do
expect(Gitlab::Experimentation).to receive(:enabled_for_value?).with(experiment_key, index_value)
it 'calls Gitlab::Experimentation.in_experiment_group?? with the name of the experiment and the calculated experimentation_subject_index based on the uuid' do
expect(Gitlab::Experimentation).to receive(:in_experiment_group?).with(experiment_key, subject: index_value)
check_experiment(experiment_key)
end
end
end
it 'returns true when DNT: 0 is set in the request' do
allow(Gitlab::Experimentation).to receive(:enabled_for_value?) { true }
controller.request.headers['DNT'] = '0'
context 'when subject is given' do
let(:user) { build(:user) }
is_expected.to be_truthy
it 'uses the subject' do
expect(Gitlab::Experimentation).to receive(:in_experiment_group?).with(:test_experiment, subject: user)
check_experiment(:test_experiment, user)
end
end
end
it 'returns false when DNT: 1 is set in the request' do
allow(Gitlab::Experimentation).to receive(:enabled_for_value?) { true }
controller.request.headers['DNT'] = '1'
context 'do not track' do
before do
allow(Gitlab::Experimentation).to receive(:in_experiment_group?) { true }
end
is_expected.to be_falsy
context 'when do not track is disabled' do
before do
controller.request.headers['DNT'] = '0'
end
it { is_expected.to eq(true) }
end
context 'when do not track is enabled' do
before do
controller.request.headers['DNT'] = '1'
end
it { is_expected.to eq(false) }
end
end
describe 'URL parameter to force enable experiment' do
context 'URL parameter to force enable experiment' do
it 'returns true unconditionally' do
get :index, params: { force_experiment: :test_experiment }
is_expected.to be_truthy
is_expected.to eq(true)
end
end
end
......@@ -152,7 +166,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
context 'the user is part of the experimental group' do
before do
stub_experiment_for_user(test_experiment: true)
stub_experiment_for_subject(test_experiment: true)
end
it 'tracks the event with the right parameters' do
......@@ -169,7 +183,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
context 'the user is part of the control group' do
before do
stub_experiment_for_user(test_experiment: false)
stub_experiment_for_subject(test_experiment: false)
end
it 'tracks the event with the right parameters' do
......@@ -212,6 +226,59 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
expect_no_snowplow_event
end
end
context 'subject is provided' do
before do
stub_experiment_for_subject(test_experiment: false)
end
it "provides the subject's hashed global_id as label" do
experiment_subject = double(:subject, to_global_id: 'abc')
controller.track_experiment_event(:test_experiment, 'start', 1, subject: experiment_subject)
expect_snowplow_event(
category: 'Team',
action: 'start',
property: 'control_group',
value: 1,
label: Digest::MD5.hexdigest('abc')
)
end
it "provides the subject's hashed string representation as label" do
experiment_subject = 'somestring'
controller.track_experiment_event(:test_experiment, 'start', 1, subject: experiment_subject)
expect_snowplow_event(
category: 'Team',
action: 'start',
property: 'control_group',
value: 1,
label: Digest::MD5.hexdigest('somestring')
)
end
end
context 'no subject is provided but cookie is set' do
before do
get :index
stub_experiment_for_subject(test_experiment: false)
end
it 'uses the experimentation_subject_id as fallback' do
controller.track_experiment_event(:test_experiment, 'start', 1)
expect_snowplow_event(
category: 'Team',
action: 'start',
property: 'control_group',
value: 1,
label: cookies.permanent.signed[:experimentation_subject_id]
)
end
end
end
context 'when the experiment is disabled' do
......@@ -235,7 +302,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
context 'the user is part of the experimental group' do
before do
stub_experiment_for_user(test_experiment: true)
stub_experiment_for_subject(test_experiment: true)
end
it 'pushes the right parameters to gon' do
......@@ -253,9 +320,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
context 'the user is part of the control group' do
before do
allow_next_instance_of(described_class) do |instance|
allow(instance).to receive(:experiment_enabled?).with(:test_experiment).and_return(false)
end
stub_experiment_for_subject(test_experiment: false)
end
it 'pushes the right parameters to gon' do
......@@ -308,7 +373,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
it 'does not push data to gon' do
controller.frontend_experimentation_tracking_data(:test_experiment, 'start')
expect(Gon.method_defined?(:tracking_data)).to be_falsey
expect(Gon.method_defined?(:tracking_data)).to eq(false)
end
end
end
......@@ -319,7 +384,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
end
it 'does not push data to gon' do
expect(Gon.method_defined?(:tracking_data)).to be_falsey
expect(Gon.method_defined?(:tracking_data)).to eq(false)
controller.track_experiment_event(:test_experiment, 'start')
end
end
......@@ -336,7 +401,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
context 'the user is part of the experimental group' do
before do
stub_experiment_for_user(test_experiment: true)
stub_experiment_for_subject(test_experiment: true)
end
it 'calls add_user on the Experiment model' do
......@@ -348,9 +413,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
context 'the user is part of the control group' do
before do
allow_next_instance_of(described_class) do |instance|
allow(instance).to receive(:experiment_enabled?).with(:test_experiment).and_return(false)
end
stub_experiment_for_subject(test_experiment: false)
end
it 'calls add_user on the Experiment model' do
......@@ -395,6 +458,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
context 'is disabled' do
before do
request.headers['DNT'] = '0'
stub_experiment_for_subject(test_experiment: false)
end
it 'calls add_user on the Experiment model' do
......@@ -475,7 +539,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do
it 'returns a string with the experiment tracking category & group joined with a ":"' do
expect(controller).to receive(:tracking_category).with(experiment_key).and_return('Experiment::Category')
expect(controller).to receive(:tracking_group).with(experiment_key, '_group').and_return('experimental_group')
expect(controller).to receive(:tracking_group).with(experiment_key, '_group', subject: nil).and_return('experimental_group')
expect(subject).to eq('Experiment::Category:experimental_group')
end
......
......@@ -20,14 +20,14 @@ RSpec.describe Gitlab::Experimentation::Experiment do
subject(:experiment) { described_class.new(:experiment_key, **params) }
describe '#enabled?' do
describe '#active?' do
before do
allow(Gitlab).to receive(:dev_env_or_com?).and_return(on_gitlab_com)
end
subject { experiment.enabled? }
subject { experiment.active? }
where(:on_gitlab_com, :percentage, :is_enabled) do
where(:on_gitlab_com, :percentage, :is_active) do
true | 0 | false
true | 10 | true
false | 0 | false
......@@ -35,7 +35,7 @@ RSpec.describe Gitlab::Experimentation::Experiment do
end
with_them do
it { is_expected.to eq(is_enabled) }
it { is_expected.to eq(is_active) }
end
end
......
......@@ -28,7 +28,7 @@ RSpec.describe Gitlab::Experimentation::EXPERIMENTS do
end
end
RSpec.describe Gitlab::Experimentation, :snowplow do
RSpec.describe Gitlab::Experimentation do
before do
stub_const('Gitlab::Experimentation::EXPERIMENTS', {
backwards_compatible_test_experiment: {
......@@ -47,92 +47,131 @@ RSpec.describe Gitlab::Experimentation, :snowplow do
let(:enabled_percentage) { 10 }
describe '.enabled?' do
subject { described_class.enabled?(:test_experiment) }
describe '.get_experiment' do
subject { described_class.get_experiment(:test_experiment) }
context 'feature toggle is enabled and we are selected' do
it { is_expected.to be_truthy }
context 'returns experiment' do
it { is_expected.to be_instance_of(Gitlab::Experimentation::Experiment) }
end
context 'experiment is not defined' do
subject { described_class.get_experiment(:missing_experiment) }
it { is_expected.to be_nil }
end
end
describe '.active?' do
subject { described_class.active?(:test_experiment) }
context 'feature toggle is enabled' do
it { is_expected.to eq(true) }
end
describe 'experiment is not defined' do
it 'returns false' do
expect(described_class.enabled?(:missing_experiment)).to be_falsey
expect(described_class.active?(:missing_experiment)).to eq(false)
end
end
describe 'experiment is disabled' do
let(:enabled_percentage) { 0 }
it { is_expected.to be_falsey }
it { is_expected.to eq(false) }
end
end
describe '.enabled_for_value?' do
subject { described_class.enabled_for_value?(:test_experiment, experimentation_subject_index) }
describe '.in_experiment_group?' do
context 'with new index calculation' do
let(:enabled_percentage) { 50 }
let(:experiment_subject) { 'z' } # Zlib.crc32('test_experimentz') % 100 = 33
let(:experimentation_subject_index) { 9 }
context 'experiment is disabled' do
before do
allow(described_class).to receive(:enabled?).and_return(false)
end
subject { described_class.in_experiment_group?(:test_experiment, subject: experiment_subject) }
it { is_expected.to be_falsey }
end
context 'when experiment is active' do
context 'when subject is part of the experiment' do
it { is_expected.to eq(true) }
end
context 'experiment is enabled' do
before do
allow(described_class).to receive(:enabled?).and_return(true)
end
context 'when subject is not part of the experiment' do
let(:experiment_subject) { 'a' } # Zlib.crc32('test_experimenta') % 100 = 61
it { is_expected.to be_truthy }
it { is_expected.to eq(false) }
end
describe 'experimentation_subject_index' do
context 'experimentation_subject_index is not set' do
let(:experimentation_subject_index) { nil }
context 'when subject has a global_id' do
let(:experiment_subject) { double(:subject, to_global_id: 'z') }
it { is_expected.to be_falsey }
it { is_expected.to eq(true) }
end
context 'experimentation_subject_index is an empty string' do
let(:experimentation_subject_index) { '' }
context 'when subject is nil' do
let(:experiment_subject) { nil }
it { is_expected.to be_falsey }
it { is_expected.to eq(false) }
end
context 'experimentation_subject_index outside enabled ratio' do
let(:experimentation_subject_index) { 11 }
context 'when subject is an empty string' do
let(:experiment_subject) { '' }
it { is_expected.to be_falsey }
it { is_expected.to eq(false) }
end
end
context 'when experiment is not active' do
before do
allow(described_class).to receive(:active?).and_return(false)
end
it { is_expected.to eq(false) }
end
end
end
describe '.enabled_for_attribute?' do
subject { described_class.enabled_for_attribute?(:test_experiment, attribute) }
context 'with backwards compatible index calculation' do
let(:experiment_subject) { 'abcd' } # Digest::SHA1.hexdigest('abcd').hex % 100 = 7
let(:attribute) { 'abcd' } # Digest::SHA1.hexdigest('abcd').hex % 100 = 7
subject { described_class.in_experiment_group?(:backwards_compatible_test_experiment, subject: experiment_subject) }
context 'experiment is disabled' do
before do
allow(described_class).to receive(:enabled?).and_return(false)
end
context 'when experiment is active' do
before do
allow(described_class).to receive(:active?).and_return(true)
end
it { is_expected.to be false }
end
context 'when subject is part of the experiment' do
it { is_expected.to eq(true) }
end
context 'experiment is enabled' do
before do
allow(described_class).to receive(:enabled?).and_return(true)
end
context 'when subject is not part of the experiment' do
let(:experiment_subject) { 'abc' } # Digest::SHA1.hexdigest('abc').hex % 100 = 17
it { is_expected.to be true }
it { is_expected.to eq(false) }
end
context 'when subject has a global_id' do
let(:experiment_subject) { double(:subject, to_global_id: 'abcd') }
it { is_expected.to eq(true) }
end
context 'outside enabled ratio' do
let(:attribute) { 'abc' } # Digest::SHA1.hexdigest('abc').hex % 100 = 17
context 'when subject is nil' do
let(:experiment_subject) { nil }
it { is_expected.to eq(false) }
end
context 'when subject is an empty string' do
let(:experiment_subject) { '' }
it { is_expected.to eq(false) }
end
end
context 'when experiment is not active' do
before do
allow(described_class).to receive(:active?).and_return(false)
end
it { is_expected.to be false }
it { is_expected.to eq(false) }
end
end
end
......
......@@ -1453,7 +1453,7 @@ RSpec.describe Notify do
shared_examples "tracks the 'sent' event for the invitation reminders experiment" do
before do
stub_experiment(invitation_reminders: true)
allow(Gitlab::Experimentation).to receive(:enabled_for_attribute?).with(:invitation_reminders, group_member.invite_email).and_return(experimental_group)
allow(Gitlab::Experimentation).to receive(:in_experiment_group?).with(:invitation_reminders, subject: group_member.invite_email).and_return(experimental_group)
end
it "tracks the 'sent' event", :snowplow do
......
......@@ -11,7 +11,7 @@ RSpec.describe Members::InvitationReminderEmailService do
context 'when the experiment is disabled' do
before do
allow(Gitlab::Experimentation).to receive(:enabled_for_attribute?).and_return(false)
allow(Gitlab::Experimentation).to receive(:in_experiment_group?).and_return(false)
invitation.expires_at = frozen_time + 2.days
end
......@@ -26,7 +26,7 @@ RSpec.describe Members::InvitationReminderEmailService do
context 'when the experiment is enabled' do
before do
allow(Gitlab::Experimentation).to receive(:enabled_for_attribute?).and_return(true)
allow(Gitlab::Experimentation).to receive(:in_experiment_group?).and_return(true)
invitation.expires_at = frozen_time + expires_at_days.days if expires_at_days
end
......
......@@ -3,15 +3,15 @@
module StubExperiments
# Stub Experiment with `key: true/false`
#
# @param [Hash] experiment where key is feature name and value is boolean whether enabled or not.
# @param [Hash] experiment where key is feature name and value is boolean whether active or not.
#
# Examples
# - `stub_experiment(signup_flow: false)` ... Disable `signup_flow` experiment globally.
# - `stub_experiment(signup_flow: false)` ... Disables `signup_flow` experiment.
def stub_experiment(experiments)
allow(Gitlab::Experimentation).to receive(:enabled?).and_call_original
allow(Gitlab::Experimentation).to receive(:active?).and_call_original
experiments.each do |experiment_key, enabled|
allow(Gitlab::Experimentation).to receive(:enabled?).with(experiment_key) { enabled }
allow(Gitlab::Experimentation).to receive(:active?).with(experiment_key) { enabled }
end
end
......@@ -20,12 +20,12 @@ module StubExperiments
# @param [Hash] experiment where key is feature name and value is boolean whether enabled or not.
#
# Examples
# - `stub_experiment_for_user(signup_flow: false)` ... Disable `signup_flow` experiment for user.
def stub_experiment_for_user(experiments)
allow(Gitlab::Experimentation).to receive(:enabled_for_value?).and_call_original
# - `stub_experiment_for_subject(signup_flow: false)` ... Disable `signup_flow` experiment for user.
def stub_experiment_for_subject(experiments)
allow(Gitlab::Experimentation).to receive(:in_experiment_group?).and_call_original
experiments.each do |experiment_key, enabled|
allow(Gitlab::Experimentation).to receive(:enabled_for_value?).with(experiment_key, anything) { enabled }
allow(Gitlab::Experimentation).to receive(:in_experiment_group?).with(experiment_key, anything) { enabled }
end
end
end
......@@ -3,7 +3,7 @@
RSpec.shared_examples 'issuable invite members experiments' do
context 'when invite_members_version_a experiment is enabled' do
before do
stub_experiment_for_user(invite_members_version_a: true)
stub_experiment_for_subject(invite_members_version_a: true)
end
it 'shows a link for inviting members and follows through to the members page' do
......@@ -28,7 +28,7 @@ RSpec.shared_examples 'issuable invite members experiments' do
context 'when invite_members_version_b experiment is enabled' do
before do
stub_experiment_for_user(invite_members_version_b: true)
stub_experiment_for_subject(invite_members_version_b: true)
end
it 'shows a link for inviting members and follows through to modal' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment