Commit 5d9d5e60 authored by Jeremy Jackson's avatar Jeremy Jackson Committed by Mayra Cabrera

Migrates Snowplow backend from EE to CE

This introduces several changes, but these are all just ported from the
EE project.
parent 7f9c653e
...@@ -297,6 +297,9 @@ gem 'batch-loader', '~> 1.4.0' ...@@ -297,6 +297,9 @@ gem 'batch-loader', '~> 1.4.0'
# Perf bar # Perf bar
gem 'peek', '~> 1.0.1' gem 'peek', '~> 1.0.1'
# Snowplow events tracking
gem 'snowplow-tracker', '~> 0.6.1'
# Memory benchmarks # Memory benchmarks
gem 'derailed_benchmarks', require: false gem 'derailed_benchmarks', require: false
......
...@@ -152,6 +152,7 @@ GEM ...@@ -152,6 +152,7 @@ GEM
concurrent-ruby-ext (1.1.5) concurrent-ruby-ext (1.1.5)
concurrent-ruby (= 1.1.5) concurrent-ruby (= 1.1.5)
connection_pool (2.2.2) connection_pool (2.2.2)
contracts (0.11.0)
crack (0.4.3) crack (0.4.3)
safe_yaml (~> 1.0.0) safe_yaml (~> 1.0.0)
crass (1.0.4) crass (1.0.4)
...@@ -901,6 +902,8 @@ GEM ...@@ -901,6 +902,8 @@ GEM
simplecov-html (~> 0.10.0) simplecov-html (~> 0.10.0)
simplecov-html (0.10.2) simplecov-html (0.10.2)
slack-notifier (1.5.1) slack-notifier (1.5.1)
snowplow-tracker (0.6.1)
contracts (~> 0.7, <= 0.11)
spring (2.0.2) spring (2.0.2)
activesupport (>= 4.2) activesupport (>= 4.2)
spring-commands-rspec (1.0.4) spring-commands-rspec (1.0.4)
...@@ -1229,6 +1232,7 @@ DEPENDENCIES ...@@ -1229,6 +1232,7 @@ DEPENDENCIES
simple_po_parser (~> 1.1.2) simple_po_parser (~> 1.1.2)
simplecov (~> 0.16.1) simplecov (~> 0.16.1)
slack-notifier (~> 1.5.1) slack-notifier (~> 1.5.1)
snowplow-tracker (~> 0.6.1)
spring (~> 2.0.0) spring (~> 2.0.0)
spring-commands-rspec (~> 1.0.4) spring-commands-rspec (~> 1.0.4)
sprockets (~> 3.7.0) sprockets (~> 3.7.0)
......
...@@ -270,7 +270,11 @@ module ApplicationSettingsHelper ...@@ -270,7 +270,11 @@ module ApplicationSettingsHelper
:diff_max_patch_bytes, :diff_max_patch_bytes,
:commit_email_hostname, :commit_email_hostname,
:protected_ci_variables, :protected_ci_variables,
:local_markdown_version :local_markdown_version,
:snowplow_collector_hostname,
:snowplow_cookie_domain,
:snowplow_enabled,
:snowplow_site_id
] ]
end end
......
...@@ -2,6 +2,21 @@ ...@@ -2,6 +2,21 @@
module TrackingHelper module TrackingHelper
def tracking_attrs(label, event, property) def tracking_attrs(label, event, property)
{} # CE has no tracking features return {} unless tracking_enabled?
{
data: {
track_label: label,
track_event: event,
track_property: property
}
}
end
private
def tracking_enabled?
Rails.env.production? &&
::Gitlab::CurrentSettings.snowplow_enabled?
end end
end end
...@@ -99,6 +99,11 @@ class ApplicationSetting < ApplicationRecord ...@@ -99,6 +99,11 @@ class ApplicationSetting < ApplicationRecord
presence: true, presence: true,
if: :plantuml_enabled if: :plantuml_enabled
validates :snowplow_collector_hostname,
presence: true,
hostname: true,
if: :snowplow_enabled
validates :max_attachment_size, validates :max_attachment_size,
presence: true, presence: true,
numericality: { only_integer: true, greater_than: 0 } numericality: { only_integer: true, greater_than: 0 }
......
...@@ -97,6 +97,10 @@ module ApplicationSettingImplementation ...@@ -97,6 +97,10 @@ module ApplicationSettingImplementation
usage_stats_set_by_user_id: nil, usage_stats_set_by_user_id: nil,
diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES,
commit_email_hostname: default_commit_email_hostname, commit_email_hostname: default_commit_email_hostname,
snowplow_collector_hostname: nil,
snowplow_cookie_domain: nil,
snowplow_enabled: false,
snowplow_site_id: nil,
protected_ci_variables: false, protected_ci_variables: false,
local_markdown_version: 0, local_markdown_version: 0,
outbound_local_requests_whitelist: [], outbound_local_requests_whitelist: [],
......
- expanded = true if !@application_setting.valid? && @application_setting.errors.any? { |k| k.to_s.start_with?('snowplow_') }
%section.settings.as-snowplow.no-animate#js-snowplow-settings{ class: ('expanded' if expanded) }
.settings-header
%h4
= _('Snowplow')
%button.btn.btn-default.js-settings-toggle{ type: 'button' }
= expanded ? _('Collapse') : _('Expand')
%p
= _('Configure the %{link} integration.').html_safe % { link: link_to('Snowplow', 'https://snowplowanalytics.com/', target: '_blank') }
.settings-content
= form_for @application_setting, url: integrations_admin_application_settings_path, html: { class: 'fieldset-form' } do |f|
= form_errors(@application_setting)
%fieldset
.form-group
.form-check
= f.check_box :snowplow_enabled, class: 'form-check-input'
= f.label :snowplow_enabled, _('Enable snowplow tracking'), class: 'form-check-label'
.form-group
= f.label :snowplow_collector_hostname, _('Collector hostname'), class: 'label-light'
= f.text_field :snowplow_collector_hostname, class: 'form-control', placeholder: 'snowplow.example.com'
.form-group
= f.label :snowplow_site_id, _('Site ID'), class: 'label-light'
= f.text_field :snowplow_site_id, class: 'form-control'
.form-group
= f.label :snowplow_cookie_domain, _('Cookie domain'), class: 'label-light'
= f.text_field :snowplow_cookie_domain, class: 'form-control'
= f.submit _('Save changes'), class: 'btn btn-success'
- return unless Gitlab::CurrentSettings.snowplow_enabled?
= javascript_tag nonce: true do
:plain
;(function(p,l,o,w,i,n,g){if(!p[i]){p.GlobalSnowplowNamespace=p.GlobalSnowplowNamespace||[];
p.GlobalSnowplowNamespace.push(i);p[i]=function(){(p[i].q=p[i].q||[]).push(arguments)
};p[i].q=p[i].q||[];n=l.createElement(o);g=l.getElementsByTagName(o)[0];n.async=1;
n.src=w;g.parentNode.insertBefore(n,g)}}(window,document,"script","#{asset_url('snowplow/sp.js')}","snowplow"));
window.snowplow('newTracker', '#{Gitlab::SnowplowTracker::NAMESPACE}', '#{Gitlab::CurrentSettings.snowplow_collector_hostname}', {
appId: '#{Gitlab::CurrentSettings.snowplow_site_id}',
cookieDomain: '#{Gitlab::CurrentSettings.snowplow_cookie_domain}',
userFingerprint: false,
respectDoNotTrack: true,
forceSecureTracker: true,
post: true,
contexts: { webPage: true },
stateStorageStrategy: "localStorage"
});
window.snowplow('enableActivityTracking', 30, 30);
window.snowplow('trackPageView');
- return unless Feature.enabled?(:additional_snowplow_tracking, @group)
= javascript_tag nonce: true do
:plain
window.snowplow('enableFormTracking');
window.snowplow('enableLinkClickTracking');
...@@ -321,4 +321,8 @@ are listed in the descriptions of the relevant settings. ...@@ -321,4 +321,8 @@ are listed in the descriptions of the relevant settings.
| `user_show_add_ssh_key_message` | boolean | no | When set to `false` disable the "You won't be able to pull or push project code via SSH" warning shown to users with no uploaded SSH key. | | `user_show_add_ssh_key_message` | boolean | no | When set to `false` disable the "You won't be able to pull or push project code via SSH" warning shown to users with no uploaded SSH key. |
| `version_check_enabled` | boolean | no | Let GitLab inform you when an update is available. | | `version_check_enabled` | boolean | no | Let GitLab inform you when an update is available. |
| `local_markdown_version` | integer | no | Increase this value when any cached markdown should be invalidated. | | `local_markdown_version` | integer | no | Increase this value when any cached markdown should be invalidated. |
| `snowplow_enabled` | boolean | no | Enable snowplow tracking. |
| `snowplow_collector_hostname` | string | required by: `snowplow_enabled` | The Snowplow collector hostname. (e.g. `snowplow.trx.gitlab.net`) |
| `snowplow_site_id` | string | no | The Snowplow site name / application id. (e.g. `gitlab`) |
| `snowplow_cookie_domain` | string | no | The Snowplow cookie domain. (e.g. `.gitlab.com`) |
| `geo_node_allowed_ips` | string | yes | **(PREMIUM)** Comma-separated list of IPs and CIDRs of allowed secondary nodes. For example, `1.1.1.1, 2.2.2.0/24`. | | `geo_node_allowed_ips` | string | yes | **(PREMIUM)** Comma-separated list of IPs and CIDRs of allowed secondary nodes. For example, `1.1.1.1, 2.2.2.0/24`. |
...@@ -125,6 +125,12 @@ module API ...@@ -125,6 +125,12 @@ module API
optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins'
optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated" optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated"
optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5 optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5
optional :snowplow_enabled, type: Grape::API::Boolean, desc: 'Enable Snowplow tracking'
given snowplow_enabled: ->(val) { val } do
requires :snowplow_collector_hostname, type: String, desc: 'The Snowplow collector hostname'
optional :snowplow_cookie_domain, type: String, desc: 'The Snowplow cookie domain'
optional :snowplow_site_id, type: String, desc: 'The Snowplow site name / application ic'
end
ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type|
optional :"#{type}_key_restriction", optional :"#{type}_key_restriction",
......
# frozen_string_literal: true
require 'snowplow-tracker'
module Gitlab
module SnowplowTracker
NAMESPACE = 'cf'
class << self
def track_event(category, action, label: nil, property: nil, value: nil, context: nil)
tracker&.track_struct_event(category, action, label, property, value, context, Time.now.to_i)
end
private
def tracker
return unless enabled?
@tracker ||= ::SnowplowTracker::Tracker.new(emitter, subject, NAMESPACE, Gitlab::CurrentSettings.snowplow_site_id)
end
def subject
::SnowplowTracker::Subject.new
end
def emitter
::SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname)
end
def enabled?
Gitlab::CurrentSettings.snowplow_enabled?
end
end
end
end
...@@ -2953,6 +2953,9 @@ msgstr "" ...@@ -2953,6 +2953,9 @@ msgstr ""
msgid "Collapse sidebar" msgid "Collapse sidebar"
msgstr "" msgstr ""
msgid "Collector hostname"
msgstr ""
msgid "ComboSearch is not defined" msgid "ComboSearch is not defined"
msgstr "" msgstr ""
...@@ -3120,6 +3123,9 @@ msgstr "" ...@@ -3120,6 +3123,9 @@ msgstr ""
msgid "Configure storage path settings." msgid "Configure storage path settings."
msgstr "" msgstr ""
msgid "Configure the %{link} integration."
msgstr ""
msgid "Configure the way a user creates a new account." msgid "Configure the way a user creates a new account."
msgstr "" msgstr ""
...@@ -3261,6 +3267,9 @@ msgstr "" ...@@ -3261,6 +3267,9 @@ msgstr ""
msgid "ConvDev Index" msgid "ConvDev Index"
msgstr "" msgstr ""
msgid "Cookie domain"
msgstr ""
msgid "Copied" msgid "Copied"
msgstr "" msgstr ""
...@@ -4253,6 +4262,9 @@ msgstr "" ...@@ -4253,6 +4262,9 @@ msgstr ""
msgid "Enable shared Runners" msgid "Enable shared Runners"
msgstr "" msgstr ""
msgid "Enable snowplow tracking"
msgstr ""
msgid "Enable two-factor authentication" msgid "Enable two-factor authentication"
msgstr "" msgstr ""
...@@ -10286,6 +10298,9 @@ msgstr "" ...@@ -10286,6 +10298,9 @@ msgstr ""
msgid "Similar issues" msgid "Similar issues"
msgstr "" msgstr ""
msgid "Site ID"
msgstr ""
msgid "Size and domain settings for static websites" msgid "Size and domain settings for static websites"
msgstr "" msgstr ""
...@@ -10316,6 +10331,9 @@ msgstr "" ...@@ -10316,6 +10331,9 @@ msgstr ""
msgid "SnippetsEmptyState|They can be either public or private." msgid "SnippetsEmptyState|They can be either public or private."
msgstr "" msgstr ""
msgid "Snowplow"
msgstr ""
msgid "Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead." msgid "Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead."
msgstr "" msgstr ""
......
...@@ -4,8 +4,32 @@ require 'spec_helper' ...@@ -4,8 +4,32 @@ require 'spec_helper'
describe TrackingHelper do describe TrackingHelper do
describe '#tracking_attrs' do describe '#tracking_attrs' do
it 'returns an empty hash' do using RSpec::Parameterized::TableSyntax
expect(helper.tracking_attrs('a', 'b', 'c')).to eq({})
let(:input) { %w(a b c) }
let(:results) do
{
no_data: {},
with_data: { data: { track_label: 'a', track_event: 'b', track_property: 'c' } }
}
end
where(:snowplow_enabled, :environment, :result) do
true | 'production' | :with_data
false | 'production' | :no_data
true | 'development' | :no_data
false | 'development' | :no_data
true | 'test' | :no_data
false | 'test' | :no_data
end
with_them do
it 'returns a hash' do
stub_application_setting(snowplow_enabled: snowplow_enabled)
allow(Rails).to receive(:env).and_return(environment.inquiry)
expect(helper.tracking_attrs(*input)).to eq(results[result])
end
end end
end end
end end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::SnowplowTracker do
let(:timestamp) { Time.utc(2017, 3, 22) }
around do |example|
Timecop.freeze(timestamp) { example.run }
end
subject { described_class.track_event('epics', 'action', property: 'what', value: 'doit') }
context '.track_event' do
context 'when Snowplow tracker is disabled' do
it 'does not track the event' do
expect(SnowplowTracker::Tracker).not_to receive(:new)
subject
end
end
context 'when Snowplow tracker is enabled' do
before do
stub_application_setting(snowplow_enabled: true)
stub_application_setting(snowplow_site_id: 'awesome gitlab')
stub_application_setting(snowplow_collector_hostname: 'url.com')
end
it 'tracks the event' do
tracker = double
expect(::SnowplowTracker::Tracker).to receive(:new)
.with(
an_instance_of(::SnowplowTracker::Emitter),
an_instance_of(::SnowplowTracker::Subject),
'cf', 'awesome gitlab'
).and_return(tracker)
expect(tracker).to receive(:track_struct_event)
.with('epics', 'action', nil, 'what', 'doit', nil, timestamp.to_i)
subject
end
end
end
end
...@@ -144,6 +144,7 @@ describe API::Settings, 'Settings' do ...@@ -144,6 +144,7 @@ describe API::Settings, 'Settings' do
external_auth_client_key_pass: "5iveL!fe" external_auth_client_key_pass: "5iveL!fe"
} }
end end
let(:attribute_names) { settings.keys.map(&:to_s) } let(:attribute_names) { settings.keys.map(&:to_s) }
it 'includes the attributes in the API' do it 'includes the attributes in the API' do
...@@ -165,6 +166,56 @@ describe API::Settings, 'Settings' do ...@@ -165,6 +166,56 @@ describe API::Settings, 'Settings' do
end end
end end
context "snowplow tracking settings" do
let(:settings) do
{
snowplow_collector_hostname: "snowplow.example.com",
snowplow_cookie_domain: ".example.com",
snowplow_enabled: true,
snowplow_site_id: "site_id"
}
end
let(:attribute_names) { settings.keys.map(&:to_s) }
it "includes the attributes in the API" do
get api("/application/settings", admin)
expect(response).to have_gitlab_http_status(200)
attribute_names.each do |attribute|
expect(json_response.keys).to include(attribute)
end
end
it "allows updating the settings" do
put api("/application/settings", admin), params: settings
expect(response).to have_gitlab_http_status(200)
settings.each do |attribute, value|
expect(ApplicationSetting.current.public_send(attribute)).to eq(value)
end
end
context "missing snowplow_collector_hostname value when snowplow_enabled is true" do
it "returns a blank parameter error message" do
put api("/application/settings", admin), params: { snowplow_enabled: true }
expect(response).to have_gitlab_http_status(400)
expect(json_response["error"]).to eq("snowplow_collector_hostname is missing")
end
it "handles validation errors" do
put api("/application/settings", admin), params: settings.merge({
snowplow_collector_hostname: nil
})
expect(response).to have_gitlab_http_status(400)
message = json_response["message"]
expect(message["snowplow_collector_hostname"]).to include("can't be blank")
end
end
end
context "missing plantuml_url value when plantuml_enabled is true" do context "missing plantuml_url value when plantuml_enabled is true" do
it "returns a blank parameter error message" do it "returns a blank parameter error message" do
put api("/application/settings", admin), params: { plantuml_enabled: true } put api("/application/settings", admin), params: { plantuml_enabled: true }
......
...@@ -70,6 +70,23 @@ describe 'layouts/_head' do ...@@ -70,6 +70,23 @@ describe 'layouts/_head' do
expect(rendered).to match('<link rel="stylesheet" media="all" href="/stylesheets/highlight/themes/solarised-light.css" />') expect(rendered).to match('<link rel="stylesheet" media="all" href="/stylesheets/highlight/themes/solarised-light.css" />')
end end
context 'when an asset_host is set and snowplow url is set' do
let(:asset_host) { 'http://test.host' }
before do
allow(ActionController::Base).to receive(:asset_host).and_return(asset_host)
allow(Gitlab::CurrentSettings).to receive(:snowplow_enabled?).and_return(true)
allow(Gitlab::CurrentSettings).to receive(:snowplow_collector_hostname).and_return('www.snow.plow')
end
it 'add a snowplow script tag with asset host' do
render
expect(rendered).to match('http://test.host/assets/snowplow/')
expect(rendered).to match('window.snowplow')
expect(rendered).to match('www.snow.plow')
end
end
def stub_helper_with_safe_string(method) def stub_helper_with_safe_string(method)
allow_any_instance_of(PageLayoutHelper).to receive(method) allow_any_instance_of(PageLayoutHelper).to receive(method)
.and_return(%q{foo" http-equiv="refresh}.html_safe) .and_return(%q{foo" http-equiv="refresh}.html_safe)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment