Merge branch 'product-analytics-ui' into 'master'

Add UI to Product Analytics feature

See merge request gitlab-org/gitlab!36693

app/controllers/projects/product_analytics_controller.rb

+# frozen_string_literal: true
+
+class Projects::ProductAnalyticsController < Projects::ApplicationController
+  before_action :feature_enabled!
+  before_action :authorize_read_product_analytics!
+  before_action :tracker_variables, only: [:setup, :test]
+
+  def index
+    @events = product_analytics_events.order_by_time.page(params[:page])
+  end
+
+  def setup
+  end
+
+  def test
+    @event = product_analytics_events.try(:first)
+  end
+
+  private
+
+  def product_analytics_events
+    @project.product_analytics_events
+  end
+
+  def tracker_variables
+    # We use project id as Snowplow appId
+    @project_id = @project.id.to_s
+
+    # Snowplow remembers values like appId and platform between reloads.
+    # That is why we have to rename the tracker with a random integer.
+    @random = rand(999999)
+
+    # Generate random platform every time a tracker is rendered.
+    @platform = %w(web mob app)[(@random % 3)]
+  end
+
+  def feature_enabled!
+    render_404 unless Feature.enabled?(:product_analytics, @project, default_enabled: false)
+  end
+end

app/helpers/product_analytics_helper.rb

+# frozen_string_literal: true
+
+module ProductAnalyticsHelper
+  def product_analytics_tracker_url
+    ProductAnalytics::Tracker::URL
+  end
+
+  def product_analytics_tracker_collector_url
+    ProductAnalytics::Tracker::COLLECTOR_URL
+  end
+end

app/helpers/projects_helper.rb

@@ -421,6 +421,10 @@ module ProjectsHelper
       nav_tabs << :operations
     end
 
+    if can_view_product_analytics?(current_user, project)
+      nav_tabs << :product_analytics
+    end
+
     tab_ability_map.each do |tab, ability|
       if can?(current_user, ability, project)
         nav_tabs << tab
@@ -479,6 +483,11 @@ module ProjectsHelper
     end
   end
 
+  def can_view_product_analytics?(current_user, project)
+    Feature.enabled?(:product_analytics, project) &&
+      can?(current_user, :read_product_analytics, project)
+  end
+
   def search_tab_ability_map
     @search_tab_ability_map ||= tab_ability_map.merge(
       blobs:          :download_code,
@@ -738,6 +747,7 @@ module ProjectsHelper
       user
       gcp
       logs
+      product_analytics
     ]
   end
 

app/models/product_analytics_event.rb

@@ -19,4 +19,8 @@ class ProductAnalyticsEvent < ApplicationRecord
   scope :timerange, ->(duration, today = Time.zone.today) {
     where('collector_tstamp BETWEEN ? AND ? ', today - duration + 1, today + 1)
   }
+
+  def as_json_wo_empty
+    as_json.compact
+  end
 end

app/models/project.rb

@@ -340,6 +340,10 @@ class Project < ApplicationRecord
   has_many :webide_pipelines, -> { webide_source }, class_name: 'Ci::Pipeline', inverse_of: :project
   has_many :reviews, inverse_of: :project
 
+  # Can be too many records. We need to implement delete_all in batches.
+  # Issue https://gitlab.com/gitlab-org/gitlab/-/issues/228637
+  has_many :product_analytics_events, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
   accepts_nested_attributes_for :project_setting, update_only: true

app/policies/project_policy.rb

@@ -264,6 +264,7 @@ class ProjectPolicy < BasePolicy
     enable :metrics_dashboard
     enable :read_confidential_issues
     enable :read_package
+    enable :read_product_analytics
   end
 
   # We define `:public_user_access` separately because there are cases in gitlab-ee

app/views/layouts/nav/sidebar/_project.html.haml

@@ -252,6 +252,12 @@
                   %span
                     = _('Error Tracking')
 
+            - if project_nav_tab?(:product_analytics)
+              = nav_link(controller: :product_analytics) do
+                = link_to project_product_analytics_path(@project), title: _('Product Analytics') do
+                  %span
+                    = _('Product Analytics')
+
             - if project_nav_tab? :serverless
               = nav_link(controller: :functions) do
                 = link_to project_serverless_functions_path(@project), title: _('Serverless') do

app/views/projects/product_analytics/_links.html.haml

+.mb-3
+  %ul.nav-links
+    = nav_link(path: 'product_analytics#index') do
+      = link_to _('Events'), project_product_analytics_path(@project)
+    = nav_link(path: 'product_analytics#test') do
+      = link_to _('Test'), test_project_product_analytics_path(@project)
+    = nav_link(path: 'product_analytics#setup') do
+      = link_to _('Setup'), setup_project_product_analytics_path(@project)

app/views/projects/product_analytics/_tracker.html.erb

+;(function(p,l,o,w,i,n,g){if(!p[i]){p.GlobalSnowplowNamespace=p.GlobalSnowplowNamespace||[];
+p.GlobalSnowplowNamespace.push(i);p[i]=function(){(p[i].q=p[i].q||[]).push(arguments)
+};p[i].q=p[i].q||[];n=l.createElement(o);g=l.getElementsByTagName(o)[0];n.async=1;
+n.src=w;g.parentNode.insertBefore(n,g)}}(window,document,"script","<%= product_analytics_tracker_url -%>","snowplow<%= @random -%>"));
+snowplow<%= @random -%>("newTracker", "sp", "<%= product_analytics_tracker_collector_url -%>", {
+  appId: "<%= @project_id -%>",
+  platform: "<%= @platform -%>",
+  eventMethod: "get"
+});
+snowplow<%= @random -%>('trackPageView');

app/views/projects/product_analytics/index.html.haml

+- page_title 'Product Analytics'
+
+= render 'links'
+
+- if @events.any?
+  %p
+    - if @events.total_count > @events.size
+      = _('Number of events for this project: %{total_count}.') % { total_count: number_with_delimiter(@events.total_count) }
+  %ol
+    - @events.each do |event|
+      %li
+        %code= event.as_json_wo_empty
+- else
+  .empty-state
+    .text-content
+      = _('There are currently no events.')

app/views/projects/product_analytics/setup.html.haml

+= render "links"
+%p
+  = _('Copy the code below to implement tracking in your application:')
+
+%pre
+  = render "tracker"
+
+%p.hint
+  = _('A platform value can be web, mob or app.')

app/views/projects/product_analytics/test.html.haml

+= render 'links'
+
+%p
+  = _('This page sends a payload. Go back to the events page to see a newly created event.')
+
+- if @event
+  %p
+    = _('Last item before this page loaded in your browser:')
+
+  %code
+    = @event.as_json_wo_empty
+
+:javascript
+  #{render 'tracker'}

config/routes/project.rb

@@ -306,6 +306,13 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
           resources :projects, only: :index
         end
 
+        resources :product_analytics, only: [:index] do
+          collection do
+            get :setup
+            get :test
+          end
+        end
+
         resources :error_tracking, only: [:index], controller: :error_tracking do
           collection do
             get ':issue_id/details',

doc/operations/index.md

@@ -16,6 +16,7 @@ your applications:
 - Manage your infrastructure with [Infrastructure as Code](../user/infrastructure/index.md) approaches.
 - Discover and view errors generated by your applications with [Error Tracking](error_tracking.md).
 - Handle incidents in your applications and services with [Incident Management](incident_management/index.md).
+- See how your application is used and analyze events with [Product Analytics](product_analytics.md).
 - Create, toggle, and remove [Feature Flags](feature_flags.md). **(PREMIUM)**
 - [Trace](tracing.md) the performance and health of a deployed application. **(ULTIMATE)**
 - Change the [settings of the Monitoring Dashboard](metrics/dashboards/settings.md).

doc/operations/product_analytics.md

+---
+stage: Monitor
+group: APM
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers
+---
+
+# Product Analytics **(CORE)**
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225167) in GitLab 13.3.
+> - It's deployed behind a feature flag, disabled by default.
+> - It's disabled on GitLab.com.
+> - It's able to be enabled or disabled per-project.
+> - It's not recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to enable it.
+
+GitLab allows you to go from planning an application to getting feedback. Feedback
+is not just observability, but also knowing how people use your product.
+Product Analytics uses events sent from your application to know how they are using it.
+It's based on [Snowplow](https://github.com/snowplow/snowplow), the best open-source
+event tracker. With Product Analytics, you can receive and analyze the Snowplow data
+inside GitLab.
+
+## Enable or disable Product Analytics
+
+Product Analytics is under development and not ready for production use. It's
+deployed behind a feature flag that's **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md)
+can enable it for your instance. Product Analytics can be enabled or disabled per-project.
+
+To enable it:
+
+```ruby
+# Instance-wide
+Feature.enable(:product_analytics)
+# or by project
+Feature.enable(:product_analytics, Project.find(<project id>))
+```
+
+To disable it:
+
+```ruby
+# Instance-wide
+Feature.disable(:product_analytics)
+# or by project
+Feature.disable(:product_analytics, Project.find(<project id>))
+```
+
+## Access Product Analytics
+
+After enabling the feature flag for Product Analytics, you can access the
+user interface:
+
+1. Sign in to GitLab as a user with Reporter or greater
+   [permissions](../user/permissions.md).
+1. Navigate to **{cloud-gear}** **Operations > TODO HERE**
+
+The user interface contains:
+
+- An Events page that shows the recent events and a total count.
+- A test page that sends a sample event.
+- A setup page containing the code to implement in your application.
+
+## Rate limits for Product Analytics
+
+While Product Analytics is under development, it's rate-limited to
+**100 events per minute** per project. This limit prevents the events table in the
+database from growing too quickly.
+
+## Data storage for Product Analytics
+
+Product Analytics stores events are stored in GitLab database.
+
+CAUTION: **Caution:**
+This data storage is experimental, and GitLab is likely to remove this data during
+future development.
+
+## Event collection
+
+Events are collected by [Rails collector](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36443),
+allowing GitLab to ship the feature fast. Due to scalability issue, GitLab plans
+to switch to a separate application, such as
+[snowplow-go-collector](https://gitlab.com/gitlab-org/snowplow-go-collector), for event collection.

lib/product_analytics/tracker.rb

+# frozen_string_literal: true
+
+module ProductAnalytics
+  class Tracker
+    # The file is located in the /public directory
+    URL = Gitlab.config.gitlab.url + '/-/sp.js'
+
+    # The collector URL minus protocol and /i
+    COLLECTOR_URL = Gitlab.config.gitlab.url.sub(/\Ahttps?\:\/\//, '') + '/-/collector'
+  end
+end

locale/gitlab.pot

@@ -1127,6 +1127,9 @@ msgstr ""
 msgid "A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features."
 msgstr ""
 
+msgid "A platform value can be web, mob or app."
+msgstr ""
+
 msgid "A project boilerplate for Salesforce App development with Salesforce Developer tools."
 msgstr ""
 
@@ -6779,6 +6782,9 @@ msgstr ""
 msgid "Copy source branch name"
 msgstr ""
 
+msgid "Copy the code below to implement tracking in your application:"
+msgstr ""
+
 msgid "Copy token"
 msgstr ""
 
@@ -13728,6 +13734,9 @@ msgstr ""
 msgid "Last edited by %{name}"
 msgstr ""
 
+msgid "Last item before this page loaded in your browser:"
+msgstr ""
+
 msgid "Last name"
 msgstr ""
 
@@ -16322,6 +16331,9 @@ msgstr ""
 msgid "Number of employees"
 msgstr ""
 
+msgid "Number of events for this project: %{total_count}."
+msgstr ""
+
 msgid "Number of files touched"
 msgstr ""
 
@@ -17824,6 +17836,9 @@ msgstr ""
 msgid "Proceed"
 msgstr ""
 
+msgid "Product Analytics"
+msgstr ""
+
 msgid "Productivity"
 msgstr ""
 
@@ -21747,6 +21762,9 @@ msgstr ""
 msgid "Settings to prevent self-approval across all projects in the instance. Only an administrator can modify these settings."
 msgstr ""
 
+msgid "Setup"
+msgstr ""
+
 msgid "Severity"
 msgstr ""
 
@@ -23891,6 +23909,9 @@ msgstr ""
 msgid "The vulnerability is no longer detected. Verify the vulnerability has been remediated before changing its status."
 msgstr ""
 
+msgid "There are currently no events."
+msgstr ""
+
 msgid "There are no %{replicableTypeName} to show"
 msgstr ""
 
@@ -24497,6 +24518,9 @@ msgstr ""
 msgid "This page is unavailable because you are not allowed to read information across multiple projects."
 msgstr ""
 
+msgid "This page sends a payload. Go back to the events page to see a newly created event."
+msgstr ""
+
 msgid "This pipeline does not use the %{codeStart}needs%{codeEnd} keyword and can't be represented as a directed acyclic graph."
 msgstr ""
 

spec/controllers/projects/product_analytics_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Projects::ProductAnalyticsController do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+
+  before(:all) do
+    project.add_maintainer(user)
+  end
+
+  before do
+    sign_in(user)
+    stub_feature_flags(product_analytics: true)
+  end
+
+  describe 'GET #index' do
+    it 'renders index with 200 status code' do
+      get :index, params: project_params
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(response).to render_template(:index)
+    end
+
+    context 'with an anonymous user' do
+      before do
+        sign_out(user)
+      end
+
+      it 'redirects to sign-in page' do
+        get :index, params: project_params
+
+        expect(response).to redirect_to(new_user_session_path)
+      end
+    end
+
+    context 'feature flag disabled' do
+      before do
+        stub_feature_flags(product_analytics: false)
+      end
+
+      it 'returns not found' do
+        get :index, params: project_params
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+  end
+
+  describe 'GET #test' do
+    it 'renders test with 200 status code' do
+      get :test, params: project_params
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(response).to render_template(:test)
+    end
+  end
+
+  describe 'GET #setup' do
+    it 'renders setup with 200 status code' do
+      get :setup, params: project_params
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(response).to render_template(:setup)
+    end
+  end
+
+  private
+
+  def project_params(opts = {})
+    opts.reverse_merge(namespace_id: project.namespace, project_id: project)
+  end
+end

spec/features/projects/product_analytics/events_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Product Analytics > Events' do
+  let_it_be(:project) { create(:project_empty_repo) }
+  let_it_be(:user) { create(:user) }
+  let(:event) { create(:product_analytics_event, project: project) }
+
+  before do
+    project.add_maintainer(user)
+    sign_in(user)
+  end
+
+  it 'shows no events message' do
+    visit(project_product_analytics_path(project))
+
+    expect(page).to have_content('There are currently no events')
+  end
+
+  it 'shows events' do
+    event
+
+    visit(project_product_analytics_path(project))
+
+    expect(page).to have_content('dvce_created_tstamp')
+    expect(page).to have_content(event.event_id)
+  end
+end

spec/features/projects/product_analytics/setup_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Product Analytics > Setup' do
+  let_it_be(:project) { create(:project_empty_repo) }
+  let_it_be(:user) { create(:user) }
+
+  before do
+    project.add_maintainer(user)
+    sign_in(user)
+  end
+
+  it 'shows the setup instructions' do
+    visit(setup_project_product_analytics_path(project))
+
+    expect(page).to have_content('Copy the code below to implement tracking in your application')
+  end
+end

spec/features/projects/product_analytics/test_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Product Analytics > Test' do
+  let_it_be(:project) { create(:project_empty_repo) }
+  let_it_be(:user) { create(:user) }
+
+  before do
+    project.add_maintainer(user)
+    sign_in(user)
+  end
+
+  it 'says it sends a payload' do
+    visit(test_project_product_analytics_path(project))
+
+    expect(page).to have_content('This page sends a payload.')
+  end
+
+  it 'shows the last event if there is one' do
+    event = create(:product_analytics_event, project: project)
+
+    visit(test_project_product_analytics_path(project))
+
+    expect(page).to have_content(event.event_id)
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -518,6 +518,7 @@ project:
 - build_report_results
 - vulnerability_statistic
 - vulnerability_historical_statistics
+- product_analytics_events
 award_emoji:
 - awardable
 - user

spec/lib/product_analytics/tracker_spec.rb

+# frozen_string_literal: true
+
+require "spec_helper"
+
+RSpec.describe ProductAnalytics::Tracker do
+  it { expect(described_class::URL).to eq('http://localhost/-/sp.js') }
+  it { expect(described_class::COLLECTOR_URL).to eq('localhost/-/collector') }
+end

spec/support/shared_contexts/navbar_structure_context.rb

@@ -67,6 +67,7 @@ RSpec.shared_context 'project navbar structure' do
           _('Incidents'),
           _('Environments'),
           _('Error Tracking'),
+          _('Product Analytics'),
           _('Serverless'),
           _('Logs'),
           _('Kubernetes')