Merge branch '12196-label-based-ca-query' into 'master'

Implement Label based CA event query

Closes #13157

See merge request gitlab-org/gitlab!19871

app/models/concerns/analytics/cycle_analytics/stage.rb

@@ -26,6 +26,7 @@ module Analytics
         alias_attribute :custom_stage?, :custom
         scope :default_stages, -> { where(custom: false) }
         scope :ordered, -> { order(:relative_position, :id) }
+        scope :for_list, -> { includes(:start_event_label, :end_event_label).ordered }
       end
 
       def parent=(_)

db/migrate/20191124150431_change_label_id_index_to_include_action_on_label_events.rb

+# frozen_string_literal: true
+
+class ChangeLabelIdIndexToIncludeActionOnLabelEvents < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:resource_label_events, %I[label_id action])
+
+    remove_concurrent_index(:resource_label_events, :label_id)
+  end
+
+  def down
+    add_concurrent_index(:resource_label_events, :label_id)
+
+    remove_concurrent_index(:resource_label_events, %I[label_id action])
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_11_19_023952) do
+ActiveRecord::Schema.define(version: 2019_11_24_150431) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -3477,7 +3477,7 @@ ActiveRecord::Schema.define(version: 2019_11_19_023952) do
     t.text "reference_html"
     t.index ["epic_id"], name: "index_resource_label_events_on_epic_id"
     t.index ["issue_id"], name: "index_resource_label_events_on_issue_id"
-    t.index ["label_id"], name: "index_resource_label_events_on_label_id"
+    t.index ["label_id", "action"], name: "index_resource_label_events_on_label_id_and_action"
     t.index ["merge_request_id"], name: "index_resource_label_events_on_merge_request_id"
     t.index ["user_id"], name: "index_resource_label_events_on_user_id"
   end

ee/app/serializers/analytics/cycle_analytics/event_entity.rb

@@ -8,6 +8,7 @@ module Analytics
       expose :type
       expose :can_be_start_event?, as: :can_be_start_event
       expose :allowed_end_events
+      expose :label_based?, as: :label_based
 
       private
 

ee/app/serializers/analytics/cycle_analytics/stage_entity.rb

@@ -11,6 +11,8 @@ module Analytics
       expose :custom
       expose :start_event_identifier, if: -> (s) { s.custom? }
       expose :end_event_identifier, if: -> (s) { s.custom? }
+      expose :start_event_label, using: LabelEntity, if: -> (s) { s.start_event_label_based? }
+      expose :end_event_label, using: LabelEntity, if: -> (s) { s.end_event_label_based? }
 
       def id
         object.id || object.name

ee/app/services/analytics/cycle_analytics/stages/list_service.rb

@@ -17,7 +17,7 @@ module Analytics
         end
 
         def persisted_stages
-          parent.cycle_analytics_stages.ordered
+          parent.cycle_analytics_stages.for_list
         end
       end
     end

ee/db/fixtures/development/30_customizable_cycle_analytics.rb

+# frozen_string_literal: true
+
+require './spec/support/sidekiq_middleware'
+
+class Gitlab::Seeder::CustomizableCycleAnalytics
+  attr_reader :project, :group, :user
+
+  ONE_WEEK_IN_HOURS = 168
+  ISSUE_COUNT = 15
+  MERGE_REQUEST_COUNT = 10
+
+  def initialize(project)
+    @project = project
+    @group = project.group.root_ancestor
+    @user = User.admins.first
+  end
+
+  def seed!
+    Sidekiq::Worker.skipping_transaction_check do
+      Sidekiq::Testing.inline! do
+        create_stages!
+
+        seed_issue_based_stages!
+        seed_issue_label_based_stages!
+
+        seed_merge_request_based_stages!
+
+        puts "."
+      end
+    end
+  end
+
+  private
+
+  def in_dev_label
+    @in_dev_label ||= GroupLabel.where(title: 'in-dev', group: group).first_or_create!
+  end
+
+  def in_review_label
+    @in_review_label ||= GroupLabel.where(title: 'in-review', group: group).first_or_create!
+  end
+
+  def create_stages!
+    stages_params = [
+      {
+        name: 'IssueCreated-IssueClosed',
+        start_event_identifier: :issue_created,
+        end_event_identifier: :issue_closed
+      },
+      {
+        name: 'IssueCreated-IssueFirstMentionedInCommit',
+        start_event_identifier: :issue_created,
+        end_event_identifier: :issue_first_mentioned_in_commit
+      },
+      {
+        name: 'IssueCreated-IssueInDevLabelAdded',
+        start_event_identifier: :issue_created,
+        end_event_identifier: :issue_label_added,
+        end_event_label_id: in_dev_label.id
+      },
+      {
+        name: 'IssueInDevLabelAdded-IssueInReviewLabelAdded',
+        start_event_identifier: :issue_label_added,
+        start_event_label_id: in_dev_label.id,
+        end_event_identifier: :issue_label_added,
+        end_event_label_id: in_review_label.id
+      },
+      {
+        name: 'MergeRequestCreated-MergeRequestClosed',
+        start_event_identifier: :merge_request_created,
+        end_event_identifier: :merge_request_closed
+      },
+      {
+        name: 'MergeRequestCreated-MergeRequestMerged',
+        start_event_identifier: :merge_request_created,
+        end_event_identifier: :merge_request_merged
+      }
+    ]
+
+    stages_params.each do |params|
+      next if Analytics::CycleAnalytics::GroupStage.where(group: group).find_by(name: params[:name])
+
+      Analytics::CycleAnalytics::Stages::CreateService.new(parent: group, current_user: user, params: params).execute
+    end
+  end
+
+  def seed_issue_based_stages!
+    # issue created - issue closed
+    issues.pop(5).each do |issue|
+      Timecop.travel random_duration_in_hours.hours.ago
+      issue.update!(created_at: Time.now)
+
+      Timecop.travel random_duration_in_hours.hours.from_now
+      issue.close!
+    end
+
+    # issue created - issue first mentioned in commit
+    issues.pop(5).each do |issue|
+      Timecop.travel random_duration_in_hours.hours.ago
+      issue.update!(created_at: Time.now)
+
+      Timecop.travel random_duration_in_hours.hours.from_now
+      issue.metrics.update!(first_mentioned_in_commit_at: Time.now)
+    end
+  end
+
+  def seed_issue_label_based_stages!
+    issues.pop(5).each do |issue|
+      Timecop.travel(issue.created_at + random_duration_in_hours.hours)
+      Issues::UpdateService.new(
+        project,
+        user,
+        label_ids: [in_dev_label.id]
+      ).execute(issue)
+
+      Timecop.travel(random_duration_in_hours.hours.from_now)
+      Issues::UpdateService.new(
+        project,
+        user,
+        label_ids: [in_review_label.id]
+      ).execute(issue)
+    end
+  end
+
+  def seed_merge_request_based_stages!
+    merge_requests.pop(5).each do |mr|
+      Timecop.travel random_duration_in_hours.hours.ago
+      mr.update!(created_at: Time.now)
+
+      Timecop.travel random_duration_in_hours.hours.from_now
+      mr.close!
+    end
+
+    merge_requests.pop(5).each do |mr|
+      Timecop.travel random_duration_in_hours.hours.ago
+      mr.update!(created_at: Time.now)
+
+      Timecop.travel random_duration_in_hours.hours.from_now
+      mr.metrics.update!(merged_at: Time.now)
+    end
+  end
+
+  def random_duration_in_hours
+    rand(ONE_WEEK_IN_HOURS)
+  end
+
+  def issues
+    @issues ||= Array.new(ISSUE_COUNT).map do
+      issue_params = {
+        title: FFaker::Lorem.sentence(6),
+        description: FFaker::Lorem.sentence,
+        state: 'opened',
+        assignees: [project.team.users.sample]
+      }
+
+      Issues::CreateService.new(@project, project.team.users.sample, issue_params).execute
+    end
+  end
+
+  def merge_requests
+    @merge_requests ||= Array.new(MERGE_REQUEST_COUNT).map do |i|
+      opts = {
+        title: 'Customized Cycle Analytics merge_request',
+        description: "some description",
+        source_branch: "#{FFaker::Lorem.word}-#{i}-#{SecureRandom.hex(5)}",
+        target_branch: 'master'
+      }
+
+      begin
+        developer = project.team.developers.sample
+        MergeRequests::CreateService.new(project, developer, opts).execute
+      rescue Gitlab::Access::AccessDeniedError
+        nil
+      end
+    end.compact
+  end
+end
+
+Gitlab::Seeder.quiet do
+  flag = 'SEED_CUSTOMIZABLE_CYCLE_ANALYTICS'
+
+  if ENV[flag]
+    Project.find_each do |project|
+      next unless project.group
+      # This seed naively assumes that every project has a repository, and every
+      # repository has a `master` branch, which may be the case for a pristine
+      # GDK seed, but is almost never true for a GDK that's actually had
+      # development performed on it.
+      next unless project.repository_exists?
+      next unless project.repository.commit('master')
+
+      seeder = Gitlab::Seeder::CustomizableCycleAnalytics.new(project)
+      seeder.seed!
+    end
+  else
+    puts "Skipped. Use the `#{flag}` environment variable to enable."
+  end
+end

ee/lib/ee/gitlab/analytics/cycle_analytics/stage_events.rb

@@ -41,25 +41,33 @@ module EE
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstAddedToBoard,
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstAssociatedWithMilestone,
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstMentionedInCommit,
-              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLastEdited
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLastEdited,
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLabelAdded,
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLabelRemoved
             ],
             ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstAddedToBoard => [
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueClosed,
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstAssociatedWithMilestone,
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstMentionedInCommit,
-              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLastEdited
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLastEdited,
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLabelAdded,
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLabelRemoved
             ],
             ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstAssociatedWithMilestone => [
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueClosed,
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstAddedToBoard,
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstMentionedInCommit,
-              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLastEdited
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLastEdited,
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLabelAdded,
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLabelRemoved
             ],
             ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstMentionedInCommit => [
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueClosed,
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstAssociatedWithMilestone,
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstAddedToBoard,
-              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLastEdited
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLastEdited,
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLabelAdded,
+              ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLabelRemoved
             ],
             ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueClosed => [
               ::Gitlab::Analytics::CycleAnalytics::StageEvents::IssueLastEdited,

ee/lib/gitlab/analytics/cycle_analytics/stage_events/issue_label_added.rb

@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def subquery
+            resource_label_events_with_subquery(:issue_id, label, ::ResourceLabelEvent.actions[:add], :asc)
+          end
         end
       end
     end

ee/lib/gitlab/analytics/cycle_analytics/stage_events/issue_label_removed.rb

@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def subquery
+            resource_label_events_with_subquery(:issue_id, label, ::ResourceLabelEvent.actions[:remove], :desc)
+          end
         end
       end
     end

ee/lib/gitlab/analytics/cycle_analytics/stage_events/label_based_stage_event.rb

@@ -6,9 +6,95 @@ module Gitlab
       module StageEvents
         # Represents an event that is related to label creation or removal, this model requires a label provided by the user
         class LabelBasedStageEvent < StageEvent
-          def label_based?
+          include ActiveRecord::ConnectionAdapters::Quoting
+
+          def label
+            params.fetch(:label)
+          end
+
+          def label_id
+            label.id
+          end
+
+          def self.label_based?
             true
           end
+
+          def timestamp_projection
+            Arel.sql("#{join_expression_name}.created_at")
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query
+              .joins("INNER JOIN (#{subquery.to_sql}) #{join_expression_name} on #{join_expression_name}.model_id = #{quote_table_name(object_type.table_name)}.id")
+              .where("#{join_expression_name}.label_assignment_order = 1")
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          private
+
+          def resource_label_events_table
+            ResourceLabelEvent.arel_table
+          end
+
+          # Labels can be assigned and unassigned multiple times, we need a way to pick only one record from `resource_label_events` table.
+          # Consider the following example:
+          #
+          # | id | action | label_id | issue_id | created_at |
+          # | -- | ------ | -------- | -------- | ---------- |
+          # | 1  | add    | 1        | 10       | 2010-01-01 |
+          # | 2  | remove | 1        | 10       | 2010-02-01 |
+          # | 3  | add    | 1        | 10       | 2015-01-01 |
+          # | 4  | remove | 1        | 10       | 2015-02-01 |
+          #
+          # In this example a label (id: 1) has been assigned and unassigned twice on the same Issue.
+          #
+          # - IssueLabelAdded event: find the first assignment (add, id = 1)
+          # - IssueLabelRemoved event: find the latest unassignment (remove, id = 4)
+          #
+          # This can be achieved with the PARTITION window function.
+          #
+          #  - IssueLabelAdded: order by `created_at` ASC and take the row number 1
+          #  - IssueLabelRemoved: order by `created_at` DESC and take the row number 1
+          #
+          #  Arguments:
+          #    foreign_key: :issue_id or :merge_request_id (based on resource_label_events table)
+          #    label: label model,
+          #    action: :add or :remove
+          #    order: :asc or :desc
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def resource_label_events_with_subquery(foreign_key, label, action, order)
+            ResourceLabelEvent
+              .select(:created_at, resource_label_events_table[foreign_key].as('model_id'), partition_select(foreign_key, order).as('label_assignment_order'))
+              .where(action: action)
+              .where(label_id: label.id)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          # The same join expression could be used multiple times in the same query, to avoid conflicts, we're adding random hex string as suffix.
+          def join_expression_name
+            @join_expression_name ||= quote_table_name("#{self.class.to_s.demodulize.underscore}_#{SecureRandom.hex(5)}")
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def partition_select(foreign_key, order)
+            order_expression = case order
+                               when :asc
+                                 resource_label_events_table[:created_at].asc
+                               when :desc
+                                 resource_label_events_table[:created_at].desc
+                               else
+                                 raise "unsupported order option: #{order}"
+                               end
+
+            Arel::Nodes::Over.new(
+              Arel::Nodes::NamedFunction.new('row_number', []),
+              Arel::Nodes::Window.new.partition(resource_label_events_table[foreign_key]).order(order_expression)
+            )
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end

ee/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_label_added.rb

@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def subquery
+            resource_label_events_with_subquery(:merge_request_id, label, ::ResourceLabelEvent.actions[:add], :asc)
+          end
         end
       end
     end

ee/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_label_removed.rb

@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def subquery
+            resource_label_events_with_subquery(:merge_request_id, label, ::ResourceLabelEvent.actions[:remove], :desc)
+          end
         end
       end
     end

ee/spec/fixtures/api/schemas/analytics/cycle_analytics/event.json

 {
   "type": "object",
-  "required": ["name", "identifier", "type", "can_be_start_event", "allowed_end_events"],
+  "required": ["name", "identifier", "type", "can_be_start_event", "allowed_end_events", "label_based"],
   "properties": {
     "name": { 
       "type": "string" 
@@ -15,6 +15,9 @@
     "can_be_start_event": {
       "type": "boolean"
     },
+    "label_based": {
+      "type": "boolean"
+    },
     "allowed_end_events": {
       "type": "array",
       "items": {

ee/spec/lib/gitlab/analytics/cycle_analytics/data_collector_spec.rb

@@ -148,6 +148,93 @@ describe Gitlab::Analytics::CycleAnalytics::DataCollector do
 
         it_behaves_like 'custom cycle analytics stage'
       end
+
+      describe 'between issue label added time and label removed time' do
+        let(:start_event_identifier) { :issue_label_added }
+        let(:end_event_identifier) { :issue_label_removed }
+
+        before do
+          stage.start_event_label = label
+          stage.end_event_label = label
+        end
+
+        def create_data_for_start_event(example_class)
+          issue = create(:issue, :opened, project: example_class.project)
+
+          Issues::UpdateService.new(
+            example_class.project,
+            user,
+            label_ids: [example_class.label.id]
+          ).execute(issue)
+
+          issue
+        end
+
+        def create_data_for_end_event(resource, example_class)
+          Issues::UpdateService.new(
+            example_class.project,
+            user,
+            label_ids: []
+          ).execute(resource)
+        end
+
+        it_behaves_like 'custom cycle analytics stage'
+      end
+
+      describe 'between issue label added time and another issue label added time' do
+        let(:start_event_identifier) { :issue_label_added }
+        let(:end_event_identifier) { :issue_label_added }
+
+        before do
+          stage.start_event_label = label
+          stage.end_event_label = other_label
+        end
+
+        def create_data_for_start_event(example_class)
+          issue = create(:issue, :opened, project: example_class.project)
+
+          Issues::UpdateService.new(
+            example_class.project,
+            user,
+            label_ids: [example_class.label.id]
+          ).execute(issue)
+
+          issue
+        end
+
+        def create_data_for_end_event(issue, example_class)
+          Issues::UpdateService.new(
+            example_class.project,
+            user,
+            label_ids: [example_class.label.id, example_class.other_label.id]
+          ).execute(issue)
+        end
+
+        it_behaves_like 'custom cycle analytics stage'
+      end
+
+      describe 'between issue creation time and issue label added time' do
+        let(:start_event_identifier) { :issue_created }
+        let(:end_event_identifier) { :issue_label_added }
+
+        before do
+          stage.end_event_label = label
+        end
+
+        def create_data_for_start_event(example_class)
+          create(:issue, :opened, project: example_class.project)
+        end
+
+        def create_data_for_end_event(issue, example_class)
+          Issues::UpdateService.new(
+            example_class.project,
+            user,
+            label_ids: [example_class.label.id]
+          ).execute(issue)
+        end
+
+        it_behaves_like 'custom cycle analytics stage'
+      end
     end
 
     context 'when `MergeRequest` based stage is given' do
@@ -229,12 +316,46 @@ describe Gitlab::Analytics::CycleAnalytics::DataCollector do
 
         it_behaves_like 'custom cycle analytics stage'
       end
+
+      describe 'between merge request label added time and label removed time' do
+        let(:start_event_identifier) { :merge_request_label_added }
+        let(:end_event_identifier) { :merge_request_label_removed }
+
+        before do
+          stage.start_event_label = label
+          stage.end_event_label = label
+        end
+
+        def create_data_for_start_event(example_class)
+          mr = create(:merge_request, source_project: example_class.project, allow_broken: true)
+
+          MergeRequests::UpdateService.new(
+            example_class.project,
+            user,
+            label_ids: [label.id]
+          ).execute(mr)
+
+          mr
+        end
+
+        def create_data_for_end_event(mr, example_class)
+          MergeRequests::UpdateService.new(
+            example_class.project,
+            user,
+            label_ids: []
+          ).execute(mr)
+        end
+
+        it_behaves_like 'custom cycle analytics stage'
+      end
     end
   end
 
   context 'when `Analytics::CycleAnalytics::ProjectStage` is given' do
     it_behaves_like 'test various start and end event combinations' do
-      let_it_be(:project) { create(:project, :repository) }
+      let_it_be(:project) { create(:project, :repository, group: create(:group)) }
+      let_it_be(:label) { create(:group_label, group: project.group) }
+      let_it_be(:other_label) { create(:group_label, group: project.group) }
 
       let(:stage) do
         Analytics::CycleAnalytics::ProjectStage.new(
@@ -255,6 +376,8 @@ describe Gitlab::Analytics::CycleAnalytics::DataCollector do
     it_behaves_like 'test various start and end event combinations' do
       let_it_be(:group) { create(:group) }
       let_it_be(:project) { create(:project, :repository, group: group) }
+      let_it_be(:label) { create(:group_label, group: group) }
+      let_it_be(:other_label) { create(:group_label, group: group) }
 
       let(:stage) do
         Analytics::CycleAnalytics::GroupStage.new(

ee/spec/services/analytics/cycle_analytics/stages/create_service_spec.rb

@@ -86,4 +86,29 @@ describe Analytics::CycleAnalytics::Stages::CreateService do
       end
     end
   end
+
+  describe 'label based stages' do
+    let(:label) { create(:group_label, group: group) }
+
+    let(:params) do
+      {
+        name: 'my stage',
+        start_event_identifier: :issue_label_added,
+        end_event_identifier: :issue_label_removed,
+        start_event_label_id: label.id,
+        end_event_label_id: label.id
+      }
+    end
+
+    it { expect(subject).to be_success }
+
+    it 'persists the `start_event_label_id` and `end_event_label_id` attributes' do
+      subject
+
+      stage = subject.payload[:stage]
+
+      expect(stage.start_event_label).to eq(label)
+      expect(stage.end_event_label).to eq(label)
+    end
+  end
 end

lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb

@@ -8,6 +8,8 @@ module Gitlab
         class StageEvent
           include Gitlab::CycleAnalytics::MetricsTables
 
+          delegate :label_based?, to: :class
+
           def initialize(params)
             @params = params
           end
@@ -35,7 +37,7 @@ module Gitlab
             query
           end
 
-          def label_based?
+          def self.label_based?
             false
           end