Merge branch '12196-group-based-ca' into 'master'

Group Based Cycle Analytics Backend

Closes #12196

See merge request gitlab-org/gitlab!18452

ee/lib/ee/gitlab/analytics/cycle_analytics/base_query_builder.rb

+# frozen_string_literal: true
+
+module EE::Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder
+  extend ::Gitlab::Utils::Override
+
+  override :build
+  def build
+    filter_by_project_ids(super)
+  end
+
+  private
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  def filter_by_project_ids(query)
+    project_ids = Array(params[:project_ids])
+
+    query = query.where(project_id: project_ids) if project_ids.any?
+    query
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  override :filter_by_parent_model
+  # rubocop: disable CodeReuse/ActiveRecord
+  def filter_by_parent_model(query)
+    return super unless parent_class.eql?(Group)
+
+    if subject_class.eql?(Issue)
+      join_groups(query.joins(:project))
+    elsif subject_class.eql?(MergeRequest)
+      join_groups(query.joins(:target_project))
+    else
+      raise ArgumentError, "unknown subject_class: #{subject_class}"
+    end
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  def join_groups(query)
+    query.joins(Arel.sql("INNER JOIN (#{stage.parent.self_and_descendants.to_sql}) namespaces ON namespaces.id=projects.namespace_id"))
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+end

ee/lib/ee/gitlab/analytics/cycle_analytics/records_fetcher.rb

+# frozen_string_literal: true
+
+module EE::Gitlab::Analytics::CycleAnalytics::RecordsFetcher
+  extend ::Gitlab::Utils::Override
+
+  override :finder_params
+  def finder_params
+    super.merge({ ::Group => { group_id: stage.parent_id } })
+  end
+end

ee/spec/lib/ee/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder do
+  let_it_be(:group) { create(:group) }
+  let_it_be(:subgroup) { create(:group, parent: group) }
+  let_it_be(:project_in_group) { create(:project, :repository, group: group) }
+  let_it_be(:project_in_subgroup) { create(:project, :repository, group: subgroup) }
+  let_it_be(:project_outside_group) { create(:project, :repository, group: create(:group)) }
+
+  context 'when the subject is `Issue`' do
+    let(:issue_in_project) { create(:issue, project: project_in_group, created_at: 5.days.ago) }
+    let(:issue_in_subgroup_project) { create(:issue, project: project_in_subgroup, created_at: 5.days.ago) }
+    let(:issue_outside_group) { create(:issue, project: project_outside_group, created_at: 5.days.ago) }
+
+    before do
+      [issue_in_project, issue_in_subgroup_project, issue_outside_group].each do |issue|
+        issue.metrics.update!(first_mentioned_in_commit_at: 2.days.ago)
+      end
+    end
+
+    it 'loads Issue records within the given Group' do
+      stage = build(:cycle_analytics_group_stage, {
+        start_event_identifier: :issue_created,
+        end_event_identifier: :issue_first_mentioned_in_commit,
+        group: group
+      })
+
+      result = described_class.new(stage: stage).build
+
+      expect(result).to contain_exactly(issue_in_project, issue_in_subgroup_project)
+    end
+  end
+
+  context 'when the subject is `MergeRequest`' do
+    let(:mr_in_project) { create(:merge_request, source_project: project_in_group, created_at: 5.days.ago) }
+    let(:mr_in_subgroup_project) { create(:merge_request, source_project: project_in_subgroup, created_at: 5.days.ago) }
+    let(:mr_outside_group) { create(:merge_request, source_project: project_outside_group, created_at: 5.days.ago) }
+
+    before do
+      [mr_in_project, mr_in_subgroup_project, mr_outside_group].each do |mr|
+        mr.metrics.update!(merged_at: 2.days.ago)
+      end
+    end
+
+    it 'loads MergeRequest records within the given Group' do
+      stage = build(:cycle_analytics_group_stage, {
+        start_event_identifier: :merge_request_created,
+        end_event_identifier: :merge_request_merged,
+        group: group
+      })
+
+      result = described_class.new(stage: stage).build
+
+      expect(result).to contain_exactly(mr_in_project, mr_in_subgroup_project)
+    end
+  end
+end

ee/spec/lib/gitlab/analytics/cycle_analytics/data_collector_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::DataCollector do
+  let_it_be(:user) { create(:user) }
+
+  around do |example|
+    Timecop.freeze { example.run }
+  end
+
+  def round_to_days(seconds)
+    seconds.fdiv(1.day.to_i).round
+  end
+
+  # Setting up test data for a stage depends on the `start_event_identifier` and
+  # `end_event_identifier` attributes. Since stages can be customized, the test
+  # uses two methods for the data preparaton: `create_data_for_start_event` and
+  # `create_data_for_end_event`. For each stage we create 3 records with a fixed
+  # durations (10, 5, 15 days) in order to easily generalize the test cases.
+  shared_examples 'custom cycle analytics stage' do
+    let(:data_collector) { described_class.new(stage: stage, params: { from: Time.new(2019), to: Time.new(2020), current_user: user }) }
+
+    before do
+      # takes 10 days
+      resource1 = Timecop.travel(Time.new(2019, 3, 5)) do
+        create_data_for_start_event(self)
+      end
+
+      Timecop.travel(Time.new(2019, 3, 15)) do
+        create_data_for_end_event(resource1, self)
+      end
+
+      # takes 5 days
+      resource2 = Timecop.travel(Time.new(2019, 3, 5)) do
+        create_data_for_start_event(self)
+      end
+
+      Timecop.travel(Time.new(2019, 3, 10)) do
+        create_data_for_end_event(resource2, self)
+      end
+
+      # takes 15 days
+      resource3 = Timecop.travel(Time.new(2019, 3, 5)) do
+        create_data_for_start_event(self)
+      end
+
+      Timecop.travel(Time.new(2019, 3, 20)) do
+        create_data_for_end_event(resource3, self)
+      end
+    end
+
+    it 'loads serialized records' do
+      items = data_collector.records_fetcher.serialized_records
+      expect(items.size).to eq(3)
+    end
+
+    it 'calculates median' do
+      expect(round_to_days(data_collector.median.seconds)).to eq(10)
+    end
+  end
+
+  shared_examples 'test various start and end event combinations' do
+    context 'when `Issue` based stage is given' do
+      context 'between issue creation time and closing time' do
+        let(:start_event_identifier) { :issue_created }
+        let(:end_event_identifier) { :issue_first_mentioned_in_commit }
+
+        def create_data_for_start_event(example_class)
+          create(:issue, :opened, project: example_class.project)
+        end
+
+        def create_data_for_end_event(issue, example_class)
+          issue.metrics.update!(first_mentioned_in_commit_at: Time.now)
+        end
+
+        it_behaves_like 'custom cycle analytics stage'
+      end
+    end
+
+    context 'when `MergeRequest` based stage is given' do
+      context 'between merge request creation time and merged at time' do
+        let(:start_event_identifier) { :merge_request_created }
+        let(:end_event_identifier) { :merge_request_merged }
+
+        def create_data_for_start_event(example_class)
+          create(:merge_request, :closed, source_project: example_class.project)
+        end
+
+        def create_data_for_end_event(mr, example_class)
+          mr.metrics.update!(merged_at: Time.now)
+        end
+
+        it_behaves_like 'custom cycle analytics stage'
+      end
+
+      context 'between merge request merrged time and first deployed to production at time' do
+        let(:start_event_identifier) { :merge_request_merged }
+        let(:end_event_identifier) { :merge_request_first_deployed_to_production }
+
+        def create_data_for_start_event(example_class)
+          create(:merge_request, :closed, source_project: example_class.project).tap do |mr|
+            mr.metrics.update!(merged_at: Time.now)
+          end
+        end
+
+        def create_data_for_end_event(mr, example_class)
+          mr.metrics.update!(first_deployed_to_production_at: Time.now)
+        end
+
+        it_behaves_like 'custom cycle analytics stage'
+      end
+
+      context 'between merge request build started time and build finished time' do
+        let(:start_event_identifier) { :merge_request_last_build_started }
+        let(:end_event_identifier) { :merge_request_last_build_finished }
+
+        def create_data_for_start_event(example_class)
+          create(:merge_request, :closed, source_project: example_class.project).tap do |mr|
+            mr.metrics.update!(latest_build_started_at: Time.now)
+          end
+        end
+
+        def create_data_for_end_event(mr, example_class)
+          mr.metrics.update!(latest_build_finished_at: Time.now)
+        end
+
+        it_behaves_like 'custom cycle analytics stage'
+      end
+    end
+  end
+
+  context 'when `Analytics::CycleAnalytics::ProjectStage` is given' do
+    it_behaves_like 'test various start and end event combinations' do
+      let_it_be(:project) { create(:project, :repository) }
+
+      let(:stage) do
+        Analytics::CycleAnalytics::ProjectStage.new(
+          name: 'My Stage',
+          project: project,
+          start_event_identifier: start_event_identifier,
+          end_event_identifier: end_event_identifier
+        )
+      end
+
+      before_all do
+        project.add_user(user, Gitlab::Access::DEVELOPER)
+      end
+    end
+  end
+
+  context 'when `Analytics::CycleAnalytics::GroupStage` is given' do
+    it_behaves_like 'test various start and end event combinations' do
+      let_it_be(:group) { create(:group) }
+      let_it_be(:project) { create(:project, :repository, group: group) }
+
+      let(:stage) do
+        Analytics::CycleAnalytics::GroupStage.new(
+          name: 'My Stage',
+          group: group,
+          start_event_identifier: start_event_identifier,
+          end_event_identifier: end_event_identifier
+        )
+      end
+
+      before_all do
+        group.add_user(user, GroupMember::MAINTAINER)
+      end
+    end
+
+    context 'when `project_ids` parameter is given' do
+      let(:group) { create(:group) }
+      let(:project1) { create(:project, :repository, group: group) }
+      let(:project2) { create(:project, :repository, group: group) }
+
+      let(:stage) do
+        Analytics::CycleAnalytics::GroupStage.new(
+          name: 'My Stage',
+          group: group,
+          start_event_identifier: :merge_request_created,
+          end_event_identifier: :merge_request_merged
+        )
+      end
+
+      let(:data_collector) do
+        described_class.new(stage: stage, params: {
+          from: Time.new(2019, 1, 1),
+          project_ids: [project2.id],
+          current_user: user
+        })
+      end
+
+      before do
+        group.add_user(user, GroupMember::MAINTAINER)
+
+        Timecop.travel(Time.new(2019, 6, 1)) do
+          mr = create(:merge_request, source_project: project1)
+          mr.metrics.update!(merged_at: 1.hour.from_now)
+
+          mr = create(:merge_request, source_project: project2)
+          mr.metrics.update!(merged_at: 1.hour.from_now)
+        end
+      end
+
+      it 'filters for the given `project_ids`' do
+        items = data_collector.records_fetcher.serialized_records
+        expect(items.size).to eq(1)
+
+        merge_request = project2.merge_requests.first
+        expect(items.first[:title]).to eq(merge_request.title)
+        expect(items.first[:iid]).to eq(merge_request.iid.to_s)
+      end
+    end
+  end
+end

lib/gitlab/analytics/cycle_analytics/base_query_builder.rb

@@ -68,3 +68,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder')

lib/gitlab/analytics/cycle_analytics/records_fetcher.rb

@@ -130,3 +130,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::RecordsFetcher')