Commit a24659e6 authored by Peter Leitzen's avatar Peter Leitzen

Merge branch '246847-store-pipeline-counts-by-status' into 'master'

Store pipeline counts by status

See merge request gitlab-org/gitlab!43027
parents 44a8c5c2 308129a2
......@@ -14,6 +14,10 @@ module Types
value 'MERGE_REQUESTS', 'Merge request count', value: :merge_requests
value 'GROUPS', 'Group count', value: :groups
value 'PIPELINES', 'Pipeline count', value: :pipelines
value 'PIPELINES_SUCCEEDED', 'Pipeline count with success status', value: :pipelines_succeeded
value 'PIPELINES_FAILED', 'Pipeline count with failed status', value: :pipelines_failed
value 'PIPELINES_CANCELED', 'Pipeline count with canceled status', value: :pipelines_canceled
value 'PIPELINES_SKIPPED', 'Pipeline count with skipped status', value: :pipelines_skipped
end
end
end
......
......@@ -3,13 +3,19 @@
module Analytics
module InstanceStatistics
class Measurement < ApplicationRecord
EXPERIMENTAL_IDENTIFIERS = %i[pipelines_succeeded pipelines_failed pipelines_canceled pipelines_skipped].freeze
enum identifier: {
projects: 1,
users: 2,
issues: 3,
merge_requests: 4,
groups: 5,
pipelines: 6
pipelines: 6,
pipelines_succeeded: 7,
pipelines_failed: 8,
pipelines_canceled: 9,
pipelines_skipped: 10
}
IDENTIFIER_QUERY_MAPPING = {
......@@ -18,7 +24,11 @@ module Analytics
identifiers[:issues] => -> { Issue },
identifiers[:merge_requests] => -> { MergeRequest },
identifiers[:groups] => -> { Group },
identifiers[:pipelines] => -> { Ci::Pipeline }
identifiers[:pipelines] => -> { Ci::Pipeline },
identifiers[:pipelines_succeeded] => -> { Ci::Pipeline.success },
identifiers[:pipelines_failed] => -> { Ci::Pipeline.failed },
identifiers[:pipelines_canceled] => -> { Ci::Pipeline.canceled },
identifiers[:pipelines_skipped] => -> { Ci::Pipeline.skipped }
}.freeze
validates :recorded_at, :identifier, :count, presence: true
......@@ -26,6 +36,14 @@ module Analytics
scope :order_by_latest, -> { order(recorded_at: :desc) }
scope :with_identifier, -> (identifier) { where(identifier: identifier) }
def self.measurement_identifier_values
if Feature.enabled?(:store_ci_pipeline_counts_by_status)
identifiers.values
else
identifiers.values - EXPERIMENTAL_IDENTIFIERS.map { |identifier| identifiers[identifier] }
end
end
end
end
end
......@@ -17,10 +17,9 @@ module Analytics
return if Feature.disabled?(:store_instance_statistics_measurements, default_enabled: true)
recorded_at = Time.zone.now
measurement_identifiers = Analytics::InstanceStatistics::Measurement.identifiers
worker_arguments = Gitlab::Analytics::InstanceStatistics::WorkersArgumentBuilder.new(
measurement_identifiers: measurement_identifiers.values,
measurement_identifiers: ::Analytics::InstanceStatistics::Measurement.measurement_identifier_values,
recorded_at: recorded_at
).execute
......
---
title: Store pipeline counts by status for instance statistics
merge_request: 43027
author:
type: changed
---
name: store_ci_pipeline_counts_by_status
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43027
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/254721
type: development
group: group::analytics
default_enabled: false
# frozen_string_literal: true
class ChangeIndexOnPipelineStatus < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
OLD_INDEX_NAME = 'index_ci_pipelines_on_status'
NEW_INDEX_NAME = 'index_ci_pipelines_on_status_and_id'
disable_ddl_transaction!
def up
add_concurrent_index :ci_pipelines, [:status, :id], name: NEW_INDEX_NAME
remove_concurrent_index_by_name :ci_pipelines, name: OLD_INDEX_NAME
end
def down
add_concurrent_index :ci_pipelines, :status, name: OLD_INDEX_NAME
remove_concurrent_index_by_name :ci_pipelines, name: NEW_INDEX_NAME
end
end
ab044b609a29e9a179813de79dab9770665917a8ed78db907755a64f2d4aa47c
\ No newline at end of file
......@@ -19711,7 +19711,7 @@ CREATE INDEX index_ci_pipelines_on_project_id_and_user_id_and_status_and_ref ON
CREATE INDEX index_ci_pipelines_on_project_idandrefandiddesc ON ci_pipelines USING btree (project_id, ref, id DESC);
CREATE INDEX index_ci_pipelines_on_status ON ci_pipelines USING btree (status);
CREATE INDEX index_ci_pipelines_on_status_and_id ON ci_pipelines USING btree (status, id);
CREATE INDEX index_ci_pipelines_on_user_id_and_created_at_and_config_source ON ci_pipelines USING btree (user_id, created_at, config_source);
......
......@@ -9388,6 +9388,26 @@ enum MeasurementIdentifier {
"""
PIPELINES
"""
Pipeline count with canceled status
"""
PIPELINES_CANCELED
"""
Pipeline count with failed status
"""
PIPELINES_FAILED
"""
Pipeline count with skipped status
"""
PIPELINES_SKIPPED
"""
Pipeline count with success status
"""
PIPELINES_SUCCEEDED
"""
Project count
"""
......
......@@ -26010,6 +26010,30 @@
"description": "Pipeline count",
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "PIPELINES_SUCCEEDED",
"description": "Pipeline count with success status",
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "PIPELINES_FAILED",
"description": "Pipeline count with failed status",
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "PIPELINES_CANCELED",
"description": "Pipeline count with canceled status",
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "PIPELINES_SKIPPED",
"description": "Pipeline count with skipped status",
"isDeprecated": false,
"deprecationReason": null
}
],
"possibleTypes": null
......@@ -3224,6 +3224,10 @@ Possible identifier types for a measurement.
| `ISSUES` | Issue count |
| `MERGE_REQUESTS` | Merge request count |
| `PIPELINES` | Pipeline count |
| `PIPELINES_CANCELED` | Pipeline count with canceled status |
| `PIPELINES_FAILED` | Pipeline count with failed status |
| `PIPELINES_SKIPPED` | Pipeline count with skipped status |
| `PIPELINES_SUCCEEDED` | Pipeline count with success status |
| `PROJECTS` | Project count |
| `USERS` | User count |
......
......@@ -13,5 +13,13 @@ FactoryBot.define do
trait :group_count do
identifier { :groups }
end
trait :pipelines_succeeded_count do
identifier { :pipelines_succeeded }
end
trait :pipelines_skipped_count do
identifier { :pipelines_skipped }
end
end
end
......@@ -5,9 +5,11 @@ require 'spec_helper'
RSpec.describe Resolvers::Admin::Analytics::InstanceStatistics::MeasurementsResolver do
include GraphqlHelpers
let_it_be(:admin_user) { create(:user, :admin) }
let(:current_user) { admin_user }
describe '#resolve' do
let_it_be(:user) { create(:user) }
let_it_be(:admin_user) { create(:user, :admin) }
let_it_be(:project_measurement_new) { create(:instance_statistics_measurement, :project_count, recorded_at: 2.days.ago) }
let_it_be(:project_measurement_old) { create(:instance_statistics_measurement, :project_count, recorded_at: 10.days.ago) }
......@@ -39,6 +41,37 @@ RSpec.describe Resolvers::Admin::Analytics::InstanceStatistics::MeasurementsReso
end
end
end
context 'when requesting pipeline counts by pipeline status' do
let_it_be(:pipelines_succeeded_measurement) { create(:instance_statistics_measurement, :pipelines_succeeded_count, recorded_at: 2.days.ago) }
let_it_be(:pipelines_skipped_measurement) { create(:instance_statistics_measurement, :pipelines_skipped_count, recorded_at: 2.days.ago) }
subject { resolve_measurements({ identifier: identifier }, { current_user: current_user }) }
context 'filter for pipelines_succeeded' do
let(:identifier) { 'pipelines_succeeded' }
it { is_expected.to eq([pipelines_succeeded_measurement]) }
end
context 'filter for pipelines_skipped' do
let(:identifier) { 'pipelines_skipped' }
it { is_expected.to eq([pipelines_skipped_measurement]) }
end
context 'filter for pipelines_failed' do
let(:identifier) { 'pipelines_failed' }
it { is_expected.to be_empty }
end
context 'filter for pipelines_canceled' do
let(:identifier) { 'pipelines_canceled' }
it { is_expected.to be_empty }
end
end
end
def resolve_measurements(args = {}, context = {})
......
......@@ -20,7 +20,11 @@ RSpec.describe Analytics::InstanceStatistics::Measurement, type: :model do
issues: 3,
merge_requests: 4,
groups: 5,
pipelines: 6
pipelines: 6,
pipelines_succeeded: 7,
pipelines_failed: 8,
pipelines_canceled: 9,
pipelines_skipped: 10
}.with_indifferent_access)
end
end
......@@ -42,4 +46,28 @@ RSpec.describe Analytics::InstanceStatistics::Measurement, type: :model do
it { is_expected.to match_array([measurement_1, measurement_2]) }
end
end
describe '#measurement_identifier_values' do
subject { described_class.measurement_identifier_values.count }
context 'when the `store_ci_pipeline_counts_by_status` feature flag is off' do
let(:expected_count) { Analytics::InstanceStatistics::Measurement.identifiers.size - Analytics::InstanceStatistics::Measurement::EXPERIMENTAL_IDENTIFIERS.size }
before do
stub_feature_flags(store_ci_pipeline_counts_by_status: false)
end
it { is_expected.to eq(expected_count) }
end
context 'when the `store_ci_pipeline_counts_by_status` feature flag is on' do
let(:expected_count) { Analytics::InstanceStatistics::Measurement.identifiers.size }
before do
stub_feature_flags(store_ci_pipeline_counts_by_status: true)
end
it { is_expected.to eq(expected_count) }
end
end
end
......@@ -18,7 +18,7 @@ RSpec.describe Analytics::InstanceStatistics::CounterJobWorker do
it 'counts a scope and stores the result' do
subject
measurement = Analytics::InstanceStatistics::Measurement.first
measurement = Analytics::InstanceStatistics::Measurement.users.first
expect(measurement.recorded_at).to be_like_time(recorded_at)
expect(measurement.identifier).to eq('users')
expect(measurement.count).to eq(2)
......@@ -33,7 +33,7 @@ RSpec.describe Analytics::InstanceStatistics::CounterJobWorker do
it 'sets 0 as the count' do
subject
measurement = Analytics::InstanceStatistics::Measurement.first
measurement = Analytics::InstanceStatistics::Measurement.groups.first
expect(measurement.recorded_at).to be_like_time(recorded_at)
expect(measurement.identifier).to eq('groups')
expect(measurement.count).to eq(0)
......@@ -51,4 +51,20 @@ RSpec.describe Analytics::InstanceStatistics::CounterJobWorker do
expect { subject }.not_to change { Analytics::InstanceStatistics::Measurement.count }
end
context 'when pipelines_succeeded identifier is passed' do
let_it_be(:pipeline) { create(:ci_pipeline, :success) }
let(:successful_pipelines_measurement_identifier) { ::Analytics::InstanceStatistics::Measurement.identifiers.fetch(:pipelines_succeeded) }
let(:job_args) { [successful_pipelines_measurement_identifier, pipeline.id, pipeline.id, recorded_at] }
it 'counts successful pipelines' do
subject
measurement = Analytics::InstanceStatistics::Measurement.pipelines_succeeded.first
expect(measurement.recorded_at).to be_like_time(recorded_at)
expect(measurement.identifier).to eq('pipelines_succeeded')
expect(measurement.count).to eq(1)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment