Commit 212d28d9 authored by Pavel Shutsin's avatar Pavel Shutsin

Separate adoption segment namespaces

Restructures models to allow multiple
segments for the same namespace

Changelog: changed
parent 95a34047
# frozen_string_literal: true
class MakeSnapshotSegmentIdOptional < ActiveRecord::Migration[6.0]
def up
change_column_null(:analytics_devops_adoption_snapshots, :segment_id, true)
end
def down
change_column_null(:analytics_devops_adoption_snapshots, :segment_id, false)
end
end
# frozen_string_literal: true
require Rails.root.join('db', 'post_migrate', '20210430134202_copy_adoption_snapshot_namespace.rb')
class RequireSnapshotNamespace < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
def up
CopyAdoptionSnapshotNamespace.new.up
add_not_null_constraint(:analytics_devops_adoption_snapshots, :namespace_id)
end
def down
remove_not_null_constraint(:analytics_devops_adoption_snapshots, :namespace_id)
end
end
3bcc5ae97f3185ea33e568f42b90d1bfd31ac7c5126dab4580b64bd9b4603721
\ No newline at end of file
1944c983dd384029cef6e456108a1ccfdb9c991c65343d3b7f26aff51f244816
\ No newline at end of file
......@@ -9130,7 +9130,7 @@ ALTER SEQUENCE analytics_devops_adoption_segments_id_seq OWNED BY analytics_devo
CREATE TABLE analytics_devops_adoption_snapshots (
id bigint NOT NULL,
segment_id bigint NOT NULL,
segment_id bigint,
recorded_at timestamp with time zone NOT NULL,
issue_opened boolean NOT NULL,
merge_request_opened boolean NOT NULL,
......@@ -9142,7 +9142,8 @@ CREATE TABLE analytics_devops_adoption_snapshots (
end_time timestamp with time zone NOT NULL,
total_projects_count integer,
code_owners_used_count integer,
namespace_id integer
namespace_id integer,
CONSTRAINT check_3f472de131 CHECK ((namespace_id IS NOT NULL))
);
CREATE SEQUENCE analytics_devops_adoption_snapshots_id_seq
......@@ -8184,7 +8184,7 @@ Segment.
| ---- | ---- | ----------- |
| <a id="devopsadoptionsegmentid"></a>`id` | [`ID!`](#id) | ID of the segment. |
| <a id="devopsadoptionsegmentlatestsnapshot"></a>`latestSnapshot` | [`DevopsAdoptionSnapshot`](#devopsadoptionsnapshot) | The latest adoption metrics for the segment. |
| <a id="devopsadoptionsegmentnamespace"></a>`namespace` | [`Namespace`](#namespace) | Segment namespace. |
| <a id="devopsadoptionsegmentnamespace"></a>`namespace` | [`Namespace`](#namespace) | Namespace which should be calculated. |
### `DevopsAdoptionSnapshot`
......
......@@ -12,16 +12,17 @@ module Types
field :id, GraphQL::ID_TYPE, null: false,
description: "ID of the segment."
field :namespace, Types::NamespaceType, null: true, description: 'Segment namespace.'
field :namespace, Types::NamespaceType, null: true,
description: 'Namespace which should be calculated.'
field :latest_snapshot, SnapshotType, null: true,
description: 'The latest adoption metrics for the segment.'
def latest_snapshot
BatchLoader::GraphQL.for(object.id).batch(key: :devops_adoption_latest_snapshots) do |ids, loader, args|
BatchLoader::GraphQL.for(object.namespace_id).batch(key: :devops_adoption_latest_snapshots) do |ids, loader, args|
snapshots = ::Analytics::DevopsAdoption::Snapshot
.latest_snapshot_for_segment_ids(ids)
.index_by(&:segment_id)
.latest_snapshot_for_namespace_ids(ids)
.index_by(&:namespace_id)
ids.each do |id|
loader.call(id, snapshots[id])
......
......@@ -6,17 +6,17 @@ class Analytics::DevopsAdoption::Segment < ApplicationRecord
belongs_to :namespace
belongs_to :display_namespace, class_name: 'Namespace', optional: true
has_many :snapshots, inverse_of: :segment
has_one :latest_snapshot, -> { order(recorded_at: :desc) }, inverse_of: :segment, class_name: 'Snapshot'
has_many :snapshots, foreign_key: :namespace_id, primary_key: :namespace_id
ignore_column :name, remove_with: '14.0', remove_after: '2021-05-22'
validates :namespace, uniqueness: true, presence: true
validates :namespace, uniqueness: { scope: :display_namespace_id }, presence: true
scope :ordered_by_name, -> { includes(:namespace).order('"namespaces"."name" ASC') }
scope :for_namespaces, -> (namespaces) { where(namespace_id: namespaces) }
scope :for_parent, -> (namespace) { for_namespaces(namespace.self_and_descendants) }
# Remove in %14.0 with https://gitlab.com/gitlab-org/gitlab/-/issues/329521
before_validation -> { self.display_namespace = namespace }
ignore_column :last_recorded_at, remove_with: '14.2', remove_after: '2021-07-22'
def latest_snapshot
snapshots.by_end_time.first
end
end
# frozen_string_literal: true
class Analytics::DevopsAdoption::Snapshot < ApplicationRecord
belongs_to :segment, inverse_of: :snapshots
belongs_to :namespace, optional: true
include IgnorableColumns
validates :segment, presence: true
belongs_to :namespace
has_many :segments, foreign_key: :namespace_id, primary_key: :namespace_id
validates :namespace, presence: true
validates :recorded_at, presence: true
validates :end_time, presence: true
validates :issue_opened, inclusion: { in: [true, false] }
......@@ -17,21 +20,21 @@ class Analytics::DevopsAdoption::Snapshot < ApplicationRecord
validates :total_projects_count, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, allow_nil: true
validates :code_owners_used_count, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, allow_nil: true
scope :latest_snapshot_for_segment_ids, -> (ids) do
ignore_column :segment_id, remove_with: '14.2', remove_after: '2021-07-22'
scope :latest_snapshot_for_namespace_ids, -> (ids) do
inner_select = model
.default_scoped
.distinct
.select("FIRST_VALUE(id) OVER (PARTITION BY segment_id ORDER BY end_time DESC) as id")
.where(segment_id: ids)
.select("FIRST_VALUE(id) OVER (PARTITION BY namespace_id ORDER BY end_time DESC) as id")
.where(namespace_id: ids)
joins("INNER JOIN (#{inner_select.to_sql}) latest_snapshots ON latest_snapshots.id = analytics_devops_adoption_snapshots.id")
end
scope :for_month, -> (month_date) { where(end_time: month_date.end_of_month) }
scope :not_finalized, -> { where(arel_table[:recorded_at].lteq(arel_table[:end_time])) }
# Remove in %14.0 with https://gitlab.com/gitlab-org/gitlab/-/issues/329521
before_save -> { self.namespace = segment.namespace }
scope :by_end_time, -> { order(end_time: :desc) }
def start_time
end_time.beginning_of_month
......
......@@ -29,7 +29,8 @@ module Analytics
def services
@services ||= params[:namespaces].map do |namespace|
FindOrCreateService.new(current_user: current_user, params: { namespace: namespace })
FindOrCreateService.new(current_user: current_user,
params: { namespace: namespace, display_namespace: namespace })
end
end
end
......
......@@ -15,7 +15,7 @@ module Analytics
def execute
authorize!
segment.assign_attributes(namespace: namespace)
segment.assign_attributes(namespace: namespace, display_namespace: namespace)
if segment.save
Analytics::DevopsAdoption::CreateSnapshotWorker.perform_async(segment.id)
......
......@@ -11,10 +11,11 @@ module Analytics
@current_user = current_user
end
# rubocop: disable CodeReuse/ActiveRecord
def execute
authorize!
segment = Analytics::DevopsAdoption::Segment.find_by_namespace_id(namespace.id)
segment = Analytics::DevopsAdoption::Segment.find_by(namespace_id: namespace.id, display_namespace_id: namespace.id)
if segment
ServiceResponse.success(payload: { segment: segment })
......@@ -22,6 +23,7 @@ module Analytics
create_service.execute
end
end
# rubocop: enable CodeReuse/ActiveRecord
def authorize!
create_service.authorize!
......
......@@ -5,8 +5,8 @@ module Analytics
module Snapshots
class UpdateService
ALLOWED_ATTRIBUTES = ([
:segment,
:segment_id,
:namespace,
:namespace_id,
:end_time,
:recorded_at
] + SnapshotCalculator::ADOPTION_METRICS).freeze
......@@ -17,17 +17,9 @@ module Analytics
end
def execute
success = false
snapshot.assign_attributes(attributes)
ActiveRecord::Base.transaction do
snapshot.assign_attributes(attributes)
success = snapshot.save && snapshot.segment.update(last_recorded_at: snapshot.recorded_at)
raise ActiveRecord::Rollback unless success
end
if success
if snapshot.save
ServiceResponse.success(payload: response_payload)
else
ServiceResponse.error(message: 'Validation error', payload: response_payload)
......
......@@ -9,8 +9,8 @@ Gitlab::Seeder.quiet do
ActiveRecord::Base.transaction do
segments = [
Analytics::DevopsAdoption::Segment.create(namespace: groups.first),
Analytics::DevopsAdoption::Segment.create(namespace: groups.last)
Analytics::DevopsAdoption::Segment.create(namespace: groups.first, display_namespace: groups.first),
Analytics::DevopsAdoption::Segment.create(namespace: groups.last, display_namespace: groups.last)
]
if segments.any?(&:invalid?)
......@@ -27,7 +27,7 @@ Gitlab::Seeder.quiet do
segments.each do |segment|
calculated_data = {
segment: segment,
namespace: segment.namespace,
issue_opened: booleans.sample,
merge_request_opened: booleans.sample,
merge_request_approved: booleans.sample,
......
......@@ -30,7 +30,7 @@ module Analytics
end
def calculate
params = { recorded_at: Time.zone.now, end_time: range_end, segment: segment }
params = { recorded_at: Time.zone.now, end_time: range_end, namespace: segment.namespace }
BOOLEAN_METRICS.each do |metric|
params[metric] = snapshot&.public_send(metric) || send(metric) # rubocop:disable GitlabSecurity/PublicSend
......
......@@ -3,5 +3,9 @@
FactoryBot.define do
factory :devops_adoption_segment, class: 'Analytics::DevopsAdoption::Segment' do
association :namespace, factory: :group
after(:build) do |instance|
instance.display_namespace = instance.namespace
end
end
end
......@@ -2,7 +2,7 @@
FactoryBot.define do
factory :devops_adoption_snapshot, class: 'Analytics::DevopsAdoption::Snapshot' do
association :segment, factory: :devops_adoption_segment
association :namespace
recorded_at { Time.zone.now }
end_time { 1.month.ago.end_of_month }
......
......@@ -169,7 +169,7 @@ RSpec.describe Analytics::DevopsAdoption::SnapshotCalculator do
context 'when snapshot already exists' do
subject(:data) { described_class.new(segment: segment, range_end: range_end, snapshot: snapshot).calculate }
let(:snapshot) { create :devops_adoption_snapshot, segment: segment, issue_opened: true, merge_request_opened: false, total_projects_count: 1 }
let(:snapshot) { create :devops_adoption_snapshot, namespace: segment.namespace, issue_opened: true, merge_request_opened: false, total_projects_count: 1 }
context 'for boolean metrics' do
let!(:fresh_merge_request) { create(:merge_request, source_project: project, created_at: 3.weeks.ago(range_end)) }
......
......@@ -8,23 +8,14 @@ RSpec.describe Analytics::DevopsAdoption::Segment, type: :model do
it { is_expected.to have_many(:snapshots) }
it { is_expected.to belong_to(:namespace) }
it { is_expected.to belong_to(:display_namespace) }
end
describe 'validation' do
subject { build(:devops_adoption_segment) }
it { is_expected.to validate_presence_of(:namespace) }
it { is_expected.to validate_uniqueness_of(:namespace) }
end
describe '#display_namespace' do
subject { build(:devops_adoption_segment) }
it 'fills display_namespace with namespace on save' do
expect do
subject.save!
end.to change { subject.display_namespace }.to(subject.namespace)
end
it { is_expected.to validate_uniqueness_of(:namespace).scoped_to(:display_namespace_id) }
end
describe '.ordered_by_name' do
......@@ -70,9 +61,9 @@ RSpec.describe Analytics::DevopsAdoption::Segment, type: :model do
describe '.latest_snapshot' do
it 'loads the latest snapshot' do
segment = create(:devops_adoption_segment)
latest_snapshot = create(:devops_adoption_snapshot, segment: segment, recorded_at: 2.days.ago)
create(:devops_adoption_snapshot, segment: segment, recorded_at: 5.days.ago)
create(:devops_adoption_snapshot, segment: create(:devops_adoption_segment), recorded_at: 1.hour.ago)
latest_snapshot = create(:devops_adoption_snapshot, namespace: segment.namespace, end_time: 2.days.ago)
create(:devops_adoption_snapshot, namespace: segment.namespace, end_time: 5.days.ago)
create(:devops_adoption_snapshot, end_time: 1.hour.ago)
expect(segment.latest_snapshot).to eq(latest_snapshot)
end
......
......@@ -3,25 +3,25 @@
require 'spec_helper'
RSpec.describe Analytics::DevopsAdoption::Snapshot, type: :model do
it { is_expected.to belong_to(:segment) }
it { is_expected.to belong_to(:namespace) }
it { is_expected.to validate_presence_of(:segment) }
it { is_expected.to validate_presence_of(:namespace) }
it { is_expected.to validate_presence_of(:recorded_at) }
it { is_expected.to validate_presence_of(:end_time) }
describe '.latest_snapshot_for_segment_ids' do
it 'returns the latest snapshot for the given segment ids based on snapshot end_time' do
segment_1 = create(:devops_adoption_segment)
segment_1_latest_snapshot = create(:devops_adoption_snapshot, segment: segment_1, end_time: 1.week.ago)
create(:devops_adoption_snapshot, segment: segment_1, end_time: 2.weeks.ago)
describe '.latest_snapshot_for_namespace_ids' do
it 'returns the latest snapshot for the given namespace ids based on snapshot end_time' do
group1 = create(:group)
group1_latest_snapshot = create(:devops_adoption_snapshot, namespace: group1, end_time: 1.week.ago)
create(:devops_adoption_snapshot, namespace: group1, end_time: 2.weeks.ago)
segment_2 = create(:devops_adoption_segment)
segment_2_latest_snapshot = create(:devops_adoption_snapshot, segment: segment_2, end_time: 1.year.ago)
create(:devops_adoption_snapshot, segment: segment_2, end_time: 2.years.ago)
group2 = create(:group)
group2_latest_snapshot = create(:devops_adoption_snapshot, namespace: group2, end_time: 1.year.ago)
create(:devops_adoption_snapshot, namespace: group2, end_time: 2.years.ago)
latest_snapshot_for_segments = described_class.latest_snapshot_for_segment_ids([segment_1.id, segment_2.id])
latest_snapshots = described_class.latest_snapshot_for_namespace_ids([group1.id, group2.id])
expect(latest_snapshot_for_segments).to match_array([segment_1_latest_snapshot, segment_2_latest_snapshot])
expect(latest_snapshots).to match_array([group1_latest_snapshot, group2_latest_snapshot])
end
end
......@@ -39,32 +39,21 @@ RSpec.describe Analytics::DevopsAdoption::Snapshot, type: :model do
describe '.not_finalized' do
it 'returns all snapshots which were recorded earlier than snapshot end_time' do
segment = create(:devops_adoption_segment)
snapshot1 = create(:devops_adoption_snapshot, segment: segment, recorded_at: 1.day.ago, end_time: Time.zone.now)
create(:devops_adoption_snapshot, segment: segment, recorded_at: 1.day.ago, end_time: 2.days.ago)
snapshot1 = create(:devops_adoption_snapshot, recorded_at: 1.day.ago, end_time: Time.zone.now)
create(:devops_adoption_snapshot, recorded_at: 1.day.ago, end_time: 2.days.ago)
expect(described_class.not_finalized).to match_array([snapshot1])
end
end
describe '#start_time' do
subject(:segment) { described_class.new(end_time: end_time) }
subject(:snapshot) { described_class.new(end_time: end_time) }
let(:end_time) { DateTime.parse('2020-12-17') }
let(:expected_start_time) { DateTime.parse('2020-12-01') }
it 'is start of the month of end_time' do
expect(segment.start_time).to eq(expected_start_time)
end
end
describe '#namespace' do
subject { build(:devops_adoption_snapshot) }
it 'fills namespace with segments namespace on save' do
expect do
subject.save!
end.to change { subject.namespace }.to(subject.segment.namespace)
expect(snapshot.start_time).to eq(expected_start_time)
end
end
end
......@@ -9,9 +9,11 @@ RSpec.describe 'DevopsAdoptionSegments' do
let_it_be(:group) { create(:group, name: 'my-group') }
let_it_be(:segment) do
create(:devops_adoption_segment, namespace: group).tap do |segment|
create(:devops_adoption_snapshot, segment: segment, issue_opened: true, merge_request_opened: false)
end
create(:devops_adoption_segment, namespace: group)
end
let_it_be(:snapshot) do
create(:devops_adoption_snapshot, namespace: group, issue_opened: true, merge_request_opened: false)
end
let(:query) do
......
......@@ -25,7 +25,7 @@ RSpec.describe Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService do
end
context 'when a snapshot for given range already exists' do
let(:snapshot) { create :devops_adoption_snapshot, end_time: range_end, segment: segment }
let(:snapshot) { create :devops_adoption_snapshot, end_time: range_end, namespace: segment.namespace }
it 'updates the snapshot with whatever snapshot calculator returns' do
expect_next_instance_of(Analytics::DevopsAdoption::Snapshots::UpdateService, snapshot: snapshot, params: 'calculated_data') do |service|
......
......@@ -17,16 +17,15 @@ RSpec.describe Analytics::DevopsAdoption::Snapshots::CreateService do
params[:recorded_at] = Time.zone.now
params[:end_time] = 1.month.ago.end_of_month
params[:segment] = segment
params[:namespace] = segment.namespace
params
end
let(:segment) { create(:devops_adoption_segment, last_recorded_at: 1.year.ago) }
let(:segment) { create(:devops_adoption_segment) }
it 'persists the snapshot & updates segment last recorded at date' do
it 'persists the snapshot' do
expect(subject).to be_success
expect(snapshot).to have_attributes(params)
expect(snapshot.segment.reload.last_recorded_at).to be_like_time(snapshot.recorded_at)
end
context 'when params are invalid' do
......@@ -36,7 +35,6 @@ RSpec.describe Analytics::DevopsAdoption::Snapshots::CreateService do
expect(subject).to be_error
expect(subject.message).to eq('Validation error')
expect(snapshot).not_to be_persisted
expect(snapshot.segment.reload.last_recorded_at).not_to eq nil
end
end
end
......@@ -5,8 +5,7 @@ require 'spec_helper'
RSpec.describe Analytics::DevopsAdoption::Snapshots::UpdateService do
subject(:service_response) { described_class.new(snapshot: snapshot, params: params).execute }
let(:snapshot) { create(:devops_adoption_snapshot, segment: segment) }
let(:segment) { create(:devops_adoption_segment, last_recorded_at: 1.year.ago) }
let(:snapshot) { create(:devops_adoption_snapshot) }
let(:params) do
params = {}
......@@ -17,14 +16,13 @@ RSpec.describe Analytics::DevopsAdoption::Snapshots::UpdateService do
params[attribute] = i
end
params[:recorded_at] = Time.zone.now
params[:segment] = segment
params[:namespace] = snapshot.namespace
params
end
it 'updates the snapshot & updates segment last recorded at date' do
it 'updates the snapshot' do
expect(subject).to be_success
expect(snapshot).to have_attributes(params)
expect(snapshot.segment.reload.last_recorded_at).to be_like_time(snapshot.recorded_at)
end
context 'when params are invalid' do
......@@ -34,7 +32,6 @@ RSpec.describe Analytics::DevopsAdoption::Snapshots::UpdateService do
expect(subject).to be_error
expect(subject.message).to eq('Validation error')
expect(snapshot.reload.recorded_at).not_to be_nil
expect(snapshot.segment.reload.last_recorded_at).not_to eq nil
end
end
end
......@@ -10,14 +10,14 @@ RSpec.describe Analytics::DevopsAdoption::CreateSnapshotWorker do
let!(:pending_snapshot) do
create(:devops_adoption_snapshot,
segment: segment,
namespace: segment.namespace,
end_time: DateTime.parse('2020-10-01').end_of_month,
recorded_at: DateTime.parse('2020-10-20')).reload
end
let!(:finalized_snapshot) do
create(:devops_adoption_snapshot,
segment: segment,
namespace: segment.namespace,
end_time: DateTime.parse('2020-09-01').end_of_month,
recorded_at: DateTime.parse('2020-10-20')).reload
end
......
# frozen_string_literal: true
#
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20210430134202_copy_adoption_snapshot_namespace.rb')
RSpec.describe CopyAdoptionSnapshotNamespace, :migration do
RSpec.describe CopyAdoptionSnapshotNamespace, :migration, schema: 20210430124630 do
let(:namespaces_table) { table(:namespaces) }
let(:segments_table) { table(:analytics_devops_adoption_segments) }
let(:snapshots_table) { table(:analytics_devops_adoption_snapshots) }
before do
it 'updates all snapshots without namespace set' do
namespaces_table.create!(id: 123, name: 'group1', path: 'group1')
namespaces_table.create!(id: 124, name: 'group2', path: 'group2')
......@@ -19,9 +19,7 @@ RSpec.describe CopyAdoptionSnapshotNamespace, :migration do
create_snapshot(id: 1, segment_id: 1)
create_snapshot(id: 2, segment_id: 2)
create_snapshot(id: 3, segment_id: 2, namespace_id: 123)
end
it 'updates all snapshots without namespace set' do
migrate!
expect(snapshots_table.find(1).namespace_id).to eq 123
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment