Commit 3b54b2df authored by Pavel Shutsin's avatar Pavel Shutsin

Update devops adoption snapshots daily

So we can get information displayed faster
parent 54b1afcb
...@@ -477,7 +477,7 @@ production: &base ...@@ -477,7 +477,7 @@ production: &base
ee_cron_jobs: ee_cron_jobs:
# Schedule snapshots for all devops adoption segments # Schedule snapshots for all devops adoption segments
analytics_devops_adoption_create_all_snapshots_worker: analytics_devops_adoption_create_all_snapshots_worker:
cron: 0 0 1 * * cron: 0 4 * * *
# Snapshot active users statistics # Snapshot active users statistics
historical_data_worker: historical_data_worker:
......
...@@ -543,7 +543,7 @@ Settings.cron_jobs['manage_evidence_worker']['job_class'] = 'Releases::ManageEvi ...@@ -543,7 +543,7 @@ Settings.cron_jobs['manage_evidence_worker']['job_class'] = 'Releases::ManageEvi
Gitlab.ee do Gitlab.ee do
Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 0 1 * *' Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 4 * * *'
Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['job_class'] = 'Analytics::DevopsAdoption::CreateAllSnapshotsWorker' Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['job_class'] = 'Analytics::DevopsAdoption::CreateAllSnapshotsWorker'
Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['active_user_count_threshold_worker']['cron'] ||= '0 12 * * *' Settings.cron_jobs['active_user_count_threshold_worker']['cron'] ||= '0 12 * * *'
......
...@@ -50,7 +50,9 @@ The DevOps Adoption tab shows you which segments of your organization are using ...@@ -50,7 +50,9 @@ The DevOps Adoption tab shows you which segments of your organization are using
- Deploys - Deploys
- Scanning - Scanning
Segments are arbitrary collections of GitLab groups that you define. You might define a segment to represent a small team, a large department, or a whole organization. You are limited to creating a maximum of 20 segments, and each segment is limited to a maximum of 20 groups. Buttons to manage your segments appear in the DevOps Adoption section of the page. Segments are arbitrary collections of GitLab groups that you define. You might define a segment to represent a small team, a large department, or a whole organization.
You are limited to creating a maximum of 20 segments, and each segment is limited to a maximum of 20 groups.
Buttons to manage your segments appear in the DevOps Adoption section of the page.
DevOps Adoption allows you to: DevOps Adoption allows you to:
......
...@@ -25,6 +25,7 @@ class Analytics::DevopsAdoption::Snapshot < ApplicationRecord ...@@ -25,6 +25,7 @@ class Analytics::DevopsAdoption::Snapshot < ApplicationRecord
end end
scope :for_month, -> (month_date) { where(end_time: month_date.end_of_month) } scope :for_month, -> (month_date) { where(end_time: month_date.end_of_month) }
scope :not_finalized, -> { where(arel_table[:recorded_at].lteq(arel_table[:end_time])) }
def start_time def start_time
end_time.beginning_of_month end_time.beginning_of_month
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
module Analytics module Analytics
module DevopsAdoption module DevopsAdoption
# Schedules update of snapshots for all segments
class CreateAllSnapshotsWorker class CreateAllSnapshotsWorker
include ApplicationWorker include ApplicationWorker
# This worker does not perform work scoped to a context # This worker does not perform work scoped to a context
...@@ -14,10 +15,8 @@ module Analytics ...@@ -14,10 +15,8 @@ module Analytics
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def perform def perform
range_end = 1.month.ago.end_of_month
::Analytics::DevopsAdoption::Segment.all.pluck(:id).each.with_index do |segment_id, i| ::Analytics::DevopsAdoption::Segment.all.pluck(:id).each.with_index do |segment_id, i|
CreateSnapshotWorker.perform_in(i * WORKERS_GAP, segment_id, range_end) CreateSnapshotWorker.perform_in(i * WORKERS_GAP, segment_id, nil)
end end
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
......
...@@ -2,17 +2,39 @@ ...@@ -2,17 +2,39 @@
module Analytics module Analytics
module DevopsAdoption module DevopsAdoption
# Updates all pending snapshots for given segment (from previous month)
# and creates or update snapshot for current month
class CreateSnapshotWorker class CreateSnapshotWorker
include ApplicationWorker include ApplicationWorker
feature_category :devops_reports feature_category :devops_reports
idempotent! idempotent!
def perform(segment_id, range_end) # range_end was deprecated and must be removed in 14.0
segment = ::Analytics::DevopsAdoption::Segment.find(segment_id) #
def perform(segment_id, _deprecated_range_end = nil)
segment = Segment.find(segment_id)
::Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService.new(segment: segment, range_end: range_end).execute pending_ranges(segment).each do |range_end|
Snapshots::CalculateAndSaveService.new(segment: segment, range_end: range_end).execute
end
end end
private
# rubocop: disable CodeReuse/ActiveRecord
def pending_ranges(segment)
end_times = segment.snapshots.not_finalized.pluck(:end_time)
now = Time.zone.now
if !now.end_of_month.to_date.in?(end_times.map(&:to_date)) && now.day > 1
end_times << now.end_of_month
end
end_times
end
# rubocop: enable CodeReuse/ActiveRecord
end end
end end
end end
---
title: Update devops adoption snapshots daily
merge_request: 50568
author:
type: changed
...@@ -30,12 +30,11 @@ Gitlab::Seeder.quiet do ...@@ -30,12 +30,11 @@ Gitlab::Seeder.quiet do
booleans = [true, false] booleans = [true, false]
# create snapshots for the last 5 months # create snapshots for the last 5 months
5.downto(1).each do |index| 4.downto(0).each do |index|
end_time = index.months.ago.at_end_of_month end_time = index.months.ago.at_end_of_month
recorded_at = end_time + 1.day
segments.each do |segment| segments.each do |segment|
Analytics::DevopsAdoption::Snapshot.create!( calculated_data = {
segment: segment, segment: segment,
issue_opened: booleans.sample, issue_opened: booleans.sample,
merge_request_opened: booleans.sample, merge_request_opened: booleans.sample,
...@@ -44,11 +43,11 @@ Gitlab::Seeder.quiet do ...@@ -44,11 +43,11 @@ Gitlab::Seeder.quiet do
pipeline_succeeded: booleans.sample, pipeline_succeeded: booleans.sample,
deploy_succeeded: booleans.sample, deploy_succeeded: booleans.sample,
security_scan_succeeded: booleans.sample, security_scan_succeeded: booleans.sample,
recorded_at: recorded_at, recorded_at: [end_time + 1.day, Time.zone.now].min,
end_time: end_time end_time: end_time
) }
segment.update!(last_recorded_at: recorded_at) Analytics::DevopsAdoption::Snapshots::CreateService.new(params: calculated_data).execute
end end
end end
end end
......
...@@ -37,6 +37,16 @@ RSpec.describe Analytics::DevopsAdoption::Snapshot, type: :model do ...@@ -37,6 +37,16 @@ RSpec.describe Analytics::DevopsAdoption::Snapshot, type: :model do
end end
end end
describe '.not_finalized' do
it 'returns all snapshots which were recorded earlier than snapshot end_time' do
segment = create(:devops_adoption_segment)
snapshot1 = create(:devops_adoption_snapshot, segment: segment, recorded_at: 1.day.ago, end_time: Time.zone.now)
create(:devops_adoption_snapshot, segment: segment, recorded_at: 1.day.ago, end_time: 2.days.ago)
expect(described_class.not_finalized).to match_array([snapshot1])
end
end
describe '#start_time' do describe '#start_time' do
subject(:segment) { described_class.new(end_time: end_time) } subject(:segment) { described_class.new(end_time: end_time) }
......
...@@ -11,9 +11,8 @@ RSpec.describe Analytics::DevopsAdoption::CreateAllSnapshotsWorker do ...@@ -11,9 +11,8 @@ RSpec.describe Analytics::DevopsAdoption::CreateAllSnapshotsWorker do
it 'schedules workers for each individual segment' do it 'schedules workers for each individual segment' do
freeze_time do freeze_time do
end_time = 1.month.ago.end_of_month expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(0, segment1.id, nil)
expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(0, segment1.id, end_time) expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(5, segment2.id, nil)
expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(5, segment2.id, end_time)
worker.perform worker.perform
end end
......
...@@ -7,14 +7,70 @@ RSpec.describe Analytics::DevopsAdoption::CreateSnapshotWorker do ...@@ -7,14 +7,70 @@ RSpec.describe Analytics::DevopsAdoption::CreateSnapshotWorker do
describe "#perform" do describe "#perform" do
let!(:segment) { create :devops_adoption_segment } let!(:segment) { create :devops_adoption_segment }
let!(:range_end) { 1.day.ago }
it 'calls for Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService service' do let!(:pending_snapshot) do
expect_next_instance_of(::Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: range_end) do |instance| create(:devops_adoption_snapshot,
expect(instance).to receive(:execute) segment: segment,
end_time: DateTime.parse('2020-10-01').end_of_month,
recorded_at: DateTime.parse('2020-10-20')).reload
end
let!(:finalized_snapshot) do
create(:devops_adoption_snapshot,
segment: segment,
end_time: DateTime.parse('2020-09-01').end_of_month,
recorded_at: DateTime.parse('2020-10-20')).reload
end
let(:service_mock) { instance_double('Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService', execute: true) }
it 'updates metrics for all not finalized snapshots and current month' do
freeze_time do
allow_next_instance_of(Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: pending_snapshot.end_time) do |instance|
expect(instance).to receive(:execute)
end
allow_next_instance_of(Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: finalized_snapshot.end_time) do |instance|
expect(instance).not_to receive(:execute)
end
allow_next_instance_of(Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: Time.zone.now.end_of_month) do |instance|
expect(instance).to receive(:execute)
end
worker.perform(segment.id)
end end
end
worker.perform(segment.id, range_end) context 'when metric for current month already exists' do
it 'calls for current month calculation only once' do
travel_to(pending_snapshot.recorded_at + 1.day) do
allow_next_instance_of(Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: pending_snapshot.end_time) do |instance|
expect(instance).to receive(:execute).once
end
allow_next_instance_of(Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: finalized_snapshot.end_time) do |instance|
expect(instance).not_to receive(:execute)
end
worker.perform(segment.id)
end
end
end
context 'when today is first day of the month' do
it 'doesnt update metrics for current month' do
travel_to((pending_snapshot.recorded_at + 1.month).beginning_of_month) do
allow_next_instance_of(Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: pending_snapshot.end_time) do |instance|
expect(instance).to receive(:execute)
end
allow_next_instance_of(Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: finalized_snapshot.end_time) do |instance|
expect(instance).not_to receive(:execute)
end
allow_next_instance_of(Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: Time.zone.now.end_of_month) do |instance|
expect(instance).not_to receive(:execute)
end
worker.perform(segment.id)
end
end
end end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment