Commit 8d8e2098 authored by Rubén Dávila's avatar Rubén Dávila Committed by James Lopez

Notify owners of namespace when CI usage is reaching his limit

Mutliple levels of notifications are suppoerted.
We just send the notification when a new level has been reached.
parent f63170de
......@@ -2116,6 +2116,7 @@ ActiveRecord::Schema.define(version: 20190703130053) do
t.boolean "auto_devops_enabled"
t.integer "extra_shared_runners_minutes_limit"
t.datetime_with_timezone "last_ci_minutes_notification_at"
t.integer "last_ci_minutes_usage_notification_level"
t.index ["created_at"], name: "index_namespaces_on_created_at", using: :btree
t.index ["custom_project_templates_group_id", "type"], name: "index_namespaces_on_custom_project_templates_group_id_and_type", where: "(custom_project_templates_group_id IS NOT NULL)", using: :btree
t.index ["file_template_project_id"], name: "index_namespaces_on_file_template_project_id", using: :btree
......
# frozen_string_literal: true
module EE
module RunnersHelper
def purchase_shared_runner_minutes_link(user, project)
if ::Gitlab.com? && can?(user, :admin_project, project)
link_to(_("Click here"), EE::SUBSCRIPTIONS_PLANS_URL, target: '_blank', rel: 'noopener') + s_("Pipelines| to purchase more minutes.")
def ci_usage_warning_message(namespace, project)
message = ci_usage_base_message(namespace)
return unless message
if ::Gitlab.com? && can?(current_user, :admin_project, project)
message += " #{purchase_shared_runner_minutes_link}"
elsif namespace.shared_runners_minutes_used?
message += s_('Pipelines|Pipelines will not run anymore on shared Runners.')
end
message.html_safe
end
def ci_usage_warning_class(namespace)
if EE::Namespace::CI_USAGE_ALERT_LEVELS.min == namespace.last_ci_minutes_usage_notification_level
'alert-danger'
else
s_("Pipelines|Pipelines will not run anymore on shared Runners.")
'alert-warning'
end
end
private
def purchase_shared_runner_minutes_link
link = link_to(_("Click here"), EE::SUBSCRIPTIONS_PLANS_URL, target: '_blank', rel: 'noopener')
link + s_("Pipelines| to purchase more minutes.")
end
def ci_usage_base_message(namespace)
if namespace.shared_runners_minutes_used?
s_("Pipelines|%{namespace_name} has exceeded its pipeline minutes quota.") % { namespace_name: namespace.name }
elsif namespace.last_ci_minutes_usage_notification_level
s_("Pipelines|%{namespace_name} has less than %{notification_level}%% of CI minutes available.") % { namespace_name: namespace.name, notification_level: namespace.last_ci_minutes_usage_notification_level }
end
end
end
......
# frozen_string_literal: true
class CiMinutesUsageMailer < BaseMailer
def notify(namespace_name, contact_email)
def notify(namespace_name, recipients)
@namespace_name = namespace_name
mail(
to: contact_email,
bcc: recipients,
subject: "GitLab CI Runner Minutes quota for #{namespace_name} has run out"
)
end
def notify_limit(namespace_name, recipients, percentage_of_available_mins)
@namespace_name = namespace_name
@percentage_of_available_mins = percentage_of_available_mins
mail(
bcc: recipients,
subject: "GitLab CI Runner Minutes quota for #{namespace_name} has \
less than #{percentage_of_available_mins}% available"
)
end
end
......@@ -27,6 +27,8 @@ module EE
LICENSE_PLANS_TO_NAMESPACE_PLANS = NAMESPACE_PLANS_TO_LICENSE_PLANS.invert.freeze
PLANS = NAMESPACE_PLANS_TO_LICENSE_PLANS.keys.freeze
CI_USAGE_ALERT_LEVELS = [30, 5].freeze
prepended do
include EachBatch
......
......@@ -2,22 +2,53 @@
class CiMinutesUsageNotifyService < BaseService
def execute
notify_on_total_usage
notify_on_partial_usage
end
private
def recipients
namespace.user? ? [namespace.owner.email] : namespace.owners.pluck(:email) # rubocop:disable CodeReuse/ActiveRecord
end
def notify_on_total_usage
return unless namespace.shared_runners_minutes_used? && namespace.last_ci_minutes_notification_at.nil?
namespace.update_columns(last_ci_minutes_notification_at: Time.now)
owners.each do |user|
CiMinutesUsageMailer.notify(namespace.name, user.email).deliver_later
end
CiMinutesUsageMailer.notify(namespace.name, recipients).deliver_later
end
private
def notify_on_partial_usage
return if namespace.shared_runners_minutes_used?
return if namespace.last_ci_minutes_usage_notification_level == current_alert_level
return if alert_levels.max < ci_minutes_percent_left
namespace.update_columns(last_ci_minutes_usage_notification_level: current_alert_level)
CiMinutesUsageMailer.notify_limit(namespace.name, recipients, current_alert_level).deliver_later
end
def namespace
@namespace ||= project.shared_runners_limit_namespace
end
def owners
namespace.user? ? [namespace.owner] : namespace.owners
def ci_minutes_percent_left
quota = namespace.actual_shared_runners_minutes_limit
used = namespace.shared_runners_minutes.to_i
minutes_left = quota - used
return 0 if minutes_left <= 0
(minutes_left.to_f * 100) / quota.to_f
end
def alert_levels
@alert_levels ||= EE::Namespace::CI_USAGE_ALERT_LEVELS.sort
end
def current_alert_level
@current_alert_level ||= alert_levels.find { |level| level >= ci_minutes_percent_left }
end
end
%p
This is an automated notification to let you know that your CI Runner Minutes quota for "#{@namespace_name}" is below #{@percentage_of_available_mins}%.
%p
Click #{link_to('here', EE::SUBSCRIPTIONS_PLANS_URL)} to purchase more minutes.
%p
If you need assistance, please contact #{link_to('GitLab support', 'https://support.gitlab.com')}.
This is an automated notification to let you know that your CI Runner Minutes
quota for "<%= @namespace_name %>" is below <%= @percentage_of_available_mins %>%.
Please visit <%= EE::SUBSCRIPTIONS_PLANS_URL %> to purchase more minutes.
If you need assistance, please contact GitLab support (https://support.gitlab.com).
......@@ -3,11 +3,11 @@
- scope = (project || namespace).full_path
- has_limit = (project || namespace).shared_runners_minutes_limit_enabled?
- can_see_status = project.nil? || can?(current_user, :create_pipeline, project)
- ci_warning_message = ci_usage_warning_message(namespace, project)
- if cookies[:hide_shared_runner_quota_message].blank? && has_limit && namespace.shared_runners_minutes_used? && can_see_status
.shared-runner-quota-message.alert.alert-warning.d-none.d-sm-block{ data: { scope: scope } }
= namespace.name
has exceeded its pipeline minutes quota. #{purchase_shared_runner_minutes_link(current_user, project)}
- if cookies[:hide_shared_runner_quota_message].blank? && has_limit && can_see_status && ci_warning_message.present?
.shared-runner-quota-message.alert.d-none.d-sm-block{ class: ci_usage_warning_class(namespace), data: { scope: scope } }
= ci_warning_message
.float-right
= link_to 'Remind later', '#', class: 'hide-shared-runner-limit-message alert-link'
......@@ -19,8 +19,9 @@ class ClearSharedRunnersMinutesWorker
.update_all("extra_shared_runners_minutes_limit = #{extra_minutes_left_sql} FROM namespace_statistics")
end
Namespace.where.not(last_ci_minutes_notification_at: nil).each_batch do |relation|
relation.update_all(last_ci_minutes_notification_at: nil)
Namespace.where('last_ci_minutes_notification_at IS NOT NULL OR last_ci_minutes_usage_notification_level IS NOT NULL')
.each_batch do |relation|
relation.update_all(last_ci_minutes_notification_at: nil, last_ci_minutes_usage_notification_level: nil)
end
NamespaceStatistics.where.not(shared_runners_seconds: 0)
......
---
title: Add notifications for CI Minutes quota limit approaching
merge_request: 14328
author:
type: added
......@@ -25,6 +25,7 @@ module EE
# Reset last_ci_minutes_notification_at if customer purchased extra CI minutes.
if params[:extra_shared_runners_minutes_limit].present?
update_attrs[:last_ci_minutes_notification_at] = nil
update_attrs[:last_ci_minutes_usage_notification_level] = nil
end
namespace.update(update_attrs)
......
......@@ -27,6 +27,24 @@ describe 'CI shared runner limits' do
end
context 'when limit is defined' do
before do
stub_const("EE::Namespace::CI_USAGE_ALERT_LEVELS", [30, 5])
end
context 'when usage has reached a notification level' do
let(:group) { create(:group, :with_build_minutes_limit, last_ci_minutes_usage_notification_level: 30) }
it 'displays a warning message on pipelines page' do
visit_project_pipelines
expect_quota_exceeded_alert("#{group.name} has less than 30% of CI minutes available.")
end
it 'displays a warning message on project homepage' do
visit_project_home
expect_quota_exceeded_alert("#{group.name} has less than 30% of CI minutes available.")
end
end
context 'when limit is exceeded' do
let(:group) { create(:group, :with_used_build_minutes_limit) }
......
# frozen_string_literal: true
require "spec_helper"
describe EE::RunnersHelper do
describe '.ci_usage_warning_message' do
let(:project) { create(:project, namespace: namespace) }
let(:minutes_used) { 0 }
let(:user) { create(:user) }
let(:namespace) do
create(:group, shared_runners_minutes_limit: 100)
end
let!(:statistics) do
create(:namespace_statistics, namespace: namespace, shared_runners_seconds: minutes_used * 60)
end
before do
allow(::Gitlab).to receive(:com?).and_return(true)
allow(helper).to receive(:current_user).and_return(user)
allow(helper).to receive(:can?).with(user, :admin_project, project) { false }
stub_const("EE::Namespace::CI_USAGE_ALERT_LEVELS", [50])
end
subject { helper.ci_usage_warning_message(namespace, project) }
context 'when CI minutes quota is above the warning limits' do
let(:minutes_used) { 40 }
it 'does not return a message' do
expect(subject).to be_nil
end
end
context 'when current user is an owner' do
before do
allow(helper).to receive(:can?).with(user, :admin_project, project) { true }
end
context 'when usage has reached first level of notification' do
before do
namespace.update_attribute(:last_ci_minutes_usage_notification_level, 50)
end
it 'shows the partial usage message' do
expect(subject).to match("#{namespace.name} has less than 50% of CI minutes available.")
expect(subject).to match('to purchase more minutes')
end
end
context 'when usage is above the quota' do
let(:minutes_used) { 120 }
it 'shows the total usage message' do
expect(subject).to match("#{namespace.name} has exceeded its pipeline minutes quota.")
expect(subject).to match('to purchase more minutes')
end
end
end
context 'when current user is not an owner' do
context 'when usage has reached first level of notification' do
before do
namespace.update_attribute(:last_ci_minutes_usage_notification_level, 50)
end
it 'shows the partial usage message without the purchase link' do
expect(subject).to match("#{namespace.name} has less than 50% of CI minutes available.")
expect(subject).not_to match('to purchase more minutes')
end
end
context 'when usage is above the quota' do
let(:minutes_used) { 120 }
it 'shows the total usage message without the purchase link' do
expect(subject).to match("#{namespace.name} has exceeded its pipeline minutes quota.")
expect(subject).not_to match('to purchase more minutes')
end
end
end
end
end
......@@ -157,15 +157,17 @@ describe API::Namespaces do
end
end
context 'when namespace has a value for last_ci_minutes_notification_at' do
before do
group1.update_attribute(:last_ci_minutes_notification_at, Time.now)
end
[:last_ci_minutes_notification_at, :last_ci_minutes_usage_notification_level].each do |attr|
context "when namespace has a value for #{attr}" do
before do
group1.update_attribute(attr, Time.now)
end
it 'resets that value when assigning extra CI minutes' do
expect do
put api("/namespaces/#{group1.full_path}", admin), params: { plan: 'silver', extra_shared_runners_minutes_limit: 1000 }
end.to change { group1.reload.last_ci_minutes_notification_at }.to(nil)
it 'resets that value when assigning extra CI minutes' do
expect do
put api("/namespaces/#{group1.full_path}", admin), params: { plan: 'silver', extra_shared_runners_minutes_limit: 1000 }
end.to change { group1.reload.send(attr) }.to(nil)
end
end
end
end
......
......@@ -16,28 +16,28 @@ describe CiMinutesUsageNotifyService do
shared_examples 'namespace with all CI minutes used' do
context 'when usage is over the quote' do
it 'sends the email to the owner' do
expect(CiMinutesUsageMailer).to receive(:notify).once.with(namespace.name, user.email).and_return(spy)
expect(CiMinutesUsageMailer).to receive(:notify).once.with(namespace.name, [user.email]).and_return(spy)
subject
end
end
end
let(:project) { create(:project, namespace: namespace) }
let(:user) { create(:user) }
let(:user_2) { create(:user) }
let(:ci_minutes_used) { 0 }
let!(:namespace_statistics) do
create(:namespace_statistics, namespace: namespace, shared_runners_seconds: ci_minutes_used * 60)
end
describe '#execute' do
let(:extra_ci_minutes) { 0 }
let(:namespace) do
create(:namespace, shared_runners_minutes_limit: 2000, extra_shared_runners_minutes_limit: extra_ci_minutes)
end
let(:project) { create(:project, namespace: namespace) }
let(:user) { create(:user) }
let(:user_2) { create(:user) }
let(:ci_minutes_used) { 0 }
let!(:namespace_statistics) do
create(:namespace_statistics, namespace: namespace, shared_runners_seconds: ci_minutes_used * 60)
end
subject { described_class.new(project).execute }
context 'with a personal namespace' do
......@@ -99,8 +99,9 @@ describe CiMinutesUsageNotifyService do
let(:ci_minutes_used) { 2001 }
it 'sends the email to all the owners' do
expect(CiMinutesUsageMailer).to receive(:notify).with(namespace.name, user.email).and_return(spy)
expect(CiMinutesUsageMailer).to receive(:notify).with(namespace.name, user_2.email).and_return(spy)
expect(CiMinutesUsageMailer).to receive(:notify)
.with(namespace.name, [user_2.email, user.email])
.and_return(spy)
subject
end
......@@ -120,4 +121,72 @@ describe CiMinutesUsageNotifyService do
end
end
end
describe 'CI usage limit approaching' do
let(:namespace) { create(:group, shared_runners_minutes_limit: 2000) }
def notify_owners
described_class.new(project).execute
end
shared_examples 'no notification is sent' do
it 'does not notify owners' do
expect(CiMinutesUsageMailer).not_to receive(:notify_limit)
notify_owners
end
end
shared_examples 'notification for custom level is sent' do |minutes_used, expected_level|
before do
namespace_statistics.update_attribute(:shared_runners_seconds, minutes_used * 60)
end
it 'notifies the the owners about it' do
expect(CiMinutesUsageMailer).to receive(:notify_limit)
.with(namespace.name, array_including(user_2.email, user.email), expected_level)
.and_call_original
notify_owners
end
end
before do
stub_const("EE::Namespace::CI_USAGE_ALERT_LEVELS", [30, 5])
namespace.add_owner(user)
namespace.add_owner(user_2)
end
context 'when available minutes are above notification levels' do
let(:ci_minutes_used) { 1000 }
it_behaves_like 'no notification is sent'
end
context 'when available minutes have reached the first level of alert' do
it_behaves_like 'notification for custom level is sent', 1500, 30
context 'when other Pipeline has finished but second level of alert has not been reached' do
before do
namespace_statistics.update_attribute(:shared_runners_seconds, 1500 * 60)
notify_owners
namespace_statistics.update_attribute(:shared_runners_seconds, 1600 * 60)
end
it_behaves_like 'no notification is sent'
end
end
context 'when available minutes have reached the second level of alert' do
it_behaves_like 'notification for custom level is sent', 1500, 30
it_behaves_like 'notification for custom level is sent', 1980, 5
end
context 'when there are not available minutes to use' do
include_examples 'no notification is sent'
end
end
end
......@@ -37,7 +37,7 @@ describe BuildFinishedWorker do
namespace.update_attribute(:shared_runners_minutes_limit, 2000)
namespace_stats.update_attribute(:shared_runners_seconds, 2100 * 60)
expect(CiMinutesUsageMailer).to receive(:notify).once.with(namespace.name, namespace.owner.email).and_return(spy)
expect(CiMinutesUsageMailer).to receive(:notify).once.with(namespace.name, [namespace.owner.email]).and_return(spy)
subject
end
......
......@@ -90,6 +90,22 @@ describe ClearSharedRunnersMinutesWorker do
end
end
end
[:last_ci_minutes_notification_at, :last_ci_minutes_usage_notification_level].each do |attr|
context "when #{attr} is present" do
before do
namespace.update_attribute(attr, Time.now)
end
it 'nullifies the field' do
expect(namespace.send(attr)).to be_present
subject
expect(namespace.reload.send(attr)).not_to be_present
end
end
end
end
end
end
......@@ -9848,6 +9848,12 @@ msgstr ""
msgid "Pipelines| to purchase more minutes."
msgstr ""
msgid "Pipelines|%{namespace_name} has exceeded its pipeline minutes quota."
msgstr ""
msgid "Pipelines|%{namespace_name} has less than %{notification_level}%% of CI minutes available."
msgstr ""
msgid "Pipelines|API"
msgstr ""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment