Commit 14741902 authored by Shinya Maeda's avatar Shinya Maeda

Execute Auto Rollback When a critical alert is raised

This commit integrates the Auto Rollback service with
Alert model.
parent e733d1f4
...@@ -263,3 +263,5 @@ module AlertManagement ...@@ -263,3 +263,5 @@ module AlertManagement
end end
end end
end end
AlertManagement::Alert.prepend_if_ee('EE::AlertManagement::Alert')
...@@ -894,6 +894,32 @@ longer visible on the environment page. ...@@ -894,6 +894,32 @@ longer visible on the environment page.
If the alert requires a [rollback](#retrying-and-rolling-back), you can select the If the alert requires a [rollback](#retrying-and-rolling-back), you can select the
deployment tab from the environment page and select which deployment to roll back to. deployment tab from the environment page and select which deployment to roll back to.
#### Auto Rollback **(ULTIMATE)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35404) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.7.
In a typical Continuous Deployment workflow, the CI pipeline tests every commit before deploying to
production. However, problematic code can still make it to production. For example, inefficient code
that is logically correct can pass tests even though it causes severe performance degradation.
Operators and SREs monitor the system to catch such problems as soon as possible. If they find a
problematic deployment, they can roll back to a previous stable version.
GitLab Auto Rollback eases this workflow by automatically triggering a rollback when a
[critical alert](../../operations/incident_management/alerts.md)
is detected. GitLab selects and redeploys the most recent successful deployment.
Limitations of GitLab Auto Rollback:
- The rollback is skipped if a deployment is running when the alert is detected.
- A rollback can happen only once in three minutes. If multiple alerts are detected at once, only
one rollback is performed.
GitLab Auto Rollback is turned off by default. To turn it on:
1. Visit **Project > Settings > CI/CD > Automatic deployment rollbacks**.
1. Select the checkbox for **Enable automatic rollbacks**.
1. Click **Save changes**.
### Monitoring environments ### Monitoring environments
If you have enabled [Prometheus for monitoring system and response metrics](../../user/project/integrations/prometheus.md), If you have enabled [Prometheus for monitoring system and response metrics](../../user/project/integrations/prometheus.md),
......
# frozen_string_literal: true
module EE
module AlertManagement
module Alert
extend ActiveSupport::Concern
prepended do
include AfterCommitQueue
after_create do |alert|
run_after_commit { alert.trigger_auto_rollback }
end
end
def trigger_auto_rollback
return unless triggered? && critical? && environment&.auto_rollback_enabled?
::Deployments::AutoRollbackWorker.perform_async(environment.id)
end
end
end
end
...@@ -23,7 +23,6 @@ ...@@ -23,7 +23,6 @@
= s_('AutoRollback|Enable automatic rollbacks') = s_('AutoRollback|Enable automatic rollbacks')
%small.form-text.text-gl-muted %small.form-text.text-gl-muted
= s_('AutoRollback|Automatic rollbacks start when a critical alert is triggered. If the last successful deployment fails to roll back automatically, it can still be done manually.') = s_('AutoRollback|Automatic rollbacks start when a critical alert is triggered. If the last successful deployment fails to roll back automatically, it can still be done manually.')
-# This will be added once the documentation page has been created = link_to _('More information'), help_page_path('ci/environments/index.md', anchor: 'auto-rollback'), target: '_blank'
-# = link_to _('More information'), help_page_path('topics/auto_rollback/index.md'), target: '_blank'
= f.submit _('Save changes'), class: "gl-button btn btn-success gl-mt-5", data: { qa_selector: 'save_changes_button' } = f.submit _('Save changes'), class: "gl-button btn btn-success gl-mt-5", data: { qa_selector: 'save_changes_button' }
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe EE::AlertManagement::Alert do
let_it_be(:project, refind: true) { create(:project) }
let_it_be(:environment, refind: true) { create(:environment, project: project) }
describe 'after_create' do
it 'attempts to trigger auto rollback' do
alert = build(:alert_management_alert, :triggered, :critical)
expect(alert).to receive(:trigger_auto_rollback)
alert.save!
end
end
describe '#trigger_auto_rollback' do
subject { alert.trigger_auto_rollback }
let!(:alert) { create(:alert_management_alert, :triggered, :critical, project: project, environment: environment) }
before do
stub_licensed_features(auto_rollback: true)
environment.project.auto_rollback_enabled = true
end
it 'executes AutoRollbackWorker' do
expect(Deployments::AutoRollbackWorker).to receive(:perform_async).with(environment.id)
subject
end
context 'when status is not triggered' do
let!(:alert) { create(:alert_management_alert, :acknowledged, :critical, project: project, environment: environment) }
it 'does not execute AutoRollbackWorker' do
expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
subject
end
end
context 'when severity is not critical' do
let!(:alert) { create(:alert_management_alert, :triggered, :high, project: project, environment: environment) }
it 'does not execute AutoRollbackWorker' do
expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
subject
end
end
context 'when project does not enable auto rollback' do
before do
environment.project.auto_rollback_enabled = false
end
it 'does not execute AutoRollbackWorker' do
expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
subject
end
end
context 'when project does not have a license for auto rollback' do
before do
stub_licensed_features(auto_rollback: false)
end
it 'does not execute AutoRollbackWorker' do
expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
subject
end
end
context 'when feature flag is disabled' do
before do
stub_feature_flags(cd_auto_rollback: false)
end
it 'does not execute AutoRollbackWorker' do
expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
subject
end
end
end
end
...@@ -99,7 +99,8 @@ RSpec.describe AlertManagement::Alert do ...@@ -99,7 +99,8 @@ RSpec.describe AlertManagement::Alert do
describe 'fingerprint' do describe 'fingerprint' do
let_it_be(:fingerprint) { 'fingerprint' } let_it_be(:fingerprint) { 'fingerprint' }
let(:new_alert) { build(:alert_management_alert, fingerprint: fingerprint, project: project) } let_it_be(:project3, refind: true) { create(:project) }
let(:new_alert) { build(:alert_management_alert, fingerprint: fingerprint, project: project3) }
subject { new_alert } subject { new_alert }
...@@ -107,7 +108,7 @@ RSpec.describe AlertManagement::Alert do ...@@ -107,7 +108,7 @@ RSpec.describe AlertManagement::Alert do
context 'same project, various states' do context 'same project, various states' do
using RSpec::Parameterized::TableSyntax using RSpec::Parameterized::TableSyntax
let_it_be(:existing_alert) { create(:alert_management_alert, fingerprint: fingerprint, project: project) } let_it_be(:existing_alert, refind: true) { create(:alert_management_alert, fingerprint: fingerprint, project: project3) }
# We are only validating uniqueness for non-resolved alerts # We are only validating uniqueness for non-resolved alerts
where(:existing_status, :new_status, :valid) do where(:existing_status, :new_status, :valid) do
...@@ -130,7 +131,7 @@ RSpec.describe AlertManagement::Alert do ...@@ -130,7 +131,7 @@ RSpec.describe AlertManagement::Alert do
end end
with_them do with_them do
let(:new_alert) { build(:alert_management_alert, new_status, fingerprint: fingerprint, project: project) } let(:new_alert) { build(:alert_management_alert, new_status, fingerprint: fingerprint, project: project3) }
before do before do
existing_alert.change_status_to(existing_status) existing_alert.change_status_to(existing_status)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment