Skip to content

G
gitlab-ce
Commit 69260e86 authored by Heinrich Lee Yu's avatar Heinrich Lee Yu
Browse files
Options

Merge branch 'sy-refactor-notify-services' into 'master' 

Move shared alert processing logic to concern

See merge request gitlab-org/gitlab!52374
parents 2803fb8d 2a77b038
Hide whitespace changes
Inline Side-by-side
Showing with 171 additions and 231 deletions
app/services/alert_management/process_prometheus_alert_service.rb
View file @ 69260e86
......@@ -2,9 +2,8 @@
module AlertManagement
class ProcessPrometheusAlertService
include BaseServiceUtility
include Gitlab::Utils::StrongMemoize
include ::IncidentManagement::Settings
extend ::Gitlab::Utils::Override
include ::AlertManagement::AlertProcessing
def initialize(project, payload)
@project = project
......@@ -14,11 +13,10 @@ module AlertManagement
def execute
return bad_request unless incoming_payload.has_required_attributes?
process_alert_management_alert
process_alert
return bad_request unless alert.persisted?
process_incident_issues if process_issues?
send_alert_email if send_email?
complete_post_processing_tasks
ServiceResponse.success
end
......@@ -27,110 +25,31 @@ module AlertManagement
attr_reader :project, :payload
def process_alert_management_alert
if incoming_payload.resolved?
process_resolved_alert_management_alert
else
process_firing_alert_management_alert
end
end
def process_firing_alert_management_alert
if alert.persisted?
alert.register_new_event!
reset_alert_management_alert_status
else
create_alert_management_alert
end
end
override :process_new_alert
def process_new_alert
return if resolving_alert?
def reset_alert_management_alert_status
return if alert.trigger
logger.warn(
message: 'Unable to update AlertManagement::Alert status to triggered',
project_id: project.id,
alert_id: alert.id
)
super
end
def create_alert_management_alert
if alert.save
alert.execute_services
SystemNoteService.create_new_alert(alert, Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:prometheus])
return
end
override :process_firing_alert
def process_firing_alert
super
logger.warn(
message: 'Unable to create AlertManagement::Alert',
project_id: project.id,
alert_errors: alert.errors.messages
)
reset_alert_status
end
def process_resolved_alert_management_alert
return unless alert.persisted?
return unless auto_close_incident?
if alert.resolve(incoming_payload.ends_at)
close_issue(alert.issue)
return
end
def reset_alert_status
return if alert.trigger
logger.warn(
message: 'Unable to update AlertManagement::Alert status to resolved',
message: 'Unable to update AlertManagement::Alert status to triggered',
project_id: project.id,
alert_id: alert.id
)
end
def close_issue(issue)
return if issue.blank? || issue.closed?
Issues::CloseService
.new(project, User.alert_bot)
.execute(issue, system_note: false)
SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if issue.reset.closed?
end
def process_incident_issues
return if alert.issue || alert.resolved?
IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, alert.id)
end
def send_alert_email
notification_service
.async
.prometheus_alerts_fired(project, [alert])
end
def logger
@logger ||= Gitlab::AppLogger
end
def alert
strong_memoize(:alert) do
existing_alert || new_alert
end
end
def existing_alert
strong_memoize(:existing_alert) do
AlertManagement::Alert.not_resolved.for_fingerprint(project, incoming_payload.gitlab_fingerprint).first
end
end
def new_alert
strong_memoize(:new_alert) do
AlertManagement::Alert.new(
**incoming_payload.alert_params,
ended_at: nil
)
end
end
override :incoming_payload
def incoming_payload
strong_memoize(:incoming_payload) do
Gitlab::AlertManagement::Payload.parse(
......@@ -141,6 +60,11 @@ module AlertManagement
end
end
override :resolving_alert?
def resolving_alert?
incoming_payload.resolved?
end
def bad_request
ServiceResponse.error(message: 'Bad Request', http_status: :bad_request)
end
......
app/services/concerns/alert_management/alert_processing.rb 0 → 100644
View file @ 69260e86
# frozen_string_literal: true
module AlertManagement
# Module to support the processing of new alert payloads
# from various sources. Payloads may be for new alerts,
# existing alerts, or acting as a resolving alert.
#
# Performs processing-related tasks, such as creating system
# notes, creating or resolving related issues, and notifying
# stakeholders of the alert.
#
# Requires #project [Project] and #payload [Hash] methods
# to be defined.
module AlertProcessing
include BaseServiceUtility
include Gitlab::Utils::StrongMemoize
include ::IncidentManagement::Settings
# Updates or creates alert from payload for project
# including system notes
def process_alert
if alert.persisted?
process_existing_alert
else
process_new_alert
end
end
# Creates or closes issue for alert and notifies stakeholders
def complete_post_processing_tasks
process_incident_issues if process_issues?
send_alert_email if send_email?
end
def process_existing_alert
if resolving_alert?
process_resolved_alert
else
process_firing_alert
end
end
def process_resolved_alert
return unless auto_close_incident?
return close_issue(alert.issue) if alert.resolve(incoming_payload.ends_at)
logger.warn(
message: 'Unable to update AlertManagement::Alert status to resolved',
project_id: project.id,
alert_id: alert.id
)
end
def process_firing_alert
alert.register_new_event!
end
def close_issue(issue)
return if issue.blank? || issue.closed?
::Issues::CloseService
.new(project, User.alert_bot)
.execute(issue, system_note: false)
SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if issue.reset.closed?
end
def process_new_alert
if alert.save
alert.execute_services
SystemNoteService.create_new_alert(alert, alert_source)
else
logger.warn(
message: "Unable to create AlertManagement::Alert from #{alert_source}",
project_id: project.id,
alert_errors: alert.errors.messages
)
end
end
def process_incident_issues
return if alert.issue || alert.resolved?
::IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, alert.id)
end
def send_alert_email
notification_service
.async
.prometheus_alerts_fired(project, [alert])
end
def incoming_payload
strong_memoize(:incoming_payload) do
Gitlab::AlertManagement::Payload.parse(project, payload.to_h)
end
end
def alert
strong_memoize(:alert) do
find_existing_alert || build_new_alert
end
end
def find_existing_alert
return unless incoming_payload.gitlab_fingerprint
AlertManagement::Alert.not_resolved.for_fingerprint(project, incoming_payload.gitlab_fingerprint).first
end
def build_new_alert
AlertManagement::Alert.new(**incoming_payload.alert_params, ended_at: nil)
end
def resolving_alert?
incoming_payload.ends_at.present?
end
def alert_source
alert.monitoring_tool
end
def logger
@logger ||= Gitlab::AppLogger
end
end
end
app/services/projects/alerting/notify_service.rb
View file @ 69260e86
......@@ -3,9 +3,8 @@
module Projects
module Alerting
class NotifyService
include BaseServiceUtility
include Gitlab::Utils::StrongMemoize
include ::IncidentManagement::Settings
extend ::Gitlab::Utils::Override
include ::AlertManagement::AlertProcessing
def initialize(project, payload)
@project = project
......@@ -22,8 +21,7 @@ module Projects
process_alert
return bad_request unless alert.persisted?
process_incident_issues if process_issues?
send_alert_email if send_email?
complete_post_processing_tasks
ServiceResponse.success
end
......@@ -32,93 +30,15 @@ module Projects
attr_reader :project, :payload, :integration
def process_alert
if alert.persisted?
process_existing_alert
else
create_alert
end
end
def process_existing_alert
if incoming_payload.ends_at.present?
process_resolved_alert
else
alert.register_new_event!
end
alert
end
def process_resolved_alert
return unless auto_close_incident?
if alert.resolve(incoming_payload.ends_at)
close_issue(alert.issue)
end
alert
end
def close_issue(issue)
return if issue.blank? || issue.closed?
::Issues::CloseService
.new(project, User.alert_bot)
.execute(issue, system_note: false)
SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if issue.reset.closed?
end
def create_alert
return unless alert.save
alert.execute_services
SystemNoteService.create_new_alert(alert, notification_source)
end
def process_incident_issues
return if alert.issue || alert.resolved?
::IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, alert.id)
end
def send_alert_email
notification_service
.async
.prometheus_alerts_fired(project, [alert])
end
def alert
strong_memoize(:alert) do
existing_alert || new_alert
end
end
def existing_alert
return unless incoming_payload.gitlab_fingerprint
AlertManagement::Alert.not_resolved.for_fingerprint(project, incoming_payload.gitlab_fingerprint).first
end
def new_alert
AlertManagement::Alert.new(**incoming_payload.alert_params, ended_at: nil)
end
def incoming_payload
strong_memoize(:incoming_payload) do
Gitlab::AlertManagement::Payload.parse(project, payload.to_h)
end
def valid_payload_size?
Gitlab::Utils::DeepSize.new(payload).valid?
end
def notification_source
override :alert_source
def alert_source
alert.monitoring_tool || integration&.name || 'Generic Alert Endpoint'
end
def valid_payload_size?
Gitlab::Utils::DeepSize.new(payload).valid?
end
def active_integration?
integration&.active?
end
......
ee/app/services/alert_management/network_alert_service.rb
View file @ 69260e86
......@@ -3,7 +3,8 @@
module AlertManagement
# Create alerts coming K8 through gitlab-agent
class NetworkAlertService
include Gitlab::Utils::StrongMemoize
extend ::Gitlab::Utils::Override
include ::AlertManagement::AlertProcessing
MONITORING_TOOL = Gitlab::AlertManagement::Payload::MONITORING_TOOLS.fetch(:cilium)
......@@ -12,12 +13,10 @@ module AlertManagement
@payload = payload
end
# Users of this service need to check the agent token before calling `execute`.
# https://gitlab.com/gitlab-org/gitlab/-/issues/292707 will handle token within the service.
def execute
return bad_request unless valid_payload_size?
process_request
process_alert
return bad_request unless alert.persisted?
......@@ -32,57 +31,27 @@ module AlertManagement
Gitlab::Utils::DeepSize.new(payload).valid?
end
def process_request
if alert.persisted?
alert.register_new_event!
else
create_alert
end
end
def create_alert
if alert.save
alert.execute_services
SystemNoteService.create_new_alert(
alert,
MONITORING_TOOL
)
return
end
logger.warn(
message:
"Unable to create AlertManagement::Alert from #{MONITORING_TOOL}",
project_id: project.id,
alert_errors: alert.errors.messages
)
end
def logger
@logger ||= Gitlab::AppLogger
end
def alert
strong_memoize(:alert) { find_existing_alert || build_new_alert }
end
def find_existing_alert
AlertManagement::Alert.not_resolved.for_fingerprint(
project,
incoming_payload.gitlab_fingerprint
).first
end
override :build_new_alert
def build_new_alert
AlertManagement::Alert.new(**incoming_payload.alert_params, domain: :threat_monitoring, ended_at: nil)
AlertManagement::Alert.new(
**incoming_payload.alert_params,
domain: :threat_monitoring,
ended_at: nil
)
end
override :incoming_payload
def incoming_payload
strong_memoize(:incoming_payload) do
Gitlab::AlertManagement::Payload.parse(project, payload, monitoring_tool: MONITORING_TOOL)
end
end
override :resolving_alert?
def resolving_alert?
false
end
def bad_request
ServiceResponse.error(message: 'Bad Request', http_status: :bad_request)
end
......
spec/services/alert_management/process_prometheus_alert_service_spec.rb
View file @ 69260e86
......@@ -158,7 +158,7 @@ RSpec.describe AlertManagement::ProcessPrometheusAlertService do
it 'writes a warning to the log' do
expect(Gitlab::AppLogger).to receive(:warn).with(
message: 'Unable to create AlertManagement::Alert',
message: 'Unable to create AlertManagement::Alert from Prometheus',
project_id: project.id,
alert_errors: { hosts: ['hosts array is over 255 chars'] }
)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7