Merge branch 'mwaw/add_observability_to_snowplow_tracking' into 'master'

Add observability to Snowplow tracking

See merge request gitlab-org/gitlab!65664

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -136,6 +136,9 @@ The following metrics are available:
 | `service_desk_thank_you_email`                                   | Counter     | 14.0    | Total number of email responses to new service desk emails                                                            |                                                           |
 | `service_desk_new_note_email`                                    | Counter     | 14.0    | Total number of email notifications on new service desk comment                                                       |                                                           |
 | `email_receiver_error`                                           | Counter     | 14.1    | Total number of errors when processing incoming emails                                                                |                                                           |
+| `gitlab_snowplow_events_total`                                   | Counter     | 14.1    | Total number of GitLab Snowplow product intelligence events emitted                                                   |                                                           |
+| `gitlab_snowplow_failed_events_total`                            | Counter     | 14.1    | Total number of GitLab Snowplow product intelligence events emission failures                                         |                                                           |
+| `gitlab_snowplow_successful_events_total`                        | Counter     | 14.1    | Total number of GitLab Snowplow product intelligence events emission successes                                        |                                                           |
 
 ## Metrics controlled by a feature flag
 

lib/gitlab/tracking/destinations/snowplow.rb

@@ -13,6 +13,7 @@ module Gitlab
           return unless enabled?
 
           tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i)
+          increment_total_events_counter
         end
 
         private
@@ -33,9 +34,46 @@ module Gitlab
         def emitter
           SnowplowTracker::AsyncEmitter.new(
             Gitlab::CurrentSettings.snowplow_collector_hostname,
-            protocol: 'https'
+            protocol: 'https',
+            on_success: method(:increment_successful_events_emissions),
+            on_failure: method(:failure_callback)
           )
         end
+
+        def failure_callback(success_count, failures)
+          increment_successful_events_emissions(success_count)
+          increment_failed_events_emissions(failures.size)
+          log_failures(failures)
+        end
+
+        def increment_failed_events_emissions(value)
+          Gitlab::Metrics.counter(
+            :gitlab_snowplow_failed_events_total,
+            'Number of failed Snowplow events emissions'
+          ).increment({}, value.to_i)
+        end
+
+        def increment_successful_events_emissions(value)
+          Gitlab::Metrics.counter(
+            :gitlab_snowplow_successful_events_total,
+            'Number of successful Snowplow events emissions'
+          ).increment({}, value.to_i)
+        end
+
+        def increment_total_events_counter
+          Gitlab::Metrics.counter(
+            :gitlab_snowplow_events_total,
+            'Number of Snowplow events'
+          ).increment
+        end
+
+        def log_failures(failures)
+          hostname = Gitlab::CurrentSettings.snowplow_collector_hostname
+
+          failures.each do |failure|
+            Gitlab::AppLogger.error("#{failure["se_ca"]} #{failure["se_ac"]} failed to be reported to collector at #{hostname}")
+          end
+        end
       end
     end
   end

spec/lib/gitlab/tracking/destinations/snowplow_spec.rb

@@ -21,7 +21,10 @@ RSpec.describe Gitlab::Tracking::Destinations::Snowplow do
 
       expect(SnowplowTracker::AsyncEmitter)
         .to receive(:new)
-        .with('gitfoo.com', { protocol: 'https' })
+        .with('gitfoo.com',
+              { protocol: 'https',
+                on_success: subject.method(:increment_successful_events_emissions),
+                on_failure: subject.method(:failure_callback) })
         .and_return(emitter)
 
       expect(SnowplowTracker::Tracker)
@@ -40,6 +43,18 @@ RSpec.describe Gitlab::Tracking::Destinations::Snowplow do
           .to have_received(:track_struct_event)
           .with('category', 'action', 'label', 'property', 1.5, nil, (Time.now.to_f * 1000).to_i)
       end
+
+      it 'increase total snowplow events counter' do
+        counter = double
+
+        expect(counter).to receive(:increment)
+        expect(Gitlab::Metrics).to receive(:counter)
+                                     .with(:gitlab_snowplow_events_total,
+                                           'Number of Snowplow events')
+                                     .and_return(counter)
+
+        subject.event('category', 'action', label: 'label', property: 'property', value: 1.5)
+      end
     end
   end
 
@@ -52,4 +67,43 @@ RSpec.describe Gitlab::Tracking::Destinations::Snowplow do
       end
     end
   end
+
+  context 'callbacks' do
+    describe 'on success' do
+      it 'increase gitlab_successful_snowplow_events_total counter' do
+        counter = double
+
+        expect(counter).to receive(:increment).with({}, 2)
+        expect(Gitlab::Metrics).to receive(:counter)
+                                     .with(:gitlab_snowplow_successful_events_total,
+                                           'Number of successful Snowplow events emissions')
+                                     .and_return(counter)
+
+        subject.method(:increment_successful_events_emissions).call(2)
+      end
+    end
+
+    describe 'on failure' do
+      it 'increase gitlab_failed_snowplow_events_total counter and logs failures', :aggregate_failures do
+        counter = double
+        error_message = "Admin::AuditLogsController search_audit_event failed to be reported to collector at gitfoo.com"
+        failures = [{ "e" => "se",
+                      "se_ca" => "Admin::AuditLogsController",
+                      "se_ac" => "search_audit_event" }]
+        allow(Gitlab::Metrics).to receive(:counter)
+                                    .with(:gitlab_snowplow_successful_events_total,
+                                          'Number of successful Snowplow events emissions')
+                                    .and_call_original
+
+        expect(Gitlab::AppLogger).to receive(:error).with(error_message)
+        expect(counter).to receive(:increment).with({}, 1)
+        expect(Gitlab::Metrics).to receive(:counter)
+                                     .with(:gitlab_snowplow_failed_events_total,
+                                           'Number of failed Snowplow events emissions')
+                                     .and_return(counter)
+
+        subject.method(:failure_callback).call(2, failures)
+      end
+    end
+  end
 end