Merge branch '344670-audit-streaming-usage-data' into 'master'

Add event streaming metrics

See merge request gitlab-org/gitlab!75504

ee/app/models/ee/group.rb

@@ -91,6 +91,10 @@ module EE
         joins(:ldap_group_links).where(ldap_group_links: { provider: provider })
       end
 
+      scope :with_external_audit_event_destinations, -> do
+        joins(:external_audit_event_destinations)
+      end
+
       scope :with_managed_accounts_enabled, -> do
         joins(:saml_provider).where(saml_providers:
           {

ee/config/metrics/counts_28d/20211130085433_g_manage_compliance_audit_event_destinations.yml

+---
+key_path: usage_activity_by_stage_monthly.manage.audit_event_destinations
+description: "Count of audit event streaming destinations"
+product_section: "dev"
+product_stage: "manage"
+product_group: "compliance"
+product_category: "Audit Events"
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670
+time_frame: "28d"
+data_source: database
+data_category: optional
+instrumentation_class: CountEventStreamingDestinationsMetric
+distribution:
+- ee
+tier:
+- ultimate

ee/config/metrics/counts_28d/20211130091657_groups_with_event_streaming_destinations.yml

+---
+key_path: usage_activity_by_stage_monthly.manage.groups_with_event_streaming_destinations
+description: "Distinct count of groups with any event streaming destinations"
+product_section: "dev"
+product_stage: "manage"
+product_group: "compliance"
+product_category: "Audit Events"
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670
+time_frame: "28d"
+data_source: "database"
+data_category: optional
+instrumentation_class: CountGroupsWithEventStreamingDestinationsMetric
+distribution:
+- ee
+tier:
+- ultimate

ee/config/metrics/counts_all/20211130085433_g_manage_compliance_audit_event_destinations.yml

+---
+key_path: usage_activity_by_stage.manage.audit_event_destinations
+description: "Count of audit event streaming destinations"
+product_section: "dev"
+product_stage: "manage"
+product_group: "compliance"
+product_category: "Audit Events"
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670
+time_frame: "all"
+data_source: database
+data_category: optional
+instrumentation_class: CountEventStreamingDestinationsMetric
+distribution:
+- ee
+tier:
+- ultimate

ee/config/metrics/counts_all/20211130091657_groups_with_event_streaming_destinations.yml

+---
+key_path: usage_activity_by_stage.manage.groups_with_event_streaming_destinations
+description: "Distinct count of groups with any event streaming destinations"
+product_section: "dev"
+product_stage: "manage"
+product_group: "compliance"
+product_category: "Audit Events"
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670
+time_frame: all
+data_source: "database"
+data_category: optional
+instrumentation_class: CountGroupsWithEventStreamingDestinationsMetric
+distribution:
+- ee
+tier:
+- ultimate

ee/lib/ee/gitlab/usage_data.rb

@@ -306,6 +306,7 @@ module EE
         # Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links`
         override :usage_activity_by_stage_manage
         def usage_activity_by_stage_manage(time_period)
+          time_frame = metric_time_period(time_period)
           super.merge({
             ldap_keys: distinct_count(::LDAPKey.where(time_period), :user_id),
             ldap_users: distinct_count(::GroupMember.of_ldap_type.where(time_period), :user_id),
@@ -316,7 +317,9 @@ module EE
             ldap_servers: ldap_available_servers.size,
             ldap_group_sync_enabled: ldap_config_present_for_any_provider?(:group_base),
             ldap_admin_sync_enabled: ldap_config_present_for_any_provider?(:admin_group),
-            group_saml_enabled: omniauth_provider_names.include?('group_saml')
+            group_saml_enabled: omniauth_provider_names.include?('group_saml'),
+            audit_event_destinations: add_metric('CountEventStreamingDestinationsMetric', time_frame: time_frame),
+            groups_with_event_streaming_destinations: add_metric('CountGroupsWithEventStreamingDestinationsMetric', time_frame: time_frame)
           })
         end
 

ee/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        class CountEventStreamingDestinationsMetric < DatabaseMetric
+          operation :count
+
+          relation do
+            AuditEvents::ExternalAuditEventDestination
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        class CountGroupsWithEventStreamingDestinationsMetric < DatabaseMetric
+          operation :distinct_count
+
+          relation do
+            Group.with_external_audit_event_destinations
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountEventStreamingDestinationsMetric do
+  let_it_be(:destination) { create(:external_audit_event_destination) }
+
+  it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all', data_source: 'database' } do
+    let(:expected_value) { 1 }
+    let(:expected_query) { 'SELECT COUNT("audit_events_external_audit_event_destinations"."id") FROM "audit_events_external_audit_event_destinations"' }
+  end
+end

ee/spec/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountGroupsWithEventStreamingDestinationsMetric do
+  let_it_be(:group_with_destination) { create(:group) }
+  let_it_be(:group_without_destination) { create(:group) }
+
+  before do
+    create(:external_audit_event_destination, group: group_with_destination)
+  end
+
+  it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all', data_source: 'database' } do
+    let(:expected_value) { 1 }
+    let(:expected_query) { 'SELECT COUNT(DISTINCT "namespaces"."id") FROM "namespaces" INNER JOIN "audit_events_external_audit_event_destinations" ON "audit_events_external_audit_event_destinations"."namespace_id" = "namespaces"."id" WHERE "namespaces"."type" = \'Group\'' }
+  end
+end