Enable audit event streaming

Adds functionality to stream audit events to external
destinations.

Adds documentation.

Feature is behind a default-off flag.

Applying technical writing suggestions

app/services/audit_event_service.rb

@@ -119,6 +119,10 @@ class AuditEventService
     event
   end
 
+  def stream_event_to_external_destinations(_event)
+    # Defined in EE
+  end
+
   def log_authentication_event_to_database
     return unless Gitlab::Database.read_write? && authentication_event?
 
@@ -130,6 +134,7 @@ class AuditEventService
 
   def save_or_track(event)
     event.save!
+    stream_event_to_external_destinations(event)
   rescue StandardError => e
     Gitlab::ErrorTracking.track_exception(e, audit_event_type: event.class.to_s)
   end

config/sidekiq_queues.yml

@@ -39,6 +39,8 @@
   - 1
 - - approve_blocked_pending_approval_users
   - 1
+- - audit_events_audit_event_streaming
+  - 1
 - - authorized_keys
   - 2
 - - authorized_project_update

doc/administration/audit_event_streaming.md

+---
+stage: Manage
+group: Compliance
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
+---
+
+# Audit event streaming **(ULTIMATE)**
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.5 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events_namespace`. Disabled by default.
+
+FLAG:
+On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `ff_external_audit_events_namespace`. On GitLab.com, this feature is not available.
+You should not use this feature for production environments.
+
+Event streaming allows owners of top-level groups to set an HTTP endpoint to receive **all** audit events about the group, and its
+subgroups and projects.
+
+Top-level group owners can manage their audit logs in third-party systems such as Splunk, using the Splunk
+[HTTP Event Collector](https://docs.splunk.com/Documentation/Splunk/8.2.2/Data/UsetheHTTPEventCollector). Any service that can receive
+structured JSON data can be used as the endpoint.
+
+NOTE:
+GitLab can stream a single event more than once to the same destination. Use the `id` key in the payload to deduplicate incoming data.
+
+## Add a new event streaming destination
+
+WARNING:
+Event streaming destinations will receive **all** audit event data, which could include sensitive information. Make sure you trust the destination endpoint.
+
+To enable event streaming, a group owner must add a new event streaming destination using the `externalAuditEventDestinationCreate` mutation
+in the GraphQL API.
+
+```graphql
+mutation {
+  externalAuditEventDestinationCreate(input: { destinationUrl: "https://mydomain.io/endpoint/ingest", groupPath: "my-group" } ) {
+    errors
+    externalAuditEventDestination {
+      destinationUrl
+      group {
+        name
+      }
+    }
+  }
+}
+```
+
+Event streaming is enabled if:
+
+- The returned `errors` object is empty.
+- The API responds with `200 OK`.
+
+## List currently enabled streaming destinations
+
+Group owners can view a list of event streaming destinations at any time using the `externalAuditEventDesinations` query type.
+
+```graphql
+query {
+  group(fullPath: "my-group") {
+    id
+    externalAuditEventDestinations {
+      nodes {
+        destinationUrl
+        id
+      }
+    }
+  }
+}
+```
+
+If the resulting list is empty, then audit event streaming is not enabled for that group.

doc/administration/audit_events.md

@@ -12,7 +12,10 @@ on a [paid plan](https://about.gitlab.com/pricing/).
 GitLab system administrators can also take advantage of the logs located on the
 file system. See [the logs system documentation](logs.md#audit_jsonlog) for more details.
 
-You can generate an [Audit report](audit_reports.md) of audit events.
+You can:
+
+- Generate an [audit report](audit_reports.md) of audit events.
+- [Stream audit events](audit_event_streaming.md) to an external endpoint.
 
 ## Overview
 

ee/app/models/ee/audit_event.rb

@@ -54,6 +54,18 @@ module EE
         end
     end
 
+    def stream_to_external_destinations
+      return if entity.nil?
+      return unless ::Feature.enabled?(:ff_external_audit_events_namespace, entity)
+      return unless entity.licensed_feature_available?(:external_audit_events)
+
+      AuditEvents::AuditEventStreamingWorker.perform_async(id)
+    end
+
+    def entity_is_group_or_project?
+      %w(Group Project).include?(entity_type)
+    end
+
     private
 
     def truncate_fields

ee/app/services/ee/audit_event_service.rb

@@ -229,6 +229,13 @@ module EE
 
     private
 
+    override :stream_event_to_external_destinations
+    def stream_event_to_external_destinations(event)
+      return if event.is_a?(AuthenticationEvent)
+
+      event.stream_to_external_destinations if event.entity_is_group_or_project?
+    end
+
     override :base_payload
     def base_payload
       super.tap do |payload|

ee/app/workers/all_queues.yml

@@ -885,6 +885,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: audit_events_audit_event_streaming
+  :worker_name: AuditEvents::AuditEventStreamingWorker
+  :feature_category: :audit_events
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: ci_batch_reset_minutes
   :worker_name: Ci::BatchResetMinutesWorker
   :feature_category: :continuous_integration

ee/app/workers/audit_events/audit_event_streaming_worker.rb

+# frozen_string_literal: true
+
+module AuditEvents
+  class AuditEventStreamingWorker
+    include ApplicationWorker
+
+    REQUEST_BODY_SIZE_LIMIT = 25.megabytes
+
+    # Audit Events contains a unique ID so the ingesting system should
+    # attempt to deduplicate based on this to allow this job to be idempotent.
+    idempotent!
+    worker_has_external_dependencies!
+    data_consistency :always
+    feature_category :audit_events
+
+    def perform(audit_event_id)
+      audit_event = AuditEvent.find(audit_event_id)
+      group = group_entity(audit_event)
+
+      return if group.nil? # Do nothing if the event can't be resolved to a single group.
+      return unless ::Feature.enabled?(:ff_external_audit_events_namespace, group)
+      return unless group.licensed_feature_available?(:external_audit_events)
+
+      group.external_audit_event_destinations.each do |destination|
+        Gitlab::HTTP.post(destination.destination_url,
+                          body: Gitlab::Json::LimitedEncoder.encode(audit_event.as_json, limit: REQUEST_BODY_SIZE_LIMIT), use_read_total_timeout: true)
+      end
+    end
+
+    private
+
+    def group_entity(audit_event)
+      case audit_event.entity_type
+      when 'Group'
+        audit_event.entity
+      when 'Project'
+        # Project events should be sent to the root ancestor's streaming destinations
+        # Projects without a group root ancestor should be ignored.
+        audit_event.entity.group&.root_ancestor
+      else
+        nil
+      end
+    end
+  end
+end

ee/lib/gitlab/audit/null_entity.rb

@@ -3,6 +3,9 @@
 module Gitlab
   module Audit
     class NullEntity
+      def nil?
+        true
+      end
     end
   end
 end

ee/spec/models/audit_event_spec.rb

@@ -78,6 +78,46 @@ RSpec.describe AuditEvent, type: :model do
     end
   end
 
+  describe '#stream_to_external_destinations' do
+    let_it_be(:event) { create(:audit_event, :group_event) }
+
+    context 'feature is licensed' do
+      before do
+        stub_licensed_features(external_audit_events: true)
+      end
+
+      it 'enqueues one worker' do
+        expect(AuditEvents::AuditEventStreamingWorker).to receive(:perform_async).once
+
+        event.stream_to_external_destinations
+      end
+
+      context 'feature is disabled' do
+        before do
+          stub_feature_flags(ff_external_audit_events_namespace: false)
+        end
+
+        it 'enqueues no workers' do
+          expect(AuditEvents::AuditEventStreamingWorker).not_to receive(:perform_async)
+
+          event.stream_to_external_destinations
+        end
+      end
+    end
+
+    context 'feature is unlicensed' do
+      before do
+        stub_licensed_features(external_audit_events: false)
+      end
+
+      it 'enqueues no workers' do
+        expect(AuditEvents::AuditEventStreamingWorker).not_to receive(:perform_async)
+
+        event.stream_to_external_destinations
+      end
+    end
+  end
+
   describe '.by_entity' do
     let_it_be(:project_event_1) { create(:project_audit_event) }
     let_it_be(:project_event_2) { create(:project_audit_event) }
@@ -285,4 +325,26 @@ RSpec.describe AuditEvent, type: :model do
       end
     end
   end
+
+  describe 'entity_is_group_or_project?' do
+    subject { event.entity_is_group_or_project? }
+
+    context 'when entity is a Group' do
+      let_it_be(:event) { create(:group_audit_event) }
+
+      it { is_expected.to be true }
+    end
+
+    context 'when entity is a Project' do
+      let_it_be(:event) { create(:project_audit_event) }
+
+      it { is_expected.to be true }
+    end
+
+    context 'when entity is an Epic' do
+      let_it_be(:event) { create(:audit_event, target_type: 'Epic') }
+
+      it { is_expected.to be false }
+    end
+  end
 end

ee/spec/workers/audit_events/audit_event_streaming_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe AuditEvents::AuditEventStreamingWorker do
+  let(:worker) { described_class.new }
+
+  before do
+    stub_licensed_features(external_audit_events: true)
+  end
+
+  shared_examples 'a successful audit event stream' do
+    subject { worker.perform(event.id) }
+
+    context 'when the group has no destinations' do
+      it 'makes no HTTP calls' do
+        expect(Gitlab::HTTP).not_to receive(:post)
+
+        subject
+      end
+    end
+
+    context 'when the group has a destination' do
+      before do
+        group.external_audit_event_destinations.create!(destination_url: 'http://example.com')
+      end
+
+      it 'makes one HTTP call' do
+        expect(Gitlab::HTTP).to receive(:post).once
+
+        subject
+      end
+    end
+
+    context 'when the group has several destinations' do
+      before do
+        group.external_audit_event_destinations.create!(destination_url: 'http://example.com')
+        group.external_audit_event_destinations.create!(destination_url: 'http://example1.com')
+        group.external_audit_event_destinations.create!(destination_url: 'http://example2.com')
+      end
+
+      it 'makes the correct number of HTTP calls' do
+        expect(Gitlab::HTTP).to receive(:post).exactly(3).times
+
+        subject
+      end
+
+      context 'when feature is disabled' do
+        before do
+          stub_feature_flags(ff_external_audit_events_namespace: false)
+        end
+
+        it 'makes no HTTP calls' do
+          expect(Gitlab::HTTP).not_to receive(:post)
+
+          subject
+        end
+      end
+
+      context 'when feature is unlicensed' do
+        before do
+          stub_licensed_features(external_audit_events: false)
+        end
+
+        it 'makes no HTTP calls' do
+          expect(Gitlab::HTTP).not_to receive(:post)
+
+          subject
+        end
+      end
+    end
+  end
+
+  describe "#perform" do
+    context 'when the entity type is a group' do
+      it_behaves_like 'a successful audit event stream' do
+        let_it_be(:event) { create(:audit_event, :group_event) }
+
+        let(:group) { event.entity }
+      end
+    end
+
+    context 'when the entity type is a project that belongs to a group' do
+      it_behaves_like 'a successful audit event stream' do
+        let_it_be(:group) { create(:group) }
+        let_it_be(:project) { create(:project, group: group) }
+        let_it_be(:event) { create(:audit_event, :project_event, target_project: project) }
+      end
+    end
+
+    context 'when the entity type is a project at a root namespace level' do
+      let_it_be(:event) { create(:audit_event, :project_event) }
+
+      it 'makes no HTTP calls' do
+        expect(Gitlab::HTTP).not_to receive(:post)
+
+        worker.perform(event.id)
+      end
+    end
+  end
+end