Merge branch '14707-add-modsec-anomalies' into 'master'

Add WAF anomaly aggregation queries

See merge request gitlab-org/gitlab!24837

ee/app/services/security/waf_anomaly_summary_service.rb

@@ -18,16 +18,19 @@ module Security
       # Use multi-search with single query as we'll be adding nginx later
       # with https://gitlab.com/gitlab-org/gitlab/issues/14707
       aggregate_results = elasticsearch_client.msearch(body: body)
-      nginx_results = aggregate_results['responses'].first
+      nginx_results, modsec_results = aggregate_results['responses']
 
       nginx_total_requests = nginx_results.dig('hits', 'total').to_f
+      modsec_total_requests = modsec_results.dig('hits', 'total').to_f
+
+      anomalous_traffic_count = nginx_total_requests.zero? ? 0 : (modsec_total_requests / nginx_total_requests).round(2)
 
       {
         total_traffic: nginx_total_requests,
-        anomalous_traffic: 0.0,
+        anomalous_traffic: anomalous_traffic_count,
         history: {
           nominal: histogram_from(nginx_results),
-          anomalous: []
+          anomalous: histogram_from(modsec_results)
         },
         interval: @interval,
         from: @from,
@@ -37,7 +40,7 @@ module Security
     end
 
     def elasticsearch_client
-      @client ||= @environment.deployment_platform.cluster.application_elastic_stack&.elasticsearch_client
+      @client ||= @environment.deployment_platform&.cluster&.application_elastic_stack&.elasticsearch_client
     end
 
     private
@@ -49,6 +52,12 @@ module Security
           query: nginx_requests_query,
           aggs: aggregations(@interval),
           size: 0 # no docs needed, only counts
+        },
+        { index: indices },
+        {
+          query: modsec_requests_query,
+          aggs: aggregations(@interval),
+          size: 0 # no docs needed, only counts
         }
       ]
     end
@@ -110,6 +119,42 @@ module Security
       }
     end
 
+    def modsec_requests_query
+      {
+        bool: {
+          must: [
+            {
+              range: {
+                '@timestamp' => {
+                  gte: @from,
+                  lte: @to
+                }
+              }
+            },
+            {
+              prefix: {
+                'transaction.unique_id': application_server_name
+              }
+            },
+            {
+              match_phrase: {
+                'kubernetes.container.name' => {
+                  query: ::Clusters::Applications::Ingress::MODSECURITY_LOG_CONTAINER_NAME
+                }
+              }
+            },
+            {
+              match_phrase: {
+                'kubernetes.namespace' => {
+                  query: Gitlab::Kubernetes::Helm::NAMESPACE
+                }
+              }
+            }
+          ]
+        }
+      }
+    end
+
     def aggregations(interval)
       {
         counts: {
@@ -130,6 +175,11 @@ module Security
       buckets.map { |bucket| [bucket['key_as_string'], bucket['doc_count']] }
     end
 
+    # Derive server_name to filter modsec audit log by environment
+    def application_server_name
+      "#{@environment.project.full_path_slug}.#{@environment.deployment_platform.cluster.base_domain}"
+    end
+
     # Derive proxy upstream name to filter nginx log by environment
     # See https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/log-format/
     def environment_proxy_upstream_name_tokens

ee/spec/services/security/waf_anomaly_summary_service_spec.rb

@@ -10,40 +10,66 @@ describe Security::WafAnomalySummaryService do
 
   let(:es_client) { double(Elasticsearch::Client) }
 
+  let(:empty_response) do
+    {
+      'took' => 40,
+      'timed_out' => false,
+      '_shards' => { 'total' => 11, 'successful' => 11, 'skipped' => 0, 'failed' => 0 },
+      'hits' => { 'total' => 0, 'max_score' => 0.0, 'hits' => [] },
+      'aggregations' => {
+        'counts' => {
+          'buckets' => []
+        }
+      },
+      'status' => 200
+    }
+  end
+
   let(:nginx_response) do
     empty_response.deep_merge(
-      "hits" => { "total" => 3 },
-      "aggregations" => {
-        "counts" => {
-          "buckets" => [
-            { "key_as_string" => "2020-02-14T23:00:00.000Z", "key" => 1575500400000, "doc_count" => 1 },
-            { "key_as_string" => "2020-02-15T00:00:00.000Z", "key" => 1575504000000, "doc_count" => 0 },
-            { "key_as_string" => "2020-02-15T01:00:00.000Z", "key" => 1575507600000, "doc_count" => 0 },
-            { "key_as_string" => "2020-02-15T08:00:00.000Z", "key" => 1575532800000, "doc_count" => 2 }
+      'hits' => { 'total' => 3 },
+      'aggregations' => {
+        'counts' => {
+          'buckets' => [
+            { 'key_as_string' => '2020-02-14T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 1 },
+            { 'key_as_string' => '2020-02-15T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
+            { 'key_as_string' => '2020-02-15T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
+            { 'key_as_string' => '2020-02-15T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 2 }
           ]
         }
       }
     )
   end
 
-  let(:empty_response) do
-    {
-      "took" => 40,
-      "timed_out" => false,
-      "_shards" => { "total" => 11, "successful" => 11, "skipped" => 0, "failed" => 0 },
-      "hits" => { "total" => 0, "max_score" => 0.0, "hits" => [] },
-      "aggregations" => {
-        "counts" => {
-          "buckets" => []
+  let(:modsec_response) do
+    empty_response.deep_merge(
+      'hits' => { 'total' => 1 },
+      'aggregations' => {
+        'counts' => {
+          'buckets' => [
+            { 'key_as_string' => '2019-12-04T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 0 },
+            { 'key_as_string' => '2019-12-05T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
+            { 'key_as_string' => '2019-12-05T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
+            { 'key_as_string' => '2019-12-05T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 1 }
+          ]
         }
-      },
-      "status" => 200
-    }
+      }
+    )
   end
 
   subject { described_class.new(environment: environment) }
 
   describe '#execute' do
+    context 'without cluster' do
+      before do
+        allow(environment).to receive(:deployment_platform) { nil }
+      end
+
+      it 'returns no results' do
+        expect(subject.execute).to be_nil
+      end
+    end
+
     context 'without elastic_stack' do
       it 'returns no results' do
         expect(subject.execute).to be_nil
@@ -53,7 +79,7 @@ describe Security::WafAnomalySummaryService do
     context 'with default histogram' do
       before do
         allow(es_client).to receive(:msearch) do
-          { "responses" => [nginx_results, modsec_results] }
+          { 'responses' => [nginx_results, modsec_results] }
         end
 
         allow(environment.deployment_platform.cluster).to receive_message_chain(
@@ -65,7 +91,7 @@ describe Security::WafAnomalySummaryService do
         let(:nginx_results) { empty_response }
         let(:modsec_results) { empty_response }
 
-        it 'returns results' do
+        it 'returns results', :aggregate_failures do
           results = subject.execute
 
           expect(results.fetch(:status)).to eq :success
@@ -79,7 +105,7 @@ describe Security::WafAnomalySummaryService do
         let(:nginx_results) { nginx_response }
         let(:modsec_results) { empty_response }
 
-        it 'returns results' do
+        it 'returns results', :aggregate_failures do
           results = subject.execute
 
           expect(results.fetch(:status)).to eq :success
@@ -88,6 +114,20 @@ describe Security::WafAnomalySummaryService do
           expect(results.fetch(:anomalous_traffic)).to eq 0.0
         end
       end
+
+      context 'with violations' do
+        let(:nginx_results) { nginx_response }
+        let(:modsec_results) { modsec_response }
+
+        it 'returns results', :aggregate_failures do
+          results = subject.execute
+
+          expect(results.fetch(:status)).to eq :success
+          expect(results.fetch(:interval)).to eq 'day'
+          expect(results.fetch(:total_traffic)).to eq 3
+          expect(results.fetch(:anomalous_traffic)).to eq 0.33
+        end
+      end
     end
 
     context 'with time window' do
@@ -122,7 +162,7 @@ describe Security::WafAnomalySummaryService do
               )
             )
           )
-        ).and_return({ 'responses' => [{}] })
+        ).and_return({ 'responses' => [{}, {}] })
 
         subject.execute
       end
@@ -151,7 +191,7 @@ describe Security::WafAnomalySummaryService do
               )
             )
           )
-        ).and_return({ 'responses' => [{}] })
+        ).and_return({ 'responses' => [{}, {}] })
 
         subject.execute
       end