Improve Threat Monitoring backend integration

Part of [WAF statistics reporting][1].

This improves the integration with the backend for WAF statistics
reporting in a few ways.

In particular, this:

- Handles 404 responses from the WAF statistics endpoint not as errors,
  but as an indication that there is no data to display.
- Displays a new empty state when there is no data to display.
- Handles 204 No Content responses from the WAF statisics endpoint, and
  continues to poll until a 200 OK response (or any error) is received.
  To do this, the existing `pollUntilComplete` utility was moved to
  a more shared location, and tests were added for it.

[1]: https://gitlab.com/gitlab-org/gitlab/issues/14707

app/assets/javascripts/lib/utils/poll_until_complete.js

+import axios from '~/lib/utils/axios_utils';
+import Poll from './poll';
+import httpStatusCodes from './http_status';
+
+/**
+ * Polls an endpoint until it returns either a 200 OK or a error status.
+ * The Poll-Interval header in the responses are used to determine how
+ * frequently to poll.
+ *
+ * Once a 200 OK is received, the promise resolves with that response. If an
+ * error status is received, the promise rejects with the error.
+ *
+ * @param {string} url - The URL to poll.
+ * @param {Object} [config] - The config to provide to axios.get().
+ * @returns {Promise}
+ */
+export default (url, config = {}) =>
+  new Promise((resolve, reject) => {
+    const eTagPoll = new Poll({
+      resource: {
+        axiosGet(data) {
+          return axios.get(data.url, {
+            headers: {
+              'Content-Type': 'application/json',
+            },
+            ...data.config,
+          });
+        },
+      },
+      data: { url, config },
+      method: 'axiosGet',
+      successCallback: response => {
+        if (response.status === httpStatusCodes.OK) {
+          resolve(response);
+          eTagPoll.stop();
+        }
+      },
+      errorCallback: reject,
+    });
+
+    eTagPoll.makeRequest();
+  });

ee/app/assets/javascripts/threat_monitoring/components/app.vue

 <script>
-import { mapActions, mapState } from 'vuex';
+import { mapActions, mapGetters, mapState } from 'vuex';
 import { GlAlert, GlEmptyState, GlIcon, GlLink, GlPopover } from '@gitlab/ui';
 import { s__ } from '~/locale';
 import axios from '~/lib/utils/axios_utils';
@@ -26,6 +26,10 @@ export default {
       type: Number,
       required: true,
     },
+    chartEmptyStateSvgPath: {
+      type: String,
+      required: true,
+    },
     emptyStateSvgPath: {
       type: String,
       required: true,
@@ -60,6 +64,7 @@ export default {
   },
   computed: {
     ...mapState('threatMonitoring', ['isLoadingWafStatistics']),
+    ...mapGetters('threatMonitoring', ['hasHistory']),
   },
   created() {
     if (this.isWafMaybeSetUp) {
@@ -80,6 +85,11 @@ export default {
       });
     },
   },
+  chartEmptyStateDescription: s__(
+    `ThreatMonitoring|While it's rare to have no traffic coming to your
+    application, it can happen. In any event, we ask that you double check your
+    settings to make sure you've set up the WAF correctly.`,
+  ),
   emptyStateDescription: s__(
     `ThreatMonitoring|A Web Application Firewall (WAF) provides monitoring and
     rules to protect production applications. GitLab adds the modsecurity WAF
@@ -99,6 +109,7 @@ export default {
 <template>
   <gl-empty-state
     v-if="!isWafMaybeSetUp"
+    ref="emptyState"
     :title="s__('ThreatMonitoring|Web Application Firewall not enabled')"
     :description="$options.emptyStateDescription"
     :svg-path="emptyStateSvgPath"
@@ -138,9 +149,19 @@ export default {
 
     <waf-loading-skeleton v-if="isLoadingWafStatistics" class="mt-3" />
 
-    <template v-else>
+    <template v-else-if="hasHistory">
       <waf-statistics-summary class="mt-3" />
       <waf-statistics-history class="mt-3" />
     </template>
+
+    <gl-empty-state
+      v-else
+      ref="chartEmptyState"
+      :title="s__('ThreatMonitoring|No traffic to display')"
+      :description="$options.chartEmptyStateDescription"
+      :svg-path="chartEmptyStateSvgPath"
+      :primary-button-link="documentationPath"
+      :primary-button-text="__('Learn More')"
+    />
   </section>
 </template>

ee/app/assets/javascripts/threat_monitoring/index.js

@@ -8,6 +8,7 @@ export default () => {
   const {
     wafStatisticsEndpoint,
     environmentsEndpoint,
+    chartEmptyStateSvgPath,
     emptyStateSvgPath,
     documentationPath,
     defaultEnvironmentId,
@@ -28,6 +29,7 @@ export default () => {
     render(createElement) {
       return createElement(ThreatMonitoringApp, {
         props: {
+          chartEmptyStateSvgPath,
           emptyStateSvgPath,
           documentationPath,
           defaultEnvironmentId: parseInt(defaultEnvironmentId, 10),

ee/app/assets/javascripts/threat_monitoring/store/modules/threat_monitoring/actions.js

 import { s__ } from '~/locale';
 import axios from '~/lib/utils/axios_utils';
+import pollUntilComplete from '~/lib/utils/poll_until_complete';
+import httpStatusCodes from '~/lib/utils/http_status';
 import createFlash from '~/flash';
 import * as types from './mutation_types';
+import createState from './state';
 import { getTimeWindowParams } from './utils';
 
 export const setEndpoints = ({ commit }, endpoints) => commit(types.SET_ENDPOINTS, endpoints);
@@ -70,13 +73,22 @@ export const fetchWafStatistics = ({ state, dispatch }) => {
 
   dispatch('requestWafStatistics');
 
-  return axios
-    .get(state.wafStatisticsEndpoint, {
-      params: {
-        environment_id: state.currentEnvironmentId,
-        ...getTimeWindowParams(state.currentTimeWindow, Date.now()),
-      },
-    })
+  return pollUntilComplete(state.wafStatisticsEndpoint, {
+    params: {
+      environment_id: state.currentEnvironmentId,
+      ...getTimeWindowParams(state.currentTimeWindow, Date.now()),
+    },
+  })
     .then(({ data }) => dispatch('receiveWafStatisticsSuccess', data))
-    .catch(() => dispatch('receiveWafStatisticsError'));
+    .catch(error => {
+      // A NOT_FOUND resonse from the endpoint means that there is no data for
+      // the given parameters. There are various reasons *why* there could be
+      // no data, but we can't distinguish between them, yet. So, just render
+      // no data.
+      if (error.response.status === httpStatusCodes.NOT_FOUND) {
+        dispatch('receiveWafStatisticsSuccess', createState().wafStatistics);
+      } else {
+        dispatch('receiveWafStatisticsError');
+      }
+    });
 };

ee/app/assets/javascripts/threat_monitoring/store/modules/threat_monitoring/getters.js

@@ -8,3 +8,6 @@ export const currentEnvironmentName = ({ currentEnvironmentId, environments }) =
 
 export const currentTimeWindowName = ({ currentTimeWindow }) =>
   getTimeWindowConfig(currentTimeWindow).name;
+
+export const hasHistory = ({ wafStatistics }) =>
+  Boolean(wafStatistics.history.nominal.length || wafStatistics.history.anomalous.length);

ee/app/assets/javascripts/vue_shared/license_management/store/actions.js

 import axios from '~/lib/utils/axios_utils';
+import pollUntilComplete from '~/lib/utils/poll_until_complete';
 import * as types from './mutation_types';
 import { LICENSE_APPROVAL_STATUS } from '../constants';
 import { convertToOldReportFormat } from './utils';
-import { pollUntilComplete } from '../../security_reports/store/utils';
 
 export const setAPISettings = ({ commit }, data) => {
   commit(types.SET_API_SETTINGS, data);

ee/app/assets/javascripts/vue_shared/security_reports/store/actions.js

 import $ from 'jquery';
 import axios from '~/lib/utils/axios_utils';
+import pollUntilComplete from '~/lib/utils/poll_until_complete';
 import { s__, sprintf } from '~/locale';
 import { visitUrl } from '~/lib/utils/url_utility';
 import toast from '~/vue_shared/plugins/global_toast';
 import * as types from './mutation_types';
 import downloadPatchHelper from './utils/download_patch_helper';
-import { pollUntilComplete } from './utils';
 
 /**
  * A lot of this file has duplicate actions to

ee/app/assets/javascripts/vue_shared/security_reports/store/modules/sast/actions.js

 import axios from '~/lib/utils/axios_utils';
+import pollUntilComplete from '~/lib/utils/poll_until_complete';
 import * as types from './mutation_types';
-import { pollUntilComplete } from '../../utils';
 
 export const setHeadPath = ({ commit }, path) => commit(types.SET_HEAD_PATH, path);
 

ee/app/assets/javascripts/vue_shared/security_reports/store/utils.js

 import sha1 from 'sha1';
 import _ from 'underscore';
-import axios from 'axios';
 import { stripHtml } from '~/lib/utils/text_utility';
 import { n__, s__, sprintf } from '~/locale';
-import Poll from '~/lib/utils/poll';
-import httpStatusCodes from '~/lib/utils/http_status';
 
 /**
  * Returns the index of an issue in given list
@@ -444,36 +441,3 @@ export const parseDiff = (diff, enrichData) => {
     existing: diff.existing ? diff.existing.map(enrichVulnerability) : [],
   };
 };
-
-/**
- * Polls an endpoint untill it returns either an error or a 200.
- * Resolving or rejecting the promise accordingly.
- *
- * @param {String} endpoint the endpoint to poll.
- * @returns {Promise}
- */
-export const pollUntilComplete = endpoint =>
-  new Promise((resolve, reject) => {
-    const eTagPoll = new Poll({
-      resource: {
-        getReports(url) {
-          return axios.get(url, {
-            headers: {
-              'Content-Type': 'application/json',
-            },
-          });
-        },
-      },
-      data: endpoint,
-      method: 'getReports',
-      successCallback: response => {
-        if (response.status === httpStatusCodes.OK) {
-          resolve(response);
-          eTagPoll.stop();
-        }
-      },
-      errorCallback: reject,
-    });
-
-    eTagPoll.makeRequest();
-  });

ee/app/views/projects/threat_monitoring/show.html.haml

@@ -4,6 +4,7 @@
 - default_environment_id = @project.default_environment&.id || -1
 
 #js-threat-monitoring-app{ data: { documentation_path: help_page_path('user/clusters/applications', anchor: 'web-application-firewall-modsecurity'),
+  chart_empty_state_svg_path: image_path('illustrations/chart-empty-state.svg'),
   empty_state_svg_path: image_path('illustrations/security-dashboard-empty-state.svg'),
   waf_statistics_endpoint: 'dummy',
   environments_endpoint: project_environments_path(@project),

ee/spec/frontend/threat_monitoring/components/__snapshots__/app_spec.js.snap

 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`ThreatMonitoringApp component given there is a default environment shows the alert 1`] = `
+exports[`ThreatMonitoringApp component given there is a default environment with data shows the alert 1`] = `
 <gl-alert-stub
   class="my-3"
   dismissible="true"

ee/spec/frontend/threat_monitoring/components/app_spec.js

 import { shallowMount } from '@vue/test-utils';
 import MockAdapter from 'axios-mock-adapter';
 import axios from '~/lib/utils/axios_utils';
-import { GlAlert, GlEmptyState } from '@gitlab/ui';
+import { GlAlert } from '@gitlab/ui';
 import { TEST_HOST } from 'helpers/test_constants';
 import createStore from 'ee/threat_monitoring/store';
 import ThreatMonitoringApp from 'ee/threat_monitoring/components/app.vue';
@@ -9,9 +9,11 @@ import ThreatMonitoringFilters from 'ee/threat_monitoring/components/threat_moni
 import WafLoadingSkeleton from 'ee/threat_monitoring/components/waf_loading_skeleton.vue';
 import WafStatisticsHistory from 'ee/threat_monitoring/components/waf_statistics_history.vue';
 import WafStatisticsSummary from 'ee/threat_monitoring/components/waf_statistics_summary.vue';
+import { mockWafStatisticsResponse } from '../mock_data';
 
 const defaultEnvironmentId = 3;
 const documentationPath = '/docs';
+const chartEmptyStateSvgPath = '/chart-svgs';
 const emptyStateSvgPath = '/svgs';
 const environmentsEndpoint = `${TEST_HOST}/environments`;
 const wafStatisticsEndpoint = `${TEST_HOST}/waf`;
@@ -22,11 +24,12 @@ describe('ThreatMonitoringApp component', () => {
   let store;
   let wrapper;
 
-  const factory = propsData => {
+  const factory = ({ propsData, state } = {}) => {
     store = createStore();
     Object.assign(store.state.threatMonitoring, {
       environmentsEndpoint,
       wafStatisticsEndpoint,
+      ...state,
     });
 
     jest.spyOn(store, 'dispatch').mockImplementation();
@@ -34,6 +37,7 @@ describe('ThreatMonitoringApp component', () => {
     wrapper = shallowMount(ThreatMonitoringApp, {
       propsData: {
         defaultEnvironmentId,
+        chartEmptyStateSvgPath,
         emptyStateSvgPath,
         documentationPath,
         showUserCallout: true,
@@ -46,9 +50,12 @@ describe('ThreatMonitoringApp component', () => {
   };
 
   const findAlert = () => wrapper.find(GlAlert);
+  const findFilters = () => wrapper.find(ThreatMonitoringFilters);
   const findWafLoadingSkeleton = () => wrapper.find(WafLoadingSkeleton);
   const findWafStatisticsHistory = () => wrapper.find(WafStatisticsHistory);
   const findWafStatisticsSummary = () => wrapper.find(WafStatisticsSummary);
+  const findEmptyState = () => wrapper.find({ ref: 'emptyState' });
+  const findChartEmptyState = () => wrapper.find({ ref: 'chartEmptyState' });
 
   afterEach(() => {
     wrapper.destroy();
@@ -59,7 +66,9 @@ describe('ThreatMonitoringApp component', () => {
     invalidEnvironmentId => {
       beforeEach(() => {
         factory({
-          defaultEnvironmentId: invalidEnvironmentId,
+          propsData: {
+            defaultEnvironmentId: invalidEnvironmentId,
+          },
         });
       });
 
@@ -68,7 +77,7 @@ describe('ThreatMonitoringApp component', () => {
       });
 
       it('shows only the empty state', () => {
-        const emptyState = wrapper.find(GlEmptyState);
+        const emptyState = findEmptyState();
         expect(wrapper.element).toBe(emptyState.element);
         expect(emptyState.props()).toMatchObject({
           svgPath: emptyStateSvgPath,
@@ -78,9 +87,13 @@ describe('ThreatMonitoringApp component', () => {
     },
   );
 
-  describe('given there is a default environment', () => {
+  describe('given there is a default environment with data', () => {
     beforeEach(() => {
-      factory();
+      factory({
+        state: {
+          wafStatistics: mockWafStatisticsResponse,
+        },
+      });
     });
 
     it('dispatches the setCurrentEnvironmentId and fetchEnvironments actions', () => {
@@ -95,7 +108,7 @@ describe('ThreatMonitoringApp component', () => {
     });
 
     it('shows the filter bar', () => {
-      expect(wrapper.find(ThreatMonitoringFilters).exists()).toBe(true);
+      expect(findFilters().exists()).toBe(true);
     });
 
     it('shows the summary and history statistics', () => {
@@ -135,7 +148,9 @@ describe('ThreatMonitoringApp component', () => {
   describe('given showUserCallout is false', () => {
     beforeEach(() => {
       factory({
-        showUserCallout: false,
+        propsData: {
+          showUserCallout: false,
+        },
       });
     });
 
@@ -158,4 +173,23 @@ describe('ThreatMonitoringApp component', () => {
       });
     });
   });
+
+  describe('given there is a default environment with no data to display', () => {
+    beforeEach(() => {
+      factory();
+    });
+
+    it('shows the filter bar', () => {
+      expect(findFilters().exists()).toBe(true);
+    });
+
+    it('does not show the summary or history statistics', () => {
+      expect(findWafStatisticsSummary().exists()).toBe(false);
+      expect(findWafStatisticsHistory().exists()).toBe(false);
+    });
+
+    it('shows the chart empty state', () => {
+      expect(findChartEmptyState().exists()).toBe(true);
+    });
+  });
 });

ee/spec/frontend/threat_monitoring/store/modules/threat_monitoring/actions_spec.js

 import MockAdapter from 'axios-mock-adapter';
 import axios from '~/lib/utils/axios_utils';
+import httpStatus from '~/lib/utils/http_status';
 import createFlash from '~/flash';
 import testAction from 'helpers/vuex_action_helper';
 
@@ -105,7 +106,7 @@ describe('Threat Monitoring actions', () => {
 
     describe('on success', () => {
       beforeEach(() => {
-        mock.onGet(environmentsEndpoint).replyOnce(200, mockEnvironmentsResponse);
+        mock.onGet(environmentsEndpoint).replyOnce(httpStatus.OK, mockEnvironmentsResponse);
       });
 
       it('should dispatch the request and success actions', () =>
@@ -128,7 +129,7 @@ describe('Threat Monitoring actions', () => {
       beforeEach(() => {
         const oneEnvironmentPerPage = ({ totalPages }) => config => {
           const { page } = config.params;
-          const response = [200, { environments: [{ id: page }] }];
+          const response = [httpStatus.OK, { environments: [{ id: page }] }];
           if (page < totalPages) {
             response.push({ 'x-next-page': page + 1 });
           }
@@ -286,7 +287,7 @@ describe('Threat Monitoring actions', () => {
               interval: 'day',
             },
           })
-          .replyOnce(200, mockWafStatisticsResponse);
+          .replyOnce(httpStatus.OK, mockWafStatisticsResponse);
       });
 
       it('should dispatch the request and success actions', () =>
@@ -305,6 +306,34 @@ describe('Threat Monitoring actions', () => {
         ));
     });
 
+    describe('on NOT_FOUND', () => {
+      beforeEach(() => {
+        mock.onGet(wafStatisticsEndpoint).replyOnce(httpStatus.NOT_FOUND);
+      });
+
+      it('should dispatch the request and success action with empty data', () =>
+        testAction(
+          actions.fetchWafStatistics,
+          undefined,
+          state,
+          [],
+          [
+            { type: 'requestWafStatistics' },
+            {
+              type: 'receiveWafStatisticsSuccess',
+              payload: expect.objectContaining({
+                totalTraffic: 0,
+                anomalousTraffic: 0,
+                history: {
+                  nominal: [],
+                  anomalous: [],
+                },
+              }),
+            },
+          ],
+        ));
+    });
+
     describe('on error', () => {
       beforeEach(() => {
         mock.onGet(wafStatisticsEndpoint).replyOnce(500);

ee/spec/frontend/threat_monitoring/store/modules/threat_monitoring/getters_spec.js

@@ -40,4 +40,17 @@ describe('threatMonitoring module getters', () => {
       expect(getters.currentTimeWindowName(state)).toBe('30 days');
     });
   });
+
+  describe('hasHistory', () => {
+    it.each(['nominal', 'anomalous'])('returns true if there is any %s history data', type => {
+      state.wafStatistics.history[type] = ['foo'];
+      expect(getters.hasHistory(state)).toBe(true);
+    });
+
+    it('returns false if there is no history', () => {
+      state.wafStatistics.history.nominal = [];
+      state.wafStatistics.history.anomalous = [];
+      expect(getters.hasHistory(state)).toBe(false);
+    });
+  });
 });

locale/gitlab.pot

@@ -18821,6 +18821,9 @@ msgstr ""
 msgid "ThreatMonitoring|Environment"
 msgstr ""
 
+msgid "ThreatMonitoring|No traffic to display"
+msgstr ""
+
 msgid "ThreatMonitoring|Requests"
 msgstr ""
 
@@ -18851,6 +18854,9 @@ msgstr ""
 msgid "ThreatMonitoring|Web Application Firewall not enabled"
 msgstr ""
 
+msgid "ThreatMonitoring|While it's rare to have no traffic coming to your application, it can happen. In any event, we ask that you double check your settings to make sure you've set up the WAF correctly."
+msgstr ""
+
 msgid "Thursday"
 msgstr ""
 

spec/frontend/lib/utils/poll_until_complete_spec.js

+import AxiosMockAdapter from 'axios-mock-adapter';
+import axios from '~/lib/utils/axios_utils';
+import pollUntilComplete from '~/lib/utils/poll_until_complete';
+import httpStatusCodes from '~/lib/utils/http_status';
+import { TEST_HOST } from 'helpers/test_constants';
+
+const endpoint = `${TEST_HOST}/foo`;
+const mockData = 'mockData';
+const pollInterval = 1234;
+const pollIntervalHeader = {
+  'Poll-Interval': pollInterval,
+};
+
+describe('pollUntilComplete', () => {
+  let mock;
+
+  beforeEach(() => {
+    mock = new AxiosMockAdapter(axios);
+  });
+
+  afterEach(() => {
+    mock.restore();
+  });
+
+  describe('given an immediate success response', () => {
+    beforeEach(() => {
+      mock.onGet(endpoint).replyOnce(httpStatusCodes.OK, mockData);
+    });
+
+    it('resolves with the response', () =>
+      pollUntilComplete(endpoint).then(({ data }) => {
+        expect(data).toBe(mockData);
+      }));
+  });
+
+  describe(`given the endpoint returns NO_CONTENT with a Poll-Interval before succeeding`, () => {
+    beforeEach(() => {
+      mock
+        .onGet(endpoint)
+        .replyOnce(httpStatusCodes.NO_CONTENT, undefined, pollIntervalHeader)
+        .onGet(endpoint)
+        .replyOnce(httpStatusCodes.OK, mockData);
+    });
+
+    it('calls the endpoint until it succeeds, and resolves with the response', () =>
+      Promise.all([
+        pollUntilComplete(endpoint).then(({ data }) => {
+          expect(data).toBe(mockData);
+          expect(mock.history.get).toHaveLength(2);
+        }),
+
+        // To ensure the above pollUntilComplete() promise is actually
+        // fulfilled, we must explictly run the timers forward by the time
+        // indicated in the headers *after* each previous request has been
+        // fulfilled.
+        axios
+          // wait for initial NO_CONTENT response to be fulfilled
+          .waitForAll()
+          .then(() => {
+            jest.advanceTimersByTime(pollInterval);
+          }),
+      ]));
+  });
+
+  describe('given the endpoint returns an error status', () => {
+    const errorMessage = 'error message';
+
+    beforeEach(() => {
+      mock.onGet(endpoint).replyOnce(httpStatusCodes.NOT_FOUND, errorMessage);
+    });
+
+    it('rejects with the error response', () =>
+      pollUntilComplete(endpoint).catch(error => {
+        expect(error.response.data).toBe(errorMessage);
+      }));
+  });
+
+  describe('given params', () => {
+    const params = { foo: 'bar' };
+    beforeEach(() => {
+      mock.onGet(endpoint, { params }).replyOnce(httpStatusCodes.OK, mockData);
+    });
+
+    it('requests the expected URL', () =>
+      pollUntilComplete(endpoint, { params }).then(({ data }) => {
+        expect(data).toBe(mockData);
+      }));
+  });
+});