Improve scatterplot performance

- Store transformed data in store
and optimize median computation

ee/app/assets/javascripts/analytics/productivity_analytics/store/modules/charts/actions.js

 import axios from '~/lib/utils/axios_utils';
 import * as types from './mutation_types';
-import { chartKeys } from '../../../constants';
+import { getDateInPast } from '~/lib/utils/datetime_utility';
+import { chartKeys, scatterPlotAddonQueryDays } from '../../../constants';
+import { transformScatterData } from '../../../utils';
 
 /**
  * Fetches data for all charts except for the main chart
@@ -27,14 +29,28 @@ export const fetchChartData = ({ dispatch, getters, state, rootState }, chartKey
       .get(rootState.endpoint, { params })
       .then(response => {
         const { data } = response;
+
+        if (chartKey === chartKeys.scatterplot) {
+          const transformedData = transformScatterData(
+            data,
+            new Date(getDateInPast(rootState.filters.startDate, scatterPlotAddonQueryDays)),
+            new Date(rootState.filters.endDate),
+          );
+
+          dispatch('receiveChartDataSuccess', { chartKey, data, transformedData });
+        } else {
           dispatch('receiveChartDataSuccess', { chartKey, data });
+        }
       })
       .catch(error => dispatch('receiveChartDataError', { chartKey, error }));
   }
 };
 
-export const receiveChartDataSuccess = ({ commit }, { chartKey, data = {} }) => {
-  commit(types.RECEIVE_CHART_DATA_SUCCESS, { chartKey, data });
+export const receiveChartDataSuccess = (
+  { commit },
+  { chartKey, data = {}, transformedData = null },
+) => {
+  commit(types.RECEIVE_CHART_DATA_SUCCESS, { chartKey, data, transformedData });
 };
 
 export const receiveChartDataError = ({ commit }, { chartKey, error }) => {

ee/app/assets/javascripts/analytics/productivity_analytics/store/modules/charts/getters.js

@@ -54,28 +54,12 @@ export const getColumnChartData = state => chartKey => {
 
 export const chartHasData = state => chartKey => !_.isEmpty(state.charts[chartKey].data);
 
-/**
- * Creates a series array of main data for the scatterplot chart.
- *
- * Takes an object of the form
- * {
- *   "1": { "metric": 138", merged_at": "2019-07-09T14:58:07.756Z" },
- *   "2": { "metric": 139, "merged_at": "2019-07-10T11:13:23.557Z" },
- *   "3": { "metric": 24, "merged_at": "2019-07-01T07:06:23.193Z" }
- * }
- *
- * and creates the following structure:
- *
- * [
- *   ["2019-07-01T07:06:23.193Z", 24],
- *   ["2019-07-09T14:58:07.756Z", 138],
- *   ["2019-07-10T11:13:23.557Z", 139],
- * ]
- *
- * It eliminates items which were merged before the startDate (minus an additional days offset).
- */
 export const getScatterPlotMainData = (state, getters, rootState) =>
-  getScatterPlotData(state.charts.scatterplot.data, rootState.filters.startDate);
+  getScatterPlotData(
+    state.charts.scatterplot.transformedData,
+    new Date(rootState.filters.startDate),
+    new Date(rootState.filters.endDate),
+  );
 
 /**
  * Creates a series array of median data for the scatterplot chart.
@@ -83,10 +67,11 @@ export const getScatterPlotMainData = (state, getters, rootState) =>
  * It calls getMedianLineData internally with the raw scatterplot data and the computed by getters.getScatterPlotMainData.
  * scatterPlotAddonQueryDays is necessary since we query the API with an additional day offset to compute the median.
  */
-export const getScatterPlotMedianData = (state, getters) =>
+export const getScatterPlotMedianData = (state, getters, rootState) =>
   getMedianLineData(
-    state.charts.scatterplot.data,
-    getters.getScatterPlotMainData,
+    state.charts.scatterplot.transformedData,
+    new Date(rootState.filters.startDate),
+    new Date(rootState.filters.endDate),
     scatterPlotAddonQueryDays,
   );
 

ee/app/assets/javascripts/analytics/productivity_analytics/store/modules/charts/mutations.js

 import * as types from './mutation_types';
+import { chartKeys } from '../../../constants';
 
 export default {
   [types.RESET_CHART_DATA](state, chartKey) {
@@ -8,15 +9,23 @@ export default {
   [types.REQUEST_CHART_DATA](state, chartKey) {
     state.charts[chartKey].isLoading = true;
   },
-  [types.RECEIVE_CHART_DATA_SUCCESS](state, { chartKey, data }) {
+  [types.RECEIVE_CHART_DATA_SUCCESS](state, { chartKey, data, transformedData }) {
     state.charts[chartKey].isLoading = false;
     state.charts[chartKey].errorCode = null;
     state.charts[chartKey].data = data;
+
+    if (chartKey === chartKeys.scatterplot) {
+      state.charts[chartKey].transformedData = transformedData;
+    }
   },
   [types.RECEIVE_CHART_DATA_ERROR](state, { chartKey, status }) {
     state.charts[chartKey].isLoading = false;
     state.charts[chartKey].errorCode = status;
     state.charts[chartKey].data = {};
+
+    if (chartKey === chartKeys.scatterplot) {
+      state.charts[chartKey].transformedData = [];
+    }
   },
   [types.SET_METRIC_TYPE](state, { chartKey, metricType }) {
     state.charts[chartKey].params.metricType = metricType;

ee/app/assets/javascripts/analytics/productivity_analytics/store/modules/charts/state.js

@@ -39,6 +39,7 @@ export default () => ({
       errorCode: null,
       enabled: true,
       data: {},
+      transformedData: [],
       selected: [],
       params: {
         chartType: chartTypes.scatterplot,

ee/app/assets/javascripts/analytics/productivity_analytics/utils.js

@@ -90,38 +90,6 @@ export const transformScatterData = (data, startDate, endDate) => {
   return result;
 };
 
-/**
- * Transforms a given data object into an array
- * which will be used as series data for the scatterplot chart.
- * It eliminates items which were merged before a "dateInPast" and sorts
- * the result by date (ascending)
- *
- * Takes an object of the form
- * {
- *   "1": { "metric": 138", merged_at": "2019-07-09T14:58:07.756Z" },
- *   "2": { "metric": 139, "merged_at": "2019-07-10T11:13:23.557Z" },
- *   "3": { "metric": 24, "merged_at": "2019-07-01T07:06:23.193Z" }
- * }
- *
- * and creates the following two-dimensional array
- * where the first value is the "merged_at" date and the second value is the metric:
- *
- * [
- *   ["2019-07-01T07:06:23.193Z", 24],
- *   ["2019-07-09T14:58:07.756Z", 138],
- *   ["2019-07-10T11:13:23.557Z", 139],
- * ]
- *
- * @param {Object} data The raw data which will be transformed
- * @param {Date} dateInPast Date in the past
- * @returns {Array} The transformed data array sorted by date ascending
- */
-export const getScatterPlotData = (data, dateInPast) =>
-  Object.keys(data)
-    .filter(key => new Date(data[key].merged_at) >= dateInPast)
-    .map(key => [data[key].merged_at, data[key].metric])
-    .sort((a, b) => new Date(a[0]) - new Date(b[0]));
-
 /**
  * Brings the data the we receive from transformScatterData into a format that can be passed to the chart.
  * Since transformScatterData contains more data than we actually want to display on the scatterplot
@@ -145,7 +113,7 @@ export const getScatterPlotData = (data, dateInPast) =>
  * @param {*} endDate - The end date selected by the user
  * @returns {Array} An array with each item being another arry of two items (date, computed median)
  */
-export const getScatterPlotDataNew = (data, startDate, endDate) => {
+export const getScatterPlotData = (data, startDate, endDate) => {
   if (!data.length) return [];
 
   const startIndex = data.length - 1 - getDayDifference(startDate, endDate);
@@ -161,39 +129,6 @@ export const getScatterPlotDataNew = (data, startDate, endDate) => {
   return result;
 };
 
-/**
- * Computes the moving median line data.
- * It takes the raw data object (which contains historical data) and the scatterData (from getScatterPlotData)
- * and computes the median for every date in scatterData.
- * The median for a given date in scatterData (called item) is computed by taking all metrics of the raw data into account
- * which are before (or eqaul to) the the item's merged_at date
- * and after (or equal to) the item's merged_at date minus a given "daysOffset" (e.g., 30 days for "30 day rolling median")
- *
- * i.e., moving median for a given DAY is the median the range of values (DAY-30 ... DAY)
- *
- * @param {Object} data The raw data which will be used for computing the median
- * @param {Array} scatterData The transformed data from getScatterPlotData
- * @param {Number} daysOffset The number of days that is substracted from each date in scatterData (e.g. 30 days in the past)
- * @returns {Array} An array with each item being another arry of two items (date, computed median)
- */
-export const getMedianLineData = (data, scatterData, daysOffset) =>
-  scatterData.map(item => {
-    const [dateString] = item;
-    const values = Object.keys(data)
-      .filter(key => {
-        const mergedAtDate = new Date(data[key].merged_at);
-        const itemDate = new Date(dateString);
-
-        return (
-          mergedAtDate <= itemDate && mergedAtDate >= new Date(getDateInPast(itemDate, daysOffset))
-        );
-      })
-      .map(key => data[key].metric);
-
-    const computedMedian = values.length ? median(values) : 0;
-    return [dateString, computedMedian];
-  });
-
 /**
  * Computes the moving median line data, i.e, it computes the 30 day rolling median for every item displayd on the scatterplot
  * For example the 30 day rolling median for startDate=2019-09-01 and endDate=2019-09-03 is computed as follows:
@@ -210,7 +145,7 @@ export const getMedianLineData = (data, scatterData, daysOffset) =>
  * @param {Number} daysOffset The number of days that to look up data in the past (e.g. 30 days in the past for 30 day rolling median)
  * @returns {Array} An array with each item being another arry of two items (date, computed median)
  */
-export const getMedianLineDataNew = (data, startDate, endDate, daysOffset) => {
+export const getMedianLineData = (data, startDate, endDate, daysOffset) => {
   const result = [];
   const dayDiff = getDayDifference(startDate, endDate);
   const transformedData = data.map(arr => arr.map(x => x.metric));

ee/app/assets/javascripts/analytics/shared/components/scatterplot.vue

@@ -32,6 +32,7 @@ export default {
       tooltipContent: '',
       chartOption: {
         xAxis: {
+          type: 'time',
           axisLabel: {
             formatter: date => dateFormat(date, dateFormats.defaultDate),
           },
@@ -67,9 +68,9 @@ export default {
   },
   methods: {
     renderTooltip({ data }) {
-      const [xValue, yValue] = data;
-      this.tooltipTitle = yValue;
-      this.tooltipContent = dateFormat(xValue, dateFormats.defaultDateTime);
+      const [, metric, dateTime] = data;
+      this.tooltipTitle = metric;
+      this.tooltipContent = dateFormat(dateTime, dateFormats.defaultDateTime);
     },
   },
 };

ee/spec/frontend/analytics/productivity_analytics/components/app_spec.js

@@ -82,6 +82,11 @@ describe('ProductivityApp component', () => {
 
     describe('with a group being selected', () => {
       beforeEach(() => {
+        wrapper.vm.$store.dispatch('filters/setDateRange', {
+          skipFetch: true,
+          startDate: new Date('2019-09-01'),
+          endDate: new Date('2019-09-02'),
+        });
         wrapper.vm.$store.dispatch('filters/setGroupNamespace', 'gitlab-org');
         mock.onGet(wrapper.vm.$store.state.endpoint).replyOnce(200);
       });
@@ -277,9 +282,13 @@ describe('ProductivityApp component', () => {
                       wrapper.vm.$store.dispatch('charts/receiveChartDataSuccess', {
                         chartKey: chartKeys.scatterplot,
                         data: {
-                          1: { metric: 2, merged_at: '2019-07-01T07:06:23.193Z' },
-                          2: { metric: 3, merged_at: '2019-07-05T08:27:42.411Z' },
+                          1: { metric: 2, merged_at: '2019-09-01T07:06:23.193Z' },
+                          2: { metric: 3, merged_at: '2019-09-05T08:27:42.411Z' },
                         },
+                        transformedData: [
+                          [{ metric: 2, merged_at: '2019-09-01T07:06:23.193Z' }],
+                          [{ metric: 3, merged_at: '2019-09-05T08:27:42.411Z' }],
+                        ],
                       });
                     });
 

ee/spec/frontend/analytics/productivity_analytics/store/modules/charts/actions_spec.js

@@ -6,7 +6,13 @@ import * as actions from 'ee/analytics/productivity_analytics/store/modules/char
 import * as types from 'ee/analytics/productivity_analytics/store/modules/charts/mutation_types';
 import getInitialState from 'ee/analytics/productivity_analytics/store/modules/charts/state';
 import { chartKeys } from 'ee/analytics/productivity_analytics/constants';
-import { mockHistogramData } from '../../../mock_data';
+import { mockHistogramData, mockScatterplotData } from '../../../mock_data';
+
+jest.mock('ee/analytics/productivity_analytics/utils', () => ({
+  transformScatterData: jest
+    .fn()
+    .mockImplementation(() => [[{ merged_at: '2019-09-01T00:00:000Z', metric: 10 }]]),
+}));
 
 describe('Productivity analytics chart actions', () => {
   let mockedContext;
@@ -24,6 +30,10 @@ describe('Productivity analytics chart actions', () => {
       dispatch() {},
       rootState: {
         endpoint: `${TEST_HOST}/analytics/productivity_analytics.json`,
+        filters: {
+          startDate: '2019-09-01',
+          endDate: '2091-09-05',
+        },
       },
       getters: {
         getFilterParams: () => globalParams,
@@ -49,6 +59,7 @@ describe('Productivity analytics chart actions', () => {
   describe('fetchChartData', () => {
     describe('when chart is enabled', () => {
       describe('success', () => {
+        describe('histogram charts', () => {
           beforeEach(() => {
             mock.onGet(mockedState.endpoint).replyOnce(200, mockHistogramData);
           });
@@ -78,6 +89,34 @@ describe('Productivity analytics chart actions', () => {
             ));
         });
 
+        describe('scatterplot chart', () => {
+          beforeEach(() => {
+            mock.onGet(mockedState.endpoint).replyOnce(200, mockScatterplotData);
+          });
+
+          it('dispatches success with received data and transformedData', done => {
+            testAction(
+              actions.fetchChartData,
+              chartKeys.scatterplot,
+              mockedState,
+              [],
+              [
+                { type: 'requestChartData', payload: chartKeys.scatterplot },
+                {
+                  type: 'receiveChartDataSuccess',
+                  payload: {
+                    chartKey: chartKeys.scatterplot,
+                    data: mockScatterplotData,
+                    transformedData: [[{ merged_at: '2019-09-01T00:00:000Z', metric: 10 }]],
+                  },
+                },
+              ],
+              done,
+            );
+          });
+        });
+      });
+
       describe('error', () => {
         beforeEach(() => {
           mock.onGet(mockedState.endpoint).replyOnce(500);
@@ -149,7 +188,7 @@ describe('Productivity analytics chart actions', () => {
         [
           {
             type: types.RECEIVE_CHART_DATA_SUCCESS,
-            payload: { chartKey, data: mockHistogramData },
+            payload: { chartKey, data: mockHistogramData, transformedData: null },
           },
         ],
         [],

ee/spec/frontend/analytics/productivity_analytics/store/modules/charts/getters_spec.js

@@ -8,7 +8,7 @@ import {
   scatterPlotAddonQueryDays,
 } from 'ee/analytics/productivity_analytics/constants';
 import { getScatterPlotData, getMedianLineData } from 'ee/analytics/productivity_analytics/utils';
-import { mockHistogramData, mockScatterplotData } from '../../../mock_data';
+import { mockHistogramData } from '../../../mock_data';
 
 jest.mock('ee/analytics/productivity_analytics/utils');
 
@@ -17,6 +17,10 @@ describe('Productivity analytics chart getters', () => {
 
   const groupNamespace = 'gitlab-org';
   const projectPath = 'gitlab-org/gitlab-test';
+  const transformedData = [
+    [{ merged_at: '2019-09-01T00:00:000Z', metric: 10 }],
+    [{ merged_at: '2019-09-02T00:00:000Z', metric: 20 }],
+  ];
 
   beforeEach(() => {
     state = createState();
@@ -54,33 +58,42 @@ describe('Productivity analytics chart getters', () => {
 
   describe('getScatterPlotMainData', () => {
     it('calls getScatterPlotData with the raw scatterplot data and the date in past', () => {
-      state.charts.scatterplot.data = mockScatterplotData;
+      state.charts.scatterplot.transformedData = transformedData;
 
       const rootState = {
         filters: {
-          startDate: '2019-07-16',
+          startDate: '2019-09-01',
+          endDate: '2019-09-05',
         },
       };
 
       getters.getScatterPlotMainData(state, null, rootState);
 
-      expect(getScatterPlotData).toHaveBeenCalledWith(mockScatterplotData, '2019-07-16');
+      expect(getScatterPlotData).toHaveBeenCalledWith(
+        transformedData,
+        new Date(rootState.filters.startDate),
+        new Date(rootState.filters.endDate),
+      );
     });
   });
 
   describe('getScatterPlotMedianData', () => {
     it('calls getMedianLineData with the raw scatterplot data, the getScatterPlotMainData getter and the an additional days offset', () => {
-      state.charts.scatterplot.data = mockScatterplotData;
+      state.charts.scatterplot.transformedData = transformedData;
 
-      const mockGetters = {
-        getScatterPlotMainData: jest.fn(),
+      const rootState = {
+        filters: {
+          startDate: '2019-09-01',
+          endDate: '2019-09-05',
+        },
       };
 
-      getters.getScatterPlotMedianData(state, mockGetters);
+      getters.getScatterPlotMedianData(state, null, rootState);
 
       expect(getMedianLineData).toHaveBeenCalledWith(
-        mockScatterplotData,
-        mockGetters.getScatterPlotMainData,
+        transformedData,
+        new Date(rootState.filters.startDate),
+        new Date(rootState.filters.endDate),
         scatterPlotAddonQueryDays,
       );
     });

ee/spec/frontend/analytics/productivity_analytics/store/modules/charts/mutations_spec.js

@@ -2,7 +2,7 @@ import * as types from 'ee/analytics/productivity_analytics/store/modules/charts
 import mutations from 'ee/analytics/productivity_analytics/store/modules/charts/mutations';
 import getInitialState from 'ee/analytics/productivity_analytics/store/modules/charts/state';
 import { chartKeys } from 'ee/analytics/productivity_analytics/constants';
-import { mockHistogramData } from '../../../mock_data';
+import { mockHistogramData, mockScatterplotData } from '../../../mock_data';
 
 describe('Productivity analytics chart mutations', () => {
   let state;
@@ -37,23 +37,54 @@ describe('Productivity analytics chart mutations', () => {
       expect(state.charts[chartKey].errorCode).toBe(null);
       expect(state.charts[chartKey].data).toEqual(mockHistogramData);
     });
+
+    it('updates the transformedData when chartKey=scatterplot', () => {
+      const transformedData = [
+        [
+          {
+            metric: 139,
+            merged_at: '2019-08-18T22:00:00.000Z',
+          },
+        ],
+        [
+          {
+            metric: 138,
+            merged_at: '2019-08-17T22:00:00.000Z',
+          },
+        ],
+      ];
+      mutations[types.RECEIVE_CHART_DATA_SUCCESS](state, {
+        chartKey: chartKeys.scatterplot,
+        data: mockScatterplotData,
+        transformedData,
+      });
+
+      expect(state.charts[chartKey].isLoading).toBe(false);
+      expect(state.charts[chartKey].errorCode).toBe(null);
+      expect(state.charts[chartKey].data).toEqual(mockScatterplotData);
+      expect(state.charts[chartKey].transformedData).toEqual(transformedData);
+    });
   });
 
   describe(types.RECEIVE_CHART_DATA_ERROR, () => {
     const status = 500;
-    beforeEach(() => {
-      mutations[types.RECEIVE_CHART_DATA_ERROR](state, { chartKey, status });
-    });
 
     it('sets errorCode to 500', () => {
+      mutations[types.RECEIVE_CHART_DATA_ERROR](state, { chartKey, status });
       expect(state.charts[chartKey].isLoading).toBe(false);
       expect(state.charts[chartKey].errorCode).toBe(status);
     });
 
     it('clears data', () => {
+      mutations[types.RECEIVE_CHART_DATA_ERROR](state, { chartKey, status });
       expect(state.charts[chartKey].isLoading).toBe(false);
       expect(state.charts[chartKey].data).toEqual({});
     });
+
+    it('clears transformedData when chartKey=scatterplot', () => {
+      mutations[types.RECEIVE_CHART_DATA_ERROR](state, { chartKey: chartKeys.scatterplot, status });
+      expect(state.charts[chartKey].transformedData).toEqual([]);
+    });
   });
 
   describe(types.SET_METRIC_TYPE, () => {

ee/spec/frontend/analytics/productivity_analytics/utils_spec.js

@@ -4,13 +4,9 @@ import {
   initDateArray,
   transformScatterData,
   getScatterPlotData,
-  getScatterPlotDataNew,
   getMedianLineData,
-  getMedianLineDataNew,
 } from 'ee/analytics/productivity_analytics/utils';
 
-import { mockScatterplotData } from './mock_data';
-
 describe('Productivity Analytics utils', () => {
   const namespacePath = 'gitlab-org';
   const projectWithNamespace = 'gitlab-org/gitlab-test';
@@ -72,26 +68,6 @@ describe('Productivity Analytics utils', () => {
   });
 
   describe('getScatterPlotData', () => {
-    it('filters out data before given "dateInPast", transforms the data and sorts by date ascending', () => {
-      const dateInPast = new Date(2019, 7, 9); // '2019-08-09T22:00:00.000Z';
-      const result = getScatterPlotData(mockScatterplotData, dateInPast);
-      const expected = [
-        ['2019-08-09T22:00:00.000Z', 44],
-        ['2019-08-10T22:00:00.000Z', 46],
-        ['2019-08-11T22:00:00.000Z', 62],
-        ['2019-08-12T22:00:00.000Z', 60],
-        ['2019-08-13T22:00:00.000Z', 43],
-        ['2019-08-14T22:00:00.000Z', 46],
-        ['2019-08-15T22:00:00.000Z', 56],
-        ['2019-08-16T22:00:00.000Z', 24],
-        ['2019-08-17T22:00:00.000Z', 138],
-        ['2019-08-18T22:00:00.000Z', 139],
-      ];
-      expect(result).toEqual(expected);
-    });
-  });
-
-  describe('getScatterPlotDataNew', () => {
     it('returns a subset of data for the given start and end date and flattens the data', () => {
       const startDate = new Date('2019-08-02');
       const endDate = new Date('2019-08-04');
@@ -104,7 +80,7 @@ describe('Productivity Analytics utils', () => {
           { merged_at: '2019-08-04T16:00:00.000Z', metric: 60 },
         ],
       ];
-      const result = getScatterPlotDataNew(data, startDate, endDate);
+      const result = getScatterPlotData(data, startDate, endDate);
       const expected = [
         ['2019-08-02', 30, '2019-08-02T13:00:00.000Z'],
         ['2019-08-03', 40, '2019-08-03T14:00:00.000Z'],
@@ -116,25 +92,6 @@ describe('Productivity Analytics utils', () => {
   });
 
   describe('getMedianLineData', () => {
-    const daysOffset = 10;
-
-    it(`computes the median for every item in the scatterData array for the past ${daysOffset} days`, () => {
-      const scatterData = [
-        ['2019-08-16T22:00:00.000Z', 24],
-        ['2019-08-17T22:00:00.000Z', 138],
-        ['2019-08-18T22:00:00.000Z', 139],
-      ];
-      const result = getMedianLineData(mockScatterplotData, scatterData, daysOffset);
-      const expected = [
-        ['2019-08-16T22:00:00.000Z', 51],
-        ['2019-08-17T22:00:00.000Z', 51],
-        ['2019-08-18T22:00:00.000Z', 56],
-      ];
-      expect(result).toEqual(expected);
-    });
-  });
-
-  describe('getMedianLineDataNew', () => {
     const daysOffset = 2;
 
     it(`computes the median for every date in the data array based on the past ${daysOffset} days`, () => {
@@ -151,7 +108,7 @@ describe('Productivity Analytics utils', () => {
         [{ merged_at: '2019-08-05T17:00:00.000Z', metric: 70 }],
         [{ merged_at: '2019-08-06T18:00:00.000Z', metric: 80 }],
       ];
-      const result = getMedianLineDataNew(data, startDate, endDate, daysOffset);
+      const result = getMedianLineData(data, startDate, endDate, daysOffset);
       const expected = [['2019-08-04', 45], ['2019-08-05', 55], ['2019-08-06', 65]];
       expect(result).toEqual(expected);
     });