Pipeline Graph: Add ability to switch between views

app/assets/javascripts/pipelines/components/graph/graph_component.vue

@@ -2,7 +2,7 @@
 import { reportToSentry } from '../../utils';
 import LinkedGraphWrapper from '../graph_shared/linked_graph_wrapper.vue';
 import LinksLayer from '../graph_shared/links_layer.vue';
-import { DOWNSTREAM, MAIN, UPSTREAM, ONE_COL_WIDTH } from './constants';
+import { DOWNSTREAM, MAIN, UPSTREAM, ONE_COL_WIDTH, STAGE_VIEW } from './constants';
 import LinkedPipelinesColumn from './linked_pipelines_column.vue';
 import StageColumnComponent from './stage_column_component.vue';
 import { validateConfigPaths } from './utils';
@@ -25,11 +25,20 @@ export default {
       type: Object,
       required: true,
     },
+    viewType: {
+      type: String,
+      required: true,
+    },
     isLinkedPipeline: {
       type: Boolean,
       required: false,
       default: false,
     },
+    pipelineLayers: {
+      type: Array,
+      required: false,
+      default: () => [],
+    },
     type: {
       type: String,
       required: false,
@@ -63,8 +72,8 @@ export default {
     downstreamPipelines() {
       return this.hasDownstreamPipelines ? this.pipeline.downstream : [];
     },
-    graph() {
-      return this.pipeline.stages;
+    layout() {
+      return this.isStageView ? this.pipeline.stages : this.generateColumnsFromLayersList();
     },
     hasDownstreamPipelines() {
       return Boolean(this.pipeline?.downstream?.length > 0);
@@ -72,12 +81,18 @@ export default {
     hasUpstreamPipelines() {
       return Boolean(this.pipeline?.upstream?.length > 0);
     },
+    isStageView() {
+      return this.viewType === STAGE_VIEW;
+    },
     metricsConfig() {
       return {
         path: this.configPaths.metricsPath,
         collectMetrics: true,
       };
     },
+    shouldHideLinks() {
+      return this.isStageView;
+    },
     // The show downstream check prevents showing redundant linked columns
     showDownstreamPipelines() {
       return (
@@ -101,6 +116,26 @@ export default {
     this.getMeasurements();
   },
   methods: {
+    generateColumnsFromLayersList() {
+      return this.pipelineLayers.map((layers, idx) => {
+        /*
+          look up the groups in each layer,
+          then add each set of layer groups to a stage-like object
+        */
+
+        const groups = layers.map((id) => {
+          const { stageIdx, groupIdx } = this.pipeline.stagesLookup[id];
+          return this.pipeline.stages?.[stageIdx]?.groups?.[groupIdx];
+        });
+
+        return {
+          name: '',
+          id: `layer-${idx}`,
+          status: { action: null },
+          groups: groups.filter(Boolean),
+        };
+      });
+    },
     getMeasurements() {
       this.measurements = {
         width: this.$refs[this.containerId].scrollWidth,
@@ -147,29 +182,31 @@ export default {
             :linked-pipelines="upstreamPipelines"
             :column-title="__('Upstream')"
             :type="$options.pipelineTypeConstants.UPSTREAM"
+            :view-type="viewType"
             @error="onError"
           />
         </template>
         <template #main>
           <div :id="containerId" :ref="containerId">
             <links-layer
-              :pipeline-data="graph"
+              :pipeline-data="layout"
               :pipeline-id="pipeline.id"
               :container-id="containerId"
               :container-measurements="measurements"
               :highlighted-job="hoveredJobName"
               :metrics-config="metricsConfig"
-              :never-show-links="true"
+              :never-show-links="shouldHideLinks"
+              :view-type="viewType"
               default-link-color="gl-stroke-transparent"
               @error="onError"
               @highlightedJobsChange="updateHighlightedJobs"
             >
               <stage-column-component
-                v-for="stage in graph"
-                :key="stage.name"
-                :title="stage.name"
-                :groups="stage.groups"
-                :action="stage.status.action"
+                v-for="column in layout"
+                :key="column.id || column.name"
+                :title="column.name"
+                :groups="column.groups"
+                :action="column.status.action"
                 :highlighted-jobs="highlightedJobs"
                 :job-hovered="hoveredJobName"
                 :pipeline-expanded="pipelineExpanded"
@@ -189,6 +226,7 @@ export default {
             :linked-pipelines="downstreamPipelines"
             :column-title="__('Downstream')"
             :type="$options.pipelineTypeConstants.DOWNSTREAM"
+            :view-type="viewType"
             @downstreamHovered="setJob"
             @pipelineExpandToggle="togglePipelineExpanded"
             @scrollContainer="slidePipelineContainer"

app/assets/javascripts/pipelines/components/graph/graph_component_wrapper.vue

@@ -5,7 +5,8 @@ import { __ } from '~/locale';
 import glFeatureFlagMixin from '~/vue_shared/mixins/gl_feature_flags_mixin';
 import { DEFAULT, DRAW_FAILURE, LOAD_FAILURE } from '../../constants';
 import { reportToSentry } from '../../utils';
-import { IID_FAILURE, STAGE_VIEW } from './constants';
+import { listByLayers } from '../parsing_utils';
+import { IID_FAILURE, LAYER_VIEW, STAGE_VIEW } from './constants';
 import PipelineGraph from './graph_component.vue';
 import GraphViewSelector from './graph_view_selector.vue';
 import {
@@ -43,6 +44,7 @@ export default {
       alertType: null,
       currentViewType: STAGE_VIEW,
       pipeline: null,
+      pipelineLayers: null,
       showAlert: false,
     };
   },
@@ -155,6 +157,13 @@ export default {
     reportToSentry(this.$options.name, `error: ${err}, info: ${info}`);
   },
   methods: {
+    getPipelineLayers() {
+      if (this.currentViewType === LAYER_VIEW && !this.pipelineLayers) {
+        this.pipelineLayers = listByLayers(this.pipeline);
+      }
+
+      return this.pipelineLayers;
+    },
     hideAlert() {
       this.showAlert = false;
       this.alertType = null;
@@ -192,6 +201,8 @@ export default {
       v-if="pipeline"
       :config-paths="configPaths"
       :pipeline="pipeline"
+      :pipeline-layers="getPipelineLayers()"
+      :view-type="currentViewType"
       @error="reportFailure"
       @refreshPipelineGraph="refreshPipelineGraph"
     />

app/assets/javascripts/pipelines/components/graph/graph_view_selector.vue

@@ -55,16 +55,16 @@ export default {
 </script>
 
 <template>
-  <div class="gl-display-flex gl-justify-content-end gl-align-items-center gl-my-4">
+  <div class="gl-display-flex gl-align-items-center gl-my-4">
     <span>{{ $options.i18n.labelText }}</span>
-    <gl-dropdown class="gl-ml-4" :right="true">
+    <gl-dropdown class="gl-ml-4">
       <template #button-content>
         <gl-sprintf :message="currentDropdownText">
           <template #code="{ content }">
             <code> {{ content }} </code>
           </template>
         </gl-sprintf>
-        <gl-icon class="gl-px-2" name="angle-down" :size="18" />
+        <gl-icon class="gl-px-2" name="angle-down" :size="16" />
       </template>
       <gl-dropdown-item
         v-for="view in $options.views"

app/assets/javascripts/pipelines/components/graph/linked_pipelines_column.vue

@@ -2,7 +2,8 @@
 import getPipelineDetails from 'shared_queries/pipelines/get_pipeline_details.query.graphql';
 import { LOAD_FAILURE } from '../../constants';
 import { reportToSentry } from '../../utils';
-import { ONE_COL_WIDTH, UPSTREAM } from './constants';
+import { listByLayers } from '../parsing_utils';
+import { ONE_COL_WIDTH, UPSTREAM, LAYER_VIEW } from './constants';
 import LinkedPipeline from './linked_pipeline.vue';
 import {
   getQueryHeaders,
@@ -35,11 +36,16 @@ export default {
       type: String,
       required: true,
     },
+    viewType: {
+      type: String,
+      required: true,
+    },
   },
   data() {
     return {
       currentPipeline: null,
       loadingPipelineId: null,
+      pipelineLayers: {},
       pipelineExpanded: false,
     };
   },
@@ -123,6 +129,13 @@ export default {
 
       toggleQueryPollingByVisibility(this.$apollo.queries.currentPipeline);
     },
+    getPipelineLayers(id) {
+      if (this.viewType === LAYER_VIEW && !this.pipelineLayers[id]) {
+        this.pipelineLayers[id] = listByLayers(this.currentPipeline);
+      }
+
+      return this.pipelineLayers[id];
+    },
     isExpanded(id) {
       return Boolean(this.currentPipeline?.id && id === this.currentPipeline.id);
     },
@@ -203,7 +216,9 @@ export default {
               class="d-inline-block gl-mt-n2"
               :config-paths="configPaths"
               :pipeline="currentPipeline"
+              :pipeline-layers="getPipelineLayers(pipeline.id)"
               :is-linked-pipeline="true"
+              :view-type="viewType"
             />
           </div>
         </li>

app/assets/javascripts/pipelines/components/graph/utils.js

 import Visibility from 'visibilityjs';
 import { getIdFromGraphQLId } from '~/graphql_shared/utils';
-import { unwrapStagesWithNeeds } from '../unwrapping_utils';
+import { unwrapStagesWithNeedsAndLookup } from '../unwrapping_utils';
 
 const addMulti = (mainPipelineProjectPath, linkedPipeline) => {
   return {
@@ -86,12 +86,13 @@ const unwrapPipelineData = (mainPipelineProjectPath, data) => {
     stages: { nodes: stages },
   } = pipeline;
 
-  const nodes = unwrapStagesWithNeeds(stages);
+  const { stages: updatedStages, lookup } = unwrapStagesWithNeedsAndLookup(stages);
 
   return {
     ...pipeline,
     id: getIdFromGraphQLId(pipeline.id),
-    stages: nodes,
+    stages: updatedStages,
+    stagesLookup: lookup,
     upstream: upstream
       ? [upstream].map(addMulti.bind(null, mainPipelineProjectPath)).map(transformId)
       : [],

app/assets/javascripts/pipelines/components/graph_shared/links_inner.vue

@@ -11,6 +11,7 @@ import {
 import { performanceMarkAndMeasure } from '~/performance/utils';
 import { DRAW_FAILURE } from '../../constants';
 import { createJobsHash, generateJobNeedsDict, reportToSentry } from '../../utils';
+import { STAGE_VIEW } from '../graph/constants';
 import { parseData } from '../parsing_utils';
 import { reportPerformance } from './api';
 import { generateLinksData } from './drawing_utils';
@@ -54,11 +55,17 @@ export default {
       required: false,
       default: '',
     },
+    viewType: {
+      type: String,
+      required: false,
+      default: STAGE_VIEW,
+    },
   },
   data() {
     return {
       links: [],
       needsObject: null,
+      parsedData: {},
     };
   },
   computed: {
@@ -108,6 +115,15 @@ export default {
     highlightedJobs(jobs) {
       this.$emit('highlightedJobsChange', jobs);
     },
+    viewType() {
+      /*
+        We need to wait a tick so that the layout reflows
+        before the links refresh.
+      */
+      this.$nextTick(() => {
+        this.refreshLinks();
+      });
+    },
   },
   errorCaptured(err, _vm, info) {
     reportToSentry(this.$options.name, `error: ${err}, info: ${info}`);
@@ -166,14 +182,17 @@ export default {
       this.beginPerfMeasure();
       try {
         const arrayOfJobs = this.pipelineData.flatMap(({ groups }) => groups);
-        const parsedData = parseData(arrayOfJobs);
-        this.links = generateLinksData(parsedData, this.containerId, `-${this.pipelineId}`);
+        this.parsedData = parseData(arrayOfJobs);
+        this.refreshLinks();
       } catch (err) {
         this.$emit('error', { type: DRAW_FAILURE, reportToSentry: false });
         reportToSentry(this.$options.name, err);
       }
       this.finishPerfMeasureAndSend();
     },
+    refreshLinks() {
+      this.links = generateLinksData(this.parsedData, this.containerId, `-${this.pipelineId}`);
+    },
     getLinkClasses(link) {
       return [
         this.isLinkHighlighted(link.ref) ? 'gl-stroke-blue-400' : this.defaultLinkColor,

app/assets/javascripts/pipelines/components/parsing_utils.js

 import { uniqWith, isEqual } from 'lodash';
+import { createSankey } from './dag/drawing_utils';
 
 /*
     The following functions are the main engine in transforming the data as
@@ -144,3 +145,28 @@ export const getMaxNodes = (nodes) => {
 export const removeOrphanNodes = (sankeyfiedNodes) => {
   return sankeyfiedNodes.filter((node) => node.sourceLinks.length || node.targetLinks.length);
 };
+
+/*
+  This utility accepts unwrapped pipeline data in the format returned from
+  our standard pipeline GraphQL query and returns a list of names by layer
+  for the layer view. It can be combined with the stageLookup on the pipeline
+  to generate columns by layer.
+*/
+
+export const listByLayers = ({ stages }) => {
+  const arrayOfJobs = stages.flatMap(({ groups }) => groups);
+  const parsedData = parseData(arrayOfJobs);
+  const dataWithLayers = createSankey()(parsedData);
+
+  return dataWithLayers.nodes.reduce((acc, { layer, name }) => {
+    /* sort groups by layer */
+
+    if (!acc[layer]) {
+      acc[layer] = [];
+    }
+
+    acc[layer].push(name);
+
+    return acc;
+  }, []);
+};

app/assets/javascripts/pipelines/components/unwrapping_utils.js

 import { reportToSentry } from '../utils';
 
 const unwrapGroups = (stages) => {
-  return stages.map((stage) => {
+  return stages.map((stage, idx) => {
     const {
       groups: { nodes: groups },
     } = stage;
-    return { ...stage, groups };
+    return { node: { ...stage, groups }, lookup: { stageIdx: idx } };
   });
 };
 
@@ -23,20 +23,34 @@ const unwrapJobWithNeeds = (denodedJobArray) => {
   return unwrapNodesWithName(denodedJobArray, 'needs');
 };
 
-const unwrapStagesWithNeeds = (denodedStages) => {
+const unwrapStagesWithNeedsAndLookup = (denodedStages) => {
   const unwrappedNestedGroups = unwrapGroups(denodedStages);
 
-  const nodes = unwrappedNestedGroups.map((node) => {
+  const lookupMap = {};
+
+  const nodes = unwrappedNestedGroups.map(({ node, lookup }) => {
     const { groups } = node;
-    const groupsWithJobs = groups.map((group) => {
+    const groupsWithJobs = groups.map((group, idx) => {
       const jobs = unwrapJobWithNeeds(group.jobs.nodes);
+
+      lookupMap[group.name] = { ...lookup, groupIdx: idx };
       return { ...group, jobs };
     });
 
     return { ...node, groups: groupsWithJobs };
   });
 
-  return nodes;
+  return { stages: nodes, lookup: lookupMap };
 };
 
-export { unwrapGroups, unwrapNodesWithName, unwrapJobWithNeeds, unwrapStagesWithNeeds };
+const unwrapStagesWithNeeds = (denodedStages) => {
+  return unwrapStagesWithNeedsAndLookup(denodedStages).stages;
+};
+
+export {
+  unwrapGroups,
+  unwrapJobWithNeeds,
+  unwrapNodesWithName,
+  unwrapStagesWithNeeds,
+  unwrapStagesWithNeedsAndLookup,
+};

spec/frontend/pipelines/graph/graph_component_spec.js

 import { mount, shallowMount } from '@vue/test-utils';
-import { GRAPHQL } from '~/pipelines/components/graph/constants';
+import { GRAPHQL, STAGE_VIEW } from '~/pipelines/components/graph/constants';
 import PipelineGraph from '~/pipelines/components/graph/graph_component.vue';
 import JobItem from '~/pipelines/components/graph/job_item.vue';
 import LinkedPipelinesColumn from '~/pipelines/components/graph/linked_pipelines_column.vue';
@@ -20,6 +20,7 @@ describe('graph component', () => {
 
   const defaultProps = {
     pipeline: generateResponse(mockPipelineResponse, 'root/fungi-xoxo'),
+    viewType: STAGE_VIEW,
     configPaths: {
       metricsPath: '',
       graphqlResourceEtag: 'this/is/a/path',

spec/frontend/pipelines/graph/graph_component_wrapper_spec.js

 import { GlAlert, GlLoadingIcon } from '@gitlab/ui';
-import { shallowMount } from '@vue/test-utils';
+import { mount, shallowMount } from '@vue/test-utils';
 import Vue from 'vue';
 import VueApollo from 'vue-apollo';
 import createMockApollo from 'helpers/mock_apollo_helper';
 import getPipelineDetails from 'shared_queries/pipelines/get_pipeline_details.query.graphql';
-import { IID_FAILURE } from '~/pipelines/components/graph/constants';
+import { IID_FAILURE, LAYER_VIEW, STAGE_VIEW } from '~/pipelines/components/graph/constants';
 import PipelineGraph from '~/pipelines/components/graph/graph_component.vue';
 import PipelineGraphWrapper from '~/pipelines/components/graph/graph_component_wrapper.vue';
 import GraphViewSelector from '~/pipelines/components/graph/graph_view_selector.vue';
+import StageColumnComponent from '~/pipelines/components/graph/stage_column_component.vue';
+import * as parsingUtils from '~/pipelines/components/parsing_utils';
 import { mockPipelineResponse } from './mock_data';
 
 const defaultProvide = {
@@ -24,6 +26,9 @@ describe('Pipeline graph wrapper', () => {
   const getAlert = () => wrapper.find(GlAlert);
   const getLoadingIcon = () => wrapper.find(GlLoadingIcon);
   const getGraph = () => wrapper.find(PipelineGraph);
+  const getStageColumnTitle = () => wrapper.find('[data-testid="stage-column-title"]');
+  const getAllStageColumnGroupsInColumn = () =>
+    wrapper.find(StageColumnComponent).findAll('[data-testid="stage-column-group"]');
   const getViewSelector = () => wrapper.find(GraphViewSelector);
 
   const createComponent = ({
@@ -48,12 +53,13 @@ describe('Pipeline graph wrapper', () => {
 
   const createComponentWithApollo = ({
     getPipelineDetailsHandler = jest.fn().mockResolvedValue(mockPipelineResponse),
+    mountFn = shallowMount,
     provide = {},
   } = {}) => {
     const requestHandlers = [[getPipelineDetails, getPipelineDetailsHandler]];
 
     const apolloProvider = createMockApollo(requestHandlers);
-    createComponent({ apolloProvider, provide });
+    createComponent({ apolloProvider, provide, mountFn });
   };
 
   afterEach(() => {
@@ -223,13 +229,16 @@ describe('Pipeline graph wrapper', () => {
     });
 
     describe('when feature flag is on', () => {
+      let layersFn;
       beforeEach(async () => {
+        layersFn = jest.spyOn(parsingUtils, 'listByLayers');
         createComponentWithApollo({
           provide: {
             glFeatures: {
               pipelineGraphLayersView: true,
             },
           },
+          mountFn: mount,
         });
 
         jest.runOnlyPendingTimers();
@@ -239,6 +248,26 @@ describe('Pipeline graph wrapper', () => {
       it('appears', () => {
         expect(getViewSelector().exists()).toBe(true);
       });
+
+      it('switches between views', async () => {
+        const groupsInFirstColumn =
+          mockPipelineResponse.data.project.pipeline.stages.nodes[0].groups.nodes.length;
+        expect(getAllStageColumnGroupsInColumn()).toHaveLength(groupsInFirstColumn);
+        expect(getStageColumnTitle().text()).toBe('Build');
+        await getViewSelector().vm.$emit('updateViewType', LAYER_VIEW);
+        expect(getAllStageColumnGroupsInColumn()).toHaveLength(groupsInFirstColumn + 1);
+        expect(getStageColumnTitle().text()).toBe('');
+      });
+
+      it('calls listByLayers only once no matter how many times view is switched', async () => {
+        expect(layersFn).not.toHaveBeenCalled();
+        await getViewSelector().vm.$emit('updateViewType', LAYER_VIEW);
+        expect(layersFn).toHaveBeenCalledTimes(1);
+        await getViewSelector().vm.$emit('updateViewType', STAGE_VIEW);
+        await getViewSelector().vm.$emit('updateViewType', LAYER_VIEW);
+        await getViewSelector().vm.$emit('updateViewType', STAGE_VIEW);
+        expect(layersFn).toHaveBeenCalledTimes(1);
+      });
     });
   });
 });

spec/frontend/pipelines/graph/linked_pipelines_column_spec.js

@@ -2,10 +2,17 @@ import { mount, shallowMount, createLocalVue } from '@vue/test-utils';
 import VueApollo from 'vue-apollo';
 import createMockApollo from 'helpers/mock_apollo_helper';
 import getPipelineDetails from 'shared_queries/pipelines/get_pipeline_details.query.graphql';
-import { DOWNSTREAM, GRAPHQL, UPSTREAM } from '~/pipelines/components/graph/constants';
+import {
+  DOWNSTREAM,
+  GRAPHQL,
+  UPSTREAM,
+  LAYER_VIEW,
+  STAGE_VIEW,
+} from '~/pipelines/components/graph/constants';
 import PipelineGraph from '~/pipelines/components/graph/graph_component.vue';
 import LinkedPipeline from '~/pipelines/components/graph/linked_pipeline.vue';
 import LinkedPipelinesColumn from '~/pipelines/components/graph/linked_pipelines_column.vue';
+import * as parsingUtils from '~/pipelines/components/parsing_utils';
 import { LOAD_FAILURE } from '~/pipelines/constants';
 import {
   mockPipelineResponse,
@@ -20,6 +27,7 @@ describe('Linked Pipelines Column', () => {
     columnTitle: 'Downstream',
     linkedPipelines: processedPipeline.downstream,
     type: DOWNSTREAM,
+    viewType: STAGE_VIEW,
     configPaths: {
       metricsPath: '',
       graphqlResourceEtag: 'this/is/a/path',
@@ -67,7 +75,7 @@ describe('Linked Pipelines Column', () => {
 
   describe('it renders correctly', () => {
     beforeEach(() => {
-      createComponent();
+      createComponentWithApollo();
     });
 
     it('renders the pipeline title', () => {
@@ -91,6 +99,27 @@ describe('Linked Pipelines Column', () => {
       await wrapper.vm.$nextTick();
     };
 
+    describe('layer type rendering', () => {
+      let layersFn;
+
+      beforeEach(() => {
+        layersFn = jest.spyOn(parsingUtils, 'listByLayers');
+        createComponentWithApollo({ mountFn: mount });
+      });
+
+      it('calls listByLayers only once no matter how many times view is switched', async () => {
+        expect(layersFn).not.toHaveBeenCalled();
+        await clickExpandButtonAndAwaitTimers();
+        await wrapper.setProps({ viewType: LAYER_VIEW });
+        await wrapper.vm.$nextTick();
+        expect(layersFn).toHaveBeenCalledTimes(1);
+        await wrapper.setProps({ viewType: STAGE_VIEW });
+        await wrapper.setProps({ viewType: LAYER_VIEW });
+        await wrapper.setProps({ viewType: STAGE_VIEW });
+        expect(layersFn).toHaveBeenCalledTimes(1);
+      });
+    });
+
     describe('downstream', () => {
       describe('when successful', () => {
         beforeEach(() => {

spec/frontend/pipelines/graph/mock_data.js

@@ -434,21 +434,7 @@ export const mockPipelineResponse = {
                           },
                           needs: {
                             __typename: 'CiBuildNeedConnection',
-                            nodes: [
-                              {
-                                __typename: 'CiBuildNeed',
-                                name: 'build_c',
-                              },
-                              {
-                                __typename: 'CiBuildNeed',
-                                name: 'build_b',
-                              },
-                              {
-                                __typename: 'CiBuildNeed',
-                                name:
-                                  'build_a_nlfjkdnlvskfnksvjknlfdjvlvnjdkjdf_nvjkenjkrlngjeknjkl',
-                              },
-                            ],
+                            nodes: [],
                           },
                         },
                       ],

spec/frontend/pipelines/unwrapping_utils_spec.js

@@ -96,11 +96,11 @@ const completeMock = [
 describe('Shared pipeline unwrapping utils', () => {
   describe('unwrapGroups', () => {
     it('takes stages without nodes and returns the unwrapped groups', () => {
-      expect(unwrapGroups(stagesAndGroups)[0].groups).toEqual(groupsArray);
+      expect(unwrapGroups(stagesAndGroups)[0].node.groups).toEqual(groupsArray);
     });
 
     it('keeps other stage properties intact', () => {
-      expect(unwrapGroups(stagesAndGroups)[0]).toMatchObject(basicStageInfo);
+      expect(unwrapGroups(stagesAndGroups)[0].node).toMatchObject(basicStageInfo);
     });
   });