Add data parsing utils and specs

Adds util file and spec file, plus
changes to the component

app/assets/javascripts/pipelines/components/dag/utils.js

+import * as d3 from 'd3';
+import { sankey, sankeyLeft } from 'd3-sankey';
+import { uniqWith, isEqual } from 'lodash';
+
+/**
+  The following functions are the main engine in transforming the data as
+  received from the endpoint into the format the d3 graph expects.
+
+  Input is of the form:
+  [stages]
+    stages: {name, groups}
+      groups: [{ name, size, jobs }]
+        name is a group name; in the case that the group has one job, it is
+          also the job name
+        size is the number of parallel jobs
+        jobs: [{ name, needs}]
+          job name is either the same as the group name or group x/y
+
+  Output is of the form:
+  { nodes: [node], links: [link] }
+    node: { name, category }, + unused info passed through
+    link: { source, target, value }, with source & target being node names
+      and value being a constant
+
+  We create nodes, create links, and then dedupe the links, so that in the case where
+  job 4 depends on job 1 and job 2, and job 2 depends on job 1, we show only a single link
+  from job 1 to job 2 then another from job 2 to job 4.
+
+  CREATE NODES
+  stage.name -> node.category
+  stage.group.name -> node.name (this is the group name if there are parallel jobs)
+  stage.group.jobs -> node.jobs
+  stage.group.size -> node.size
+
+  CREATE LINKS
+  stages.groups.name -> target
+  stages.groups.needs.each -> source (source is the name of the group, not the parallel job)
+  10 -> value (constant)
+**/
+
+export const createNodes = data => {
+  return data
+    .map(({ groups }, idx, stages) => {
+      return groups.map(group => {
+        return { ...group, category: stages[idx].name };
+      });
+    })
+    .flat();
+};
+
+export const createNodeDict = nodes => {
+  return nodes.reduce((acc, node) => {
+    const newNode = {
+      ...node,
+      needs: node.jobs.map(job => job.needs || []).flat(),
+    };
+
+    if (node.size > 1) {
+      node.jobs.forEach(job => {
+        acc[job.name] = newNode;
+      });
+    }
+
+    acc[node.name] = newNode;
+    return acc;
+  }, {});
+};
+
+export const createNodesStructure = data => {
+  const nodes = createNodes(data);
+  const nodeDict = createNodeDict(nodes);
+
+  return { nodes, nodeDict };
+};
+
+export const makeLinksFromNodes = (nodes, nodeDict) => {
+  const constantLinkValue = 10; // all links are the same weight
+  return nodes
+    .map(group => {
+      return group.jobs.map(job => {
+        if (!job.needs) {
+          return [];
+        }
+
+        return job.needs.map(needed => {
+          return {
+            source: nodeDict[needed]?.name,
+            target: group.name,
+            value: constantLinkValue,
+          };
+        });
+      });
+    })
+    .flat(2);
+};
+
+export const getAllAncestors = (nodes, nodeDict) => {
+  const needs = nodes
+    .map(node => {
+      return nodeDict[node].needs || '';
+    })
+    .flat()
+    .filter(Boolean);
+
+  if (needs.length) {
+    return [...needs, ...getAllAncestors(needs, nodeDict)];
+  }
+
+  return [];
+};
+
+export const filterByAncestors = (links, nodeDict) =>
+  links.filter(link => {
+    /*
+
+  for every link, check out it's target
+  for every target, get the target node's needs
+  then drop the current link source from that list
+
+  call a function to get all ancestors, recursively
+  is the current link's source in the list of all parents?
+  then we drop this link
+
+  */
+    const targetNode = link.target;
+    const targetNodeNeeds = nodeDict[targetNode].needs;
+    const targetNodeNeedsMinusSource = targetNodeNeeds.filter(need => need !== link.source);
+
+    const allAncestors = getAllAncestors(targetNodeNeedsMinusSource, nodeDict);
+    return !allAncestors.includes(link.source);
+  });
+
+export const parseData = data => {
+  const { nodes, nodeDict } = createNodesStructure(data);
+  const allLinks = makeLinksFromNodes(nodes, nodeDict);
+  const filteredLinks = filterByAncestors(allLinks, nodeDict);
+  const links = uniqWith(filteredLinks, isEqual);
+
+  return { nodes, links };
+};
+
+/**
+  createSankey calls the d3 layout to generate the relationships and positioning
+  values for the nodes and links in the graph.
+**/
+
+export const createSankey = ({ width, height, nodeWidth, nodePadding, paddingForLabels }) => {
+  const sankeyGenerator = sankey()
+    .nodeId(d => d.name)
+    .nodeAlign(sankeyLeft)
+    .nodeWidth(nodeWidth)
+    .nodePadding(nodePadding)
+    .extent([
+      [paddingForLabels, paddingForLabels],
+      [width - paddingForLabels, height - paddingForLabels],
+    ]);
+  return ({ nodes, links }) =>
+    sankeyGenerator({
+      nodes: nodes.map(d => ({ ...d })),
+      links: links.map(d => ({ ...d })),
+    });
+};
+
+/**
+  The number of nodes in the most populous generation drives the height of the graph.
+**/
+
+export const getMaxNodes = nodes => {
+  const counts = nodes.reduce((acc, currentNode) => {
+    if (!acc[currentNode.layer]) {
+      acc[currentNode.layer] = 0;
+    }
+
+    acc[currentNode.layer] += 1;
+
+    return acc;
+  }, []);
+
+  return Math.max(...counts);
+};
+
+/**
+  Because we cannot know if a node is part of a relationship until after we
+  generate the links with createSankey, this function is used after the first call
+  to find nodes that have no relations.
+**/
+
+export const removeOrphanNodes = sankeyfiedNodes => {
+  return sankeyfiedNodes.filter(node => node.sourceLinks.length || node.targetLinks.length);
+};

package.json

@@ -71,6 +71,7 @@
     "core-js": "^3.6.4",
     "cropper": "^2.3.0",
     "css-loader": "^2.1.1",
+    "d3-sankey": "^0.12.3",
     "d3-scale": "^2.2.2",
     "d3-selection": "^1.2.0",
     "dateformat": "^3.0.3",

spec/frontend/pipelines/components/dag/mock-data.js

+/**
+  It is important that the simple base include parallel jobs
+  as well as non-parallel jobs with spaces in the name to prevent
+  us relying on spaces as an indicator.
+**/
+export default {
+  stages: [
+    {
+      name: 'test',
+      groups: [
+        {
+          name: 'jest',
+          size: 2,
+          jobs: [
+            { name: 'jest 1/2', needs: ['frontend fixtures'] },
+            { name: 'jest 2/2' }
+          ]
+        },
+        {
+          name: 'rspec',
+          size: 1,
+          jobs: [
+            { name: 'rspec', needs: ['frontend fixtures'] },
+          ]
+        },
+      ]
+    },
+    {
+      name: 'fixtures',
+      groups: [
+        {
+          name: 'frontend fixtures',
+          size: 1,
+          jobs: [
+            { name: 'frontend fixtures' }
+          ]
+        }
+      ]
+    },
+    {
+      name: 'un-needed',
+      groups: [
+        {
+          name: 'un-needed',
+          size: 1,
+          jobs: [
+            { name: 'un-needed' }
+          ]
+        }
+      ]
+    }
+  ]
+}

spec/frontend/pipelines/components/dag/utils_spec.js

+import {
+  createNodesStructure,
+  makeLinksFromNodes,
+  filterByAncestors,
+  parseData,
+  createSankey,
+  removeOrphanNodes,
+  getMaxNodes,
+} from '~/pipelines/components/dag/utils';
+
+import mockGraphData from './mock-data.js';
+
+describe('DAG visualization parsing utilities', () => {
+  const { nodes, nodeDict } = createNodesStructure(mockGraphData.stages);
+  const unfilteredLinks = makeLinksFromNodes(nodes, nodeDict);
+  const parsed = parseData(mockGraphData.stages);
+
+  const layoutSettings = {
+    width: 200,
+    height: 200,
+    nodeWidth: 10,
+    nodePadding: 20,
+    paddingForLabels: 100,
+  };
+
+  const sankeyLayout = createSankey(layoutSettings)(parsed);
+
+  describe('createNodesStructure', () => {
+    const parallelGroupName = 'jest';
+    const parallelJobName = 'jest 1/2';
+    const singleJobName = 'frontend fixtures';
+
+    const { name, jobs, size } = mockGraphData.stages[0].groups[0];
+
+    it('returns the expected node structure', () => {
+      expect(nodes[0]).toHaveProperty('category', mockGraphData.stages[0].name);
+      expect(nodes[0]).toHaveProperty('name', name);
+      expect(nodes[0]).toHaveProperty('jobs', jobs);
+      expect(nodes[0]).toHaveProperty('size', size);
+    });
+
+    it('adds needs to top level of nodeDict entries', () => {
+      expect(nodeDict[parallelGroupName]).toHaveProperty('needs');
+      expect(nodeDict[parallelJobName]).toHaveProperty('needs');
+      expect(nodeDict[singleJobName]).toHaveProperty('needs');
+    });
+
+    it('makes entries in nodeDict for jobs and parallel jobs', () => {
+      const nodeNames = Object.keys(nodeDict);
+
+      expect(nodeNames.includes(parallelGroupName)).toBe(true);
+      expect(nodeNames.includes(parallelJobName)).toBe(true);
+      expect(nodeNames.includes(singleJobName)).toBe(true);
+    });
+  });
+
+  describe('makeLinksFromNodes', () => {
+    it('returns the expected link structure', () => {
+      expect(unfilteredLinks[0]).toHaveProperty('source', 'frontend fixtures');
+      expect(unfilteredLinks[0]).toHaveProperty('target', 'jest');
+      expect(unfilteredLinks[0]).toHaveProperty('value', 10);
+    });
+  });
+
+  describe('filterByAncestors', () => {
+    const allLinks = [
+      { source: 'job1', target: 'job4' },
+      { source: 'job1', target: 'job2' },
+      { source: 'job2', target: 'job4' },
+    ];
+
+    const dedupedLinks = [{ source: 'job1', target: 'job2' }, { source: 'job2', target: 'job4' }];
+
+    const nodeDict = {
+      job1: {
+        name: 'job1',
+      },
+      job2: {
+        name: 'job2',
+        needs: ['job1'],
+      },
+      job4: {
+        name: 'job4',
+        needs: ['job1', 'job2'],
+        category: 'build',
+      },
+    };
+
+    it('dedupes links', () => {
+      expect(filterByAncestors(allLinks, nodeDict)).toMatchObject(dedupedLinks);
+    });
+  });
+
+  describe('parseData parent function', () => {
+    it('returns an object containing a list of nodes and links', () => {
+      const parsed = parseData(mockGraphData.stages);
+
+      // an array of nodes exist and the values are defined
+      expect(parsed).toHaveProperty('nodes');
+      expect(Array.isArray(parsed.nodes)).toBe(true);
+      expect(parsed.nodes.filter(Boolean)).not.toHaveLength(0);
+
+      // an array of links exist and the values are defined
+      expect(parsed).toHaveProperty('links');
+      expect(Array.isArray(parsed.links)).toBe(true);
+      expect(parsed.links.filter(Boolean)).not.toHaveLength(0);
+    });
+  });
+
+  describe('createSankey', () => {
+    it('returns a nodes data structure with expected d3-added properties', () => {
+      expect(sankeyLayout.nodes[0]).toHaveProperty('sourceLinks');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('targetLinks');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('depth');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('layer');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('x0');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('x1');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('y0');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('y1');
+    });
+
+    it('returns a links data structure with expected d3-added properties', () => {
+      expect(sankeyLayout.links[0]).toHaveProperty('source');
+      expect(sankeyLayout.links[0]).toHaveProperty('target');
+      expect(sankeyLayout.links[0]).toHaveProperty('width');
+      expect(sankeyLayout.links[0]).toHaveProperty('y0');
+      expect(sankeyLayout.links[0]).toHaveProperty('y1');
+    });
+
+    describe('data structure integrity', () => {
+      const newObject = { name: 'bad-actor' };
+
+      beforeEach(() => {
+        sankeyLayout.nodes.unshift(newObject);
+      });
+
+      it('sankey does not propagate changes back to the original', () => {
+        expect(sankeyLayout.nodes[0]).toBe(newObject);
+        expect(parsed.nodes[0]).not.toBe(newObject);
+      });
+
+      afterEach(() => {
+        sankeyLayout.nodes.shift();
+      });
+    });
+  });
+
+  describe('removeOrphanNodes', () => {
+    it('removes sankey nodes that have no needs and are not needed', () => {
+      const cleanedNodes = removeOrphanNodes(sankeyLayout.nodes);
+      expect(cleanedNodes).toHaveLength(sankeyLayout.nodes.length - 1);
+    });
+  });
+
+  describe('getMaxNodes', () => {
+    it('returns the number of nodes in the most populous generation', () => {
+      const layerNodes = [
+        { layer: 0 },
+        { layer: 0 },
+        { layer: 1 },
+        { layer: 1 },
+        { layer: 0 },
+        { layer: 3 },
+        { layer: 2 },
+        { layer: 4 },
+        { layer: 1 },
+        { layer: 3 },
+        { layer: 4 },
+      ];
+      expect(getMaxNodes(layerNodes)).toBe(3);
+    });
+  });
+});

yarn.lock

@@ -3360,6 +3360,11 @@ d3-array@1, d3-array@^1.1.1, d3-array@^1.2.0:
   resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.1.tgz#d1ca33de2f6ac31efadb8e050a021d7e2396d5dc"
   integrity sha512-CyINJQ0SOUHojDdFDH4JEM0552vCR1utGyLHegJHyYH0JyCpSeTPxi4OBqHMA2jJZq4NH782LtaJWBImqI/HBw==
 
+"d3-array@1 - 2":
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-2.4.0.tgz#87f8b9ad11088769c82b5ea846bcb1cc9393f242"
+  integrity sha512-KQ41bAF2BMakf/HdKT865ALd4cgND6VcIztVQZUTt0+BH3RWy6ZYnHghVXf6NFjt2ritLr8H1T8LreAAlfiNcw==
+
 d3-axis@1:
   version "1.0.8"
   resolved "https://registry.yarnpkg.com/d3-axis/-/d3-axis-1.0.8.tgz#31a705a0b535e65759de14173a31933137f18efa"
@@ -3489,6 +3494,14 @@ d3-random@1:
   resolved "https://registry.yarnpkg.com/d3-random/-/d3-random-1.1.0.tgz#6642e506c6fa3a648595d2b2469788a8d12529d3"
   integrity sha1-ZkLlBsb6OmSFldKyRpeIqNElKdM=
 
+d3-sankey@^0.12.3:
+  version "0.12.3"
+  resolved "https://registry.yarnpkg.com/d3-sankey/-/d3-sankey-0.12.3.tgz#b3c268627bd72e5d80336e8de6acbfec9d15d01d"
+  integrity sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==
+  dependencies:
+    d3-array "1 - 2"
+    d3-shape "^1.2.0"
+
 d3-scale-chromatic@1:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/d3-scale-chromatic/-/d3-scale-chromatic-1.3.3.tgz#dad4366f0edcb288f490128979c3c793583ed3c0"
@@ -3521,6 +3534,13 @@ d3-shape@1:
   dependencies:
     d3-path "1"
 
+d3-shape@^1.2.0:
+  version "1.3.7"
+  resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-1.3.7.tgz#df63801be07bc986bc54f63789b4fe502992b5d7"
+  integrity sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==
+  dependencies:
+    d3-path "1"
+
 d3-time-format@2:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-2.1.1.tgz#85b7cdfbc9ffca187f14d3c456ffda268081bb31"