Add LRU-like behaviour to incremental compilation

This adds least-recently-used-cache-like behaviour to the incremental
webpack compiler. The `DEV_SERVER_INCREMENTAL_TTL` environment variable
now determines the number of days that page bundles are considered
"recent", and should be eagerly compiled. This number represents the
trade-off between lazy/eager compilation versus low/high memory
consumption of the webpack development server. A higher number means
fewer pages needing to be compiled on demand, at the cost of higher
memory consumption. A lower number means lower memory consumption, at
the cost of more pages being compiled on demand. A value of `0` means
that all pages in your history, regardless of how long ago you visited
them, are eagerly compiled.

This also makes the compiler record a history of visited pages even when
disabled, so that if and when it _is_ enabled, that history can still be
used to inform the LRU cache.

The history-recording function is explicitly disabled in the case that
webpack is running in CI.

Part of https://gitlab.com/gitlab-org/gitlab/-/issues/300412.

config/helpers/incremental_webpack_compiler.js

-/* eslint-disable max-classes-per-file, no-underscore-dangle */
-const fs = require('fs');
-const path = require('path');
-
-const log = (msg, ...rest) => console.log(`IncrementalWebpackCompiler: ${msg}`, ...rest);
-
-// If we force a recompile immediately, the page reload doesn't seem to work.
-// Five seconds seem to work fine and the user can read the message
-const TIMEOUT = 5000;
-
-/* eslint-disable class-methods-use-this */
-class NoopCompiler {
-  constructor() {
-    this.enabled = false;
-  }
-
-  filterEntryPoints(entryPoints) {
-    return entryPoints;
-  }
-
-  logStatus() {}
-
-  setupMiddleware() {}
-}
-/* eslint-enable class-methods-use-this */
-
-class IncrementalWebpackCompiler {
-  constructor(historyFilePath) {
-    this.enabled = true;
-    this.history = {};
-    this.compiledEntryPoints = new Set([
-      // Login page
-      'pages.sessions.new',
-      // Explore page
-      'pages.root',
-    ]);
-    this.historyFilePath = historyFilePath;
-    this._loadFromHistory();
-  }
-
-  filterEntryPoints(entrypoints) {
-    return Object.fromEntries(
-      Object.entries(entrypoints).map(([key, val]) => {
-        if (this.compiledEntryPoints.has(key)) {
-          return [key, val];
-        }
-        return [key, ['./webpack_non_compiled_placeholder.js']];
-      }),
-    );
-  }
-
-  logStatus(totalCount) {
-    const current = this.compiledEntryPoints.size;
-    log(`Currently compiling route entrypoints: ${current} of ${totalCount}`);
-  }
-
-  setupMiddleware(app, server) {
-    app.use((req, res, next) => {
-      const fileName = path.basename(req.url);
-
-      /**
-       * We are only interested in files that have a name like `pages.foo.bar.chunk.js`
-       * because those are the ones corresponding to our entry points.
-       *
-       * This filters out hot update files that are for example named "pages.foo.bar.[hash].hot-update.js"
-       */
-      if (fileName.startsWith('pages.') && fileName.endsWith('.chunk.js')) {
-        const chunk = fileName.replace(/\.chunk\.js$/, '');
-
-        this._addToHistory(chunk);
-
-        if (!this.compiledEntryPoints.has(chunk)) {
-          log(`First time we are seeing ${chunk}. Adding to compilation.`);
-
-          this.compiledEntryPoints.add(chunk);
-
-          setTimeout(() => {
-            server.middleware.invalidate(() => {
-              if (server.sockets) {
-                server.sockWrite(server.sockets, 'content-changed');
-              }
-            });
-          }, TIMEOUT);
-        }
-      }
-
-      next();
-    });
-  }
-
-  // private methods
-
-  _addToHistory(chunk) {
-    if (!this.history[chunk]) {
-      this.history[chunk] = { lastVisit: null, count: 0 };
-    }
-    this.history[chunk].lastVisit = Date.now();
-    this.history[chunk].count += 1;
-
-    try {
-      fs.writeFileSync(this.historyFilePath, JSON.stringify(this.history), 'utf8');
-    } catch (e) {
-      log('Warning – Could not write to history', e.message);
-    }
-  }
-
-  _loadFromHistory() {
-    try {
-      this.history = JSON.parse(fs.readFileSync(this.historyFilePath, 'utf8'));
-      const entryPoints = Object.keys(this.history);
-      log(`Successfully loaded history containing ${entryPoints.length} entry points`);
-      /*
-      TODO: Let's ask a few folks to give us their history file after a milestone of usage
-            Then we can make smarter decisions on when to throw out rather than rendering everything
-            Something like top 20/30/40 entries visited in the last 7/10/15 days might be sufficient
-       */
-      this.compiledEntryPoints = new Set([...this.compiledEntryPoints, ...entryPoints]);
-    } catch (e) {
-      log(`No history found...`);
-    }
-  }
-}
-
-module.exports = (enabled, historyFilePath) => {
-  log(`Status – ${enabled ? 'enabled' : 'disabled'}`);
-
-  if (enabled) {
-    return new IncrementalWebpackCompiler(historyFilePath);
-  }
-  return new NoopCompiler();
-};

config/helpers/incremental_webpack_compiler/compiler.js

+/* eslint-disable max-classes-per-file */
+
+const path = require('path');
+const { History, HistoryWithTTL } = require('./history');
+const log = require('./log');
+
+const onRequestEntryPoint = (app, callback) => {
+  app.use((req, res, next) => {
+    const fileName = path.basename(req.url);
+
+    /**
+     * We are only interested in files that have a name like `pages.foo.bar.chunk.js`
+     * because those are the ones corresponding to our entry points.
+     *
+     * This filters out hot update files that are for example named "pages.foo.bar.[hash].hot-update.js"
+     */
+    if (fileName.startsWith('pages.') && fileName.endsWith('.chunk.js')) {
+      const entryPoint = fileName.replace(/\.chunk\.js$/, '');
+      callback(entryPoint);
+    }
+
+    next();
+  });
+};
+
+/**
+ * The NoopCompiler does nothing, following the null object pattern.
+ */
+class NoopCompiler {
+  constructor() {
+    this.enabled = false;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  filterEntryPoints(entryPoints) {
+    return entryPoints;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  logStatus() {}
+
+  // eslint-disable-next-line class-methods-use-this
+  setupMiddleware() {}
+}
+
+/**
+ * The HistoryOnlyCompiler only records which entry points have been requested.
+ * This is so that if the user disables incremental compilation, history is
+ * still recorded. If they later enable incremental compilation, that history
+ * can be used.
+ */
+class HistoryOnlyCompiler extends NoopCompiler {
+  constructor(historyFilePath) {
+    super();
+    this.history = new History(historyFilePath);
+  }
+
+  setupMiddleware(app) {
+    onRequestEntryPoint(app, (entryPoint) => {
+      this.history.onRequestEntryPoint(entryPoint);
+    });
+  }
+}
+
+// If we force a recompile immediately, the page reload doesn't seem to work.
+// Five seconds seem to work fine and the user can read the message
+const TIMEOUT = 5000;
+
+/**
+ * The IncrementalWebpackCompiler tracks which entry points have been
+ * requested, and only compiles entry points visited within the last `ttl`
+ * days.
+ */
+class IncrementalWebpackCompiler {
+  constructor(historyFilePath, ttl) {
+    this.enabled = true;
+    this.history = new HistoryWithTTL(historyFilePath, ttl);
+  }
+
+  filterEntryPoints(entrypoints) {
+    return Object.fromEntries(
+      Object.entries(entrypoints).map(([entryPoint, paths]) => {
+        if (this.history.isRecentlyVisited(entryPoint)) {
+          return [entryPoint, paths];
+        }
+        return [entryPoint, ['./webpack_non_compiled_placeholder.js']];
+      }),
+    );
+  }
+
+  logStatus(totalCount) {
+    log(`Currently compiling route entrypoints: ${this.history.size} of ${totalCount}`);
+  }
+
+  setupMiddleware(app, server) {
+    onRequestEntryPoint(app, (entryPoint) => {
+      const wasVisitedRecently = this.history.onRequestEntryPoint(entryPoint);
+      if (!wasVisitedRecently) {
+        log(`Have not visited ${entryPoint} recently. Adding to compilation.`);
+
+        setTimeout(() => {
+          server.middleware.invalidate(() => {
+            if (server.sockets) {
+              server.sockWrite(server.sockets, 'content-changed');
+            }
+          });
+        }, TIMEOUT);
+      }
+    });
+  }
+}
+
+module.exports = {
+  NoopCompiler,
+  HistoryOnlyCompiler,
+  IncrementalWebpackCompiler,
+};

config/helpers/incremental_webpack_compiler/history.js

+/* eslint-disable max-classes-per-file, no-underscore-dangle */
+
+const fs = require('fs');
+const log = require('./log');
+
+/**
+ * The History class is responsible for tracking which entry points have been
+ * requested, and persisting/loading the history to/from disk.
+ */
+class History {
+  constructor(historyFilePath) {
+    this._historyFilePath = historyFilePath;
+    this._history = this._loadHistoryFile();
+  }
+
+  onRequestEntryPoint(entryPoint) {
+    const wasVisitedRecently = this.isRecentlyVisited(entryPoint);
+
+    if (!this._history[entryPoint]) {
+      this._history[entryPoint] = { lastVisit: null, count: 0 };
+    }
+
+    this._history[entryPoint].lastVisit = Date.now();
+    this._history[entryPoint].count += 1;
+
+    this._writeHistoryFile();
+
+    return wasVisitedRecently;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  isRecentlyVisited() {
+    return true;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  get size() {
+    return 0;
+  }
+
+  // Private methods
+
+  _writeHistoryFile() {
+    try {
+      fs.writeFileSync(this._historyFilePath, JSON.stringify(this._history), 'utf8');
+    } catch (e) {
+      log('Warning – Could not write to history', e.message);
+    }
+  }
+
+  _loadHistoryFile() {
+    let history = {};
+
+    try {
+      history = JSON.parse(fs.readFileSync(this._historyFilePath, 'utf8'));
+      const historySize = Object.keys(history).length;
+      log(`Successfully loaded history containing ${historySize} entry points`);
+    } catch (error) {
+      log(`Could not load history: ${error}`);
+    }
+
+    return history;
+  }
+}
+
+const MS_PER_DAY = 1000 * 60 * 60 * 24;
+
+/**
+ * The HistoryWithTTL class adds LRU-like behaviour onto the base History
+ * behaviour. Entry points visited within the last `ttl` days are considered
+ * "recent", and therefore should be eagerly compiled.
+ */
+class HistoryWithTTL extends History {
+  constructor(historyFilePath, ttl) {
+    super(historyFilePath);
+    this._ttl = ttl;
+    this._calculateRecentEntryPoints();
+  }
+
+  onRequestEntryPoint(entryPoint) {
+    const wasVisitedRecently = super.onRequestEntryPoint(entryPoint);
+
+    this._calculateRecentEntryPoints();
+
+    return wasVisitedRecently;
+  }
+
+  isRecentlyVisited(entryPoint) {
+    return this._recentEntryPoints.has(entryPoint);
+  }
+
+  get size() {
+    return this._recentEntryPoints.size;
+  }
+
+  // Private methods
+
+  _calculateRecentEntryPoints() {
+    const oldestVisitAllowed = Date.now() - MS_PER_DAY * this._ttl;
+
+    const recentEntryPoints = Object.entries(this._history).reduce(
+      (acc, [entryPoint, { lastVisit }]) => {
+        if (lastVisit > oldestVisitAllowed) {
+          acc.push(entryPoint);
+        }
+
+        return acc;
+      },
+      [],
+    );
+
+    this._recentEntryPoints = new Set([
+      // Login page
+      'pages.sessions.new',
+      // Explore page
+      'pages.root',
+      ...recentEntryPoints,
+    ]);
+  }
+}
+
+module.exports = {
+  History,
+  HistoryWithTTL,
+};

config/helpers/incremental_webpack_compiler/index.js

+const { NoopCompiler, HistoryOnlyCompiler, IncrementalWebpackCompiler } = require('./compiler');
+const log = require('./log');
+
+module.exports = (recordHistory, enabled, historyFilePath, ttl) => {
+  if (!recordHistory) {
+    log(`Status – disabled`);
+    return new NoopCompiler();
+  }
+
+  if (enabled) {
+    log(`Status – enabled, ttl=${ttl}`);
+    return new IncrementalWebpackCompiler(historyFilePath, ttl);
+  }
+
+  log(`Status – history-only`);
+  return new HistoryOnlyCompiler(historyFilePath);
+};

config/helpers/incremental_webpack_compiler/log.js

+const log = (msg, ...rest) => console.log(`IncrementalWebpackCompiler: ${msg}`, ...rest);
+
+module.exports = log;

config/webpack.config.js

@@ -48,6 +48,8 @@ const INCREMENTAL_COMPILER_ENABLED =
   IS_DEV_SERVER &&
   process.env.DEV_SERVER_INCREMENTAL &&
   process.env.DEV_SERVER_INCREMENTAL !== 'false';
+const INCREMENTAL_COMPILER_TTL = Number(process.env.DEV_SERVER_INCREMENTAL_TTL) || Infinity;
+const INCREMENTAL_COMPILER_RECORD_HISTORY = IS_DEV_SERVER && !process.env.CI;
 const WEBPACK_REPORT = process.env.WEBPACK_REPORT && process.env.WEBPACK_REPORT !== 'false';
 const WEBPACK_MEMORY_TEST =
   process.env.WEBPACK_MEMORY_TEST && process.env.WEBPACK_MEMORY_TEST !== 'false';
@@ -69,8 +71,10 @@ let watchAutoEntries = [];
 const defaultEntries = ['./main'];
 
 const incrementalCompiler = createIncrementalWebpackCompiler(
+  INCREMENTAL_COMPILER_RECORD_HISTORY,
   INCREMENTAL_COMPILER_ENABLED,
   path.join(CACHE_PATH, 'incremental-webpack-compiler-history.json'),
+  INCREMENTAL_COMPILER_TTL,
 );
 
 function generateEntries() {