Merge branch '338320-pseudonymization-of-urls-frontend' into 'master'

[RUN ALL RSPEC] [RUN AS-IF-FOSS] Pseudonymization of URLs - Frontend

See merge request gitlab-org/gitlab!68618

app/assets/javascripts/tracking/constants.js

@@ -24,3 +24,7 @@ export const LOAD_ACTION_ATTR_SELECTOR = '[data-track-action="render"]';
 
 export const DEPRECATED_EVENT_ATTR_SELECTOR = '[data-track-event]';
 export const DEPRECATED_LOAD_EVENT_ATTR_SELECTOR = '[data-track-event="render"]';
+
+export const URLS_CACHE_STORAGE_KEY = 'gl-snowplow-pseudonymized-urls';
+
+export const REFERRER_TTL = 24 * 60 * 60 * 1000;

app/assets/javascripts/tracking/index.js

@@ -39,6 +39,9 @@ export function initDefaultTrackers() {
 
   const opts = { ...DEFAULT_SNOWPLOW_OPTIONS, ...window.snowplowOptions };
 
+  // must be before initializing the trackers
+  Tracking.setAnonymousUrls();
+
   window.snowplow('enableActivityTracking', 30, 30);
   // must be after enableActivityTracking
   const standardContext = getStandardContext();

app/assets/javascripts/tracking/tracking.js

 import { LOAD_ACTION_ATTR_SELECTOR, DEPRECATED_LOAD_EVENT_ATTR_SELECTOR } from './constants';
 import { dispatchSnowplowEvent } from './dispatch_snowplow_event';
 import getStandardContext from './get_standard_context';
-import { getEventHandlers, createEventPayload, renameKey, addExperimentContext } from './utils';
+import {
+  getEventHandlers,
+  createEventPayload,
+  renameKey,
+  addExperimentContext,
+  getReferrersCache,
+  addReferrersCacheEntry,
+} from './utils';
 
 export default class Tracking {
   static queuedEvents = [];
@@ -158,6 +165,37 @@ export default class Tracking {
     }
   }
 
+  /**
+   * Replaces the URL and referrer for the default web context
+   * if the replacements are available.
+   *
+   * @returns {undefined}
+   */
+  static setAnonymousUrls() {
+    const { snowplowPseudonymizedPageUrl: pageUrl } = window.gl;
+
+    if (!pageUrl) {
+      return;
+    }
+
+    const referrers = getReferrersCache();
+    const pageLinks = Object.seal({ url: '', referrer: '', originalUrl: window.location.href });
+
+    pageLinks.url = `${pageUrl}${window.location.hash}`;
+    window.snowplow('setCustomUrl', pageLinks.url);
+
+    if (document.referrer) {
+      const node = referrers.find((links) => links.originalUrl === document.referrer);
+
+      if (node) {
+        pageLinks.referrer = node.url;
+        window.snowplow('setReferrerUrl', pageLinks.referrer);
+      }
+    }
+
+    addReferrersCacheEntry(referrers, pageLinks);
+  }
+
   /**
    * Returns an implementation of this class in the form of
    * a Vue mixin.

app/assets/javascripts/tracking/utils.js

@@ -6,6 +6,8 @@ import {
   LOAD_ACTION_ATTR_SELECTOR,
   DEPRECATED_EVENT_ATTR_SELECTOR,
   DEPRECATED_LOAD_EVENT_ATTR_SELECTOR,
+  URLS_CACHE_STORAGE_KEY,
+  REFERRER_TTL,
 } from './constants';
 
 export const addExperimentContext = (opts) => {
@@ -100,3 +102,25 @@ export const renameKey = (o, oldKey, newKey) => {
 
   return ret;
 };
+
+export const filterOldReferrersCacheEntries = (cache) => {
+  const now = Date.now();
+
+  return cache.filter((entry) => entry.timestamp && entry.timestamp > now - REFERRER_TTL);
+};
+
+export const getReferrersCache = () => {
+  try {
+    const referrers = JSON.parse(window.localStorage.getItem(URLS_CACHE_STORAGE_KEY) || '[]');
+
+    return filterOldReferrersCacheEntries(referrers);
+  } catch {
+    return [];
+  }
+};
+
+export const addReferrersCacheEntry = (cache, entry) => {
+  const referrers = JSON.stringify([{ ...entry, timestamp: Date.now() }, ...cache]);
+
+  window.localStorage.setItem(URLS_CACHE_STORAGE_KEY, referrers);
+};

app/helpers/routing/pseudonymization_helper.rb

@@ -6,6 +6,8 @@ module Routing
       return unless Feature.enabled?(:mask_page_urls, type: :ops)
 
       mask_params(Rails.application.routes.recognize_path(request.original_fullpath))
+    rescue ActionController::RoutingError, URI::InvalidURIError
+      nil
     end
 
     private
@@ -19,31 +21,37 @@ module Routing
     end
 
     def url_without_namespace_type(request_params)
-      masked_url = "#{request.protocol}#{request.host_with_port}/"
+      masked_url = "#{request.protocol}#{request.host_with_port}"
 
       masked_url += case request_params[:controller]
                     when 'groups'
-                      "namespace:#{group.id}/"
+                      "/namespace:#{group.id}"
                     when 'projects'
-                      "namespace:#{project.namespace.id}/project:#{project.id}/"
+                      "/namespace:#{project.namespace.id}/project:#{project.id}"
                     when 'root'
                       ''
+                    else
+                      "#{request.path}"
                     end
 
+      masked_url += request.query_string.present? ? "?#{request.query_string}" : ''
+
       masked_url
     end
 
     def url_with_namespace_type(request_params, namespace_type)
-      masked_url = "#{request.protocol}#{request.host_with_port}/"
+      masked_url = "#{request.protocol}#{request.host_with_port}"
 
       if request_params.has_key?(:project_id)
-        masked_url += "namespace:#{project.namespace.id}/project:#{project.id}/-/#{namespace_type}/"
+        masked_url += "/namespace:#{project.namespace.id}/project:#{project.id}/-/#{namespace_type}"
       end
 
       if request_params.has_key?(:id)
-        masked_url += namespace_type == 'blob' ? ':repository_path' : request_params[:id]
+        masked_url += namespace_type == 'blob' ? '/:repository_path' : "/#{request_params[:id]}"
       end
 
+      masked_url += request.query_string.present? ? "?#{request.query_string}" : ''
+
       masked_url
     end
   end

app/views/layouts/_snowplow.html.haml

@@ -11,3 +11,4 @@
 
     gl = window.gl || {};
     gl.snowplowStandardContext = #{Gitlab::Tracking::StandardContext.new.to_context.to_json.to_json}
+    gl.snowplowPseudonymizedPageUrl = #{masked_page_url.to_json};

spec/frontend/tracking_spec.js

 import { setHTMLFixture } from 'helpers/fixtures';
+import { TEST_HOST } from 'helpers/test_constants';
 import { TRACKING_CONTEXT_SCHEMA } from '~/experimentation/constants';
 import { getExperimentData, getAllExperimentContexts } from '~/experimentation/utils';
 import Tracking, { initUserTracking, initDefaultTrackers } from '~/tracking';
+import { REFERRER_TTL, URLS_CACHE_STORAGE_KEY } from '~/tracking/constants';
 import getStandardContext from '~/tracking/get_standard_context';
 
 jest.mock('~/experimentation/utils', () => ({
@@ -15,9 +17,11 @@ describe('Tracking', () => {
   let bindDocumentSpy;
   let trackLoadEventsSpy;
   let enableFormTracking;
+  let setAnonymousUrlsSpy;
 
   beforeAll(() => {
     window.gl = window.gl || {};
+    window.gl.snowplowUrls = {};
     window.gl.snowplowStandardContext = {
       schema: 'iglu:com.gitlab/gitlab_standard',
       data: {
@@ -74,6 +78,7 @@ describe('Tracking', () => {
       enableFormTracking = jest
         .spyOn(Tracking, 'enableFormTracking')
         .mockImplementation(() => null);
+      setAnonymousUrlsSpy = jest.spyOn(Tracking, 'setAnonymousUrls').mockImplementation(() => null);
     });
 
     it('should activate features based on what has been enabled', () => {
@@ -105,6 +110,11 @@ describe('Tracking', () => {
       expect(trackLoadEventsSpy).toHaveBeenCalled();
     });
 
+    it('calls the anonymized URLs method', () => {
+      initDefaultTrackers();
+      expect(setAnonymousUrlsSpy).toHaveBeenCalled();
+    });
+
     describe('when there are experiment contexts', () => {
       const experimentContexts = [
         {
@@ -295,6 +305,110 @@ describe('Tracking', () => {
     });
   });
 
+  describe('.setAnonymousUrls', () => {
+    afterEach(() => {
+      window.gl.snowplowPseudonymizedPageUrl = '';
+      localStorage.removeItem(URLS_CACHE_STORAGE_KEY);
+    });
+
+    it('does nothing if URLs are not provided', () => {
+      Tracking.setAnonymousUrls();
+
+      expect(snowplowSpy).not.toHaveBeenCalled();
+      expect(localStorage.getItem(URLS_CACHE_STORAGE_KEY)).toBe(null);
+    });
+
+    it('sets the page URL when provided and populates the cache', () => {
+      window.gl.snowplowPseudonymizedPageUrl = TEST_HOST;
+
+      Tracking.setAnonymousUrls();
+
+      expect(snowplowSpy).toHaveBeenCalledWith('setCustomUrl', TEST_HOST);
+      expect(JSON.parse(localStorage.getItem(URLS_CACHE_STORAGE_KEY))[0]).toStrictEqual({
+        url: TEST_HOST,
+        referrer: '',
+        originalUrl: window.location.href,
+        timestamp: Date.now(),
+      });
+    });
+
+    it('appends the hash/fragment to the pseudonymized URL', () => {
+      const hash = 'first-heading';
+      window.gl.snowplowPseudonymizedPageUrl = TEST_HOST;
+      window.location.hash = hash;
+
+      Tracking.setAnonymousUrls();
+
+      expect(snowplowSpy).toHaveBeenCalledWith('setCustomUrl', `${TEST_HOST}#${hash}`);
+    });
+
+    it('does not set the referrer URL by default', () => {
+      window.gl.snowplowPseudonymizedPageUrl = TEST_HOST;
+
+      Tracking.setAnonymousUrls();
+
+      expect(snowplowSpy).not.toHaveBeenCalledWith('setReferrerUrl', expect.any(String));
+    });
+
+    describe('with referrers cache', () => {
+      const testUrl = '/namespace:1/project:2/-/merge_requests/5';
+      const testOriginalUrl = '/my-namespace/my-project/-/merge_requests/';
+      const setUrlsCache = (data) =>
+        localStorage.setItem(URLS_CACHE_STORAGE_KEY, JSON.stringify(data));
+
+      beforeEach(() => {
+        window.gl.snowplowPseudonymizedPageUrl = TEST_HOST;
+        Object.defineProperty(document, 'referrer', { value: '', configurable: true });
+      });
+
+      it('does nothing if a referrer can not be found', () => {
+        setUrlsCache([
+          {
+            url: testUrl,
+            originalUrl: TEST_HOST,
+            timestamp: Date.now(),
+          },
+        ]);
+
+        Tracking.setAnonymousUrls();
+
+        expect(snowplowSpy).not.toHaveBeenCalledWith('setReferrerUrl', expect.any(String));
+      });
+
+      it('sets referrer URL from the page URL found in cache', () => {
+        Object.defineProperty(document, 'referrer', { value: testOriginalUrl });
+        setUrlsCache([
+          {
+            url: testUrl,
+            originalUrl: testOriginalUrl,
+            timestamp: Date.now(),
+          },
+        ]);
+
+        Tracking.setAnonymousUrls();
+
+        expect(snowplowSpy).toHaveBeenCalledWith('setReferrerUrl', testUrl);
+      });
+
+      it('ignores and removes old entries from the cache', () => {
+        const oldTimestamp = Date.now() - (REFERRER_TTL + 1);
+        Object.defineProperty(document, 'referrer', { value: testOriginalUrl });
+        setUrlsCache([
+          {
+            url: testUrl,
+            originalUrl: testOriginalUrl,
+            timestamp: oldTimestamp,
+          },
+        ]);
+
+        Tracking.setAnonymousUrls();
+
+        expect(snowplowSpy).not.toHaveBeenCalledWith('setReferrerUrl', testUrl);
+        expect(localStorage.getItem(URLS_CACHE_STORAGE_KEY)).not.toContain(oldTimestamp);
+      });
+    });
+  });
+
   describe.each`
     term
     ${'event'}

spec/helpers/routing/pseudonymization_helper_spec.rb

@@ -56,7 +56,7 @@ RSpec.describe ::Routing::PseudonymizationHelper do
     end
 
     context 'with controller for groups with subgroups and project' do
-      let(:masked_url) { "http://test.host/namespace:#{subgroup.id}/project:#{project.id}/"}
+      let(:masked_url) { "http://test.host/namespace:#{subgroup.id}/project:#{project.id}"}
 
       before do
         allow(helper).to receive(:group).and_return(subgroup)
@@ -73,7 +73,7 @@ RSpec.describe ::Routing::PseudonymizationHelper do
     end
 
     context 'with controller for groups and subgroups' do
-      let(:masked_url) { "http://test.host/namespace:#{subgroup.id}/"}
+      let(:masked_url) { "http://test.host/namespace:#{subgroup.id}"}
 
       before do
         allow(helper).to receive(:group).and_return(subgroup)
@@ -102,10 +102,25 @@ RSpec.describe ::Routing::PseudonymizationHelper do
 
       it_behaves_like 'masked url'
     end
+
+    context 'with non identifiable controller' do
+      let(:masked_url) { "http://test.host/dashboard/issues?assignee_username=root" }
+
+      before do
+        controller.request.path = '/dashboard/issues'
+        controller.request.query_string = 'assignee_username=root'
+        allow(Rails.application.routes).to receive(:recognize_path).and_return({
+          controller: 'dashboard',
+          action: 'issues'
+        })
+      end
+
+      it_behaves_like 'masked url'
+    end
   end
 
   describe 'when url has no params to mask' do
-    let(:root_url) { 'http://test.host/' }
+    let(:root_url) { 'http://test.host' }
 
     context 'returns root url' do
       it 'masked_page_url' do