Merge branch '29642-Replace-Unicode-with-ASCII-Equivalent' into 'master'

Replace Unicode Characters with ASCII Equivalent in New Project Slug

Closes #29642

See merge request gitlab-org/gitlab!39971

app/assets/javascripts/lib/utils/text_utility.js

@@ -275,6 +275,81 @@ export const convertToSentenceCase = string => {
  */
 export const convertToTitleCase = string => string.replace(/\b[a-z]/g, s => s.toUpperCase());
 
+const unicodeConversion = [
+  [/[ÀÁÂÃÅĀĂĄ]/g, 'A'],
+  [/[Æ]/g, 'AE'],
+  [/[ÇĆĈĊČ]/g, 'C'],
+  [/[ÈÉÊËĒĔĖĘĚ]/g, 'E'],
+  [/[ÌÍÎÏĨĪĬĮİ]/g, 'I'],
+  [/[Ððĥħ]/g, 'h'],
+  [/[ÑŃŅŇʼn]/g, 'N'],
+  [/[ÒÓÔÕÖØŌŎŐ]/g, 'O'],
+  [/[ÙÚÛŨŪŬŮŰŲ]/g, 'U'],
+  [/[ÝŶŸ]/g, 'Y'],
+  [/[Þñþńņň]/g, 'n'],
+  [/[ߌŜŞŠ]/g, 'S'],
+  [/[àáâãåāăąĸ]/g, 'a'],
+  [/[æ]/g, 'ae'],
+  [/[çćĉċč]/g, 'c'],
+  [/[èéêëēĕėęě]/g, 'e'],
+  [/[ìíîïĩīĭį]/g, 'i'],
+  [/[òóôõöøōŏő]/g, 'o'],
+  [/[ùúûũūŭůűų]/g, 'u'],
+  [/[ýÿŷ]/g, 'y'],
+  [/[ĎĐ]/g, 'D'],
+  [/[ďđ]/g, 'd'],
+  [/[ĜĞĠĢ]/g, 'G'],
+  [/[ĝğġģŊŋſ]/g, 'g'],
+  [/[ĤĦ]/g, 'H'],
+  [/[ıśŝşš]/g, 's'],
+  [/[IJ]/g, 'IJ'],
+  [/[ij]/g, 'ij'],
+  [/[Ĵ]/g, 'J'],
+  [/[ĵ]/g, 'j'],
+  [/[Ķ]/g, 'K'],
+  [/[ķ]/g, 'k'],
+  [/[ĹĻĽĿŁ]/g, 'L'],
+  [/[ĺļľŀł]/g, 'l'],
+  [/[Œ]/g, 'OE'],
+  [/[œ]/g, 'oe'],
+  [/[ŔŖŘ]/g, 'R'],
+  [/[ŕŗř]/g, 'r'],
+  [/[ŢŤŦ]/g, 'T'],
+  [/[ţťŧ]/g, 't'],
+  [/[Ŵ]/g, 'W'],
+  [/[ŵ]/g, 'w'],
+  [/[ŹŻŽ]/g, 'Z'],
+  [/[źżž]/g, 'z'],
+  [/ö/g, 'oe'],
+  [/ü/g, 'ue'],
+  [/ä/g, 'ae'],
+  // eslint-disable-next-line @gitlab/require-i18n-strings
+  [/Ö/g, 'Oe'],
+  // eslint-disable-next-line @gitlab/require-i18n-strings
+  [/Ü/g, 'Ue'],
+  // eslint-disable-next-line @gitlab/require-i18n-strings
+  [/Ä/g, 'Ae'],
+];
+
+/**
+ * Converts each non-ascii character in a string to
+ * it's ascii equivalent (if defined).
+ *
+ * e.g. "Dĭd söméònê äšk fœŕ Ůnĭċődę?" => "Did someone aesk foer Unicode?"
+ *
+ * @param {String} string
+ * @returns {String}
+ */
+export const convertUnicodeToAscii = string => {
+  let convertedString = string;
+
+  unicodeConversion.forEach(([regex, replacer]) => {
+    convertedString = convertedString.replace(regex, replacer);
+  });
+
+  return convertedString;
+};
+
 /**
  * Splits camelCase or PascalCase words
  * e.g. HelloWorld => Hello World

app/assets/javascripts/projects/project_new.js

 import $ from 'jquery';
 import DEFAULT_PROJECT_TEMPLATES from 'ee_else_ce/projects/default_project_templates';
 import { addSelectOnFocusBehaviour } from '../lib/utils/common_utils';
-import { convertToTitleCase, humanize, slugify } from '../lib/utils/text_utility';
+import {
+  convertToTitleCase,
+  humanize,
+  slugify,
+  convertUnicodeToAscii,
+} from '../lib/utils/text_utility';
 
 let hasUserDefinedProjectPath = false;
 let hasUserDefinedProjectName = false;
 
 const onProjectNameChange = ($projectNameInput, $projectPathInput) => {
-  const slug = slugify($projectNameInput.val());
+  const slug = slugify(convertUnicodeToAscii($projectNameInput.val()));
   $projectPathInput.val(slug);
 };
 

changelogs/unreleased/29642-Replace-Unicode-with-ASCII-Equivalent.yml

+---
+title: Replace Unicode Characters with ASCII Equivalent in New Project Slug
+merge_request: 39971
+author: Kev @KevSlashNull
+type: changed

spec/frontend/lib/utils/text_utility_spec.js

@@ -205,6 +205,26 @@ describe('text_utility', () => {
     });
   });
 
+  describe('convertUnicodeToAscii', () => {
+    it('does nothing on an empty string', () => {
+      expect(textUtils.convertUnicodeToAscii('')).toBe('');
+    });
+
+    it('does nothing on an already ascii string', () => {
+      expect(textUtils.convertUnicodeToAscii('The quick brown fox jumps over the lazy dog.')).toBe(
+        'The quick brown fox jumps over the lazy dog.',
+      );
+    });
+
+    it('replaces Unicode characters', () => {
+      expect(textUtils.convertUnicodeToAscii('Dĭd söméònê äšk fœŕ Ůnĭċődę?')).toBe(
+        'Did someone aesk foer Unicode?',
+      );
+
+      expect(textUtils.convertUnicodeToAscii("Jürgen's Projekt")).toBe("Juergen's Projekt");
+    });
+  });
+
   describe('splitCamelCase', () => {
     it('separates a PascalCase word to two', () => {
       expect(textUtils.splitCamelCase('HelloWorld')).toBe('Hello World');