Merge branch 'tz-reorganise-digests-json' into 'master'

Reorganise Setup of Emojis and loading of digests.json

Closes #39000

See merge request gitlab-org/gitlab-ce!24543

app/assets/javascripts/awards_handler.js

@@ -615,10 +615,18 @@ export class AwardsHandler {
 let awardsHandlerPromise = null;
 export default function loadAwardsHandler(reload = false) {
   if (!awardsHandlerPromise || reload) {
-    awardsHandlerPromise = import(/* webpackChunkName: 'emoji' */ './emoji').then(Emoji => {
+    awardsHandlerPromise = new Promise((resolve, reject) => {
+      import(/* webpackChunkName: 'emoji' */ './emoji')
+        .then(Emoji => {
+          Emoji.initEmojiMap()
+            .then(() => {
               const awardsHandler = new AwardsHandler(Emoji);
               awardsHandler.bindEvents();
-      return awardsHandler;
+              resolve(awardsHandler);
+            })
+            .catch(() => reject);
+        })
+        .catch(() => reject);
     });
   }
   return awardsHandlerPromise;

app/assets/javascripts/behaviors/gl_emoji.js

 import 'document-register-element';
 import isEmojiUnicodeSupported from '../emoji/support';
+import { initEmojiMap, getEmojiInfo, emojiFallbackImageSrc, emojiImageTag } from '../emoji';
 
 class GlEmoji extends HTMLElement {
   constructor() {
     super();
 
-    const emojiUnicode = this.textContent.trim();
-    const { name, unicodeVersion, fallbackSrc, fallbackSpriteClass } = this.dataset;
+    let emojiUnicode = this.textContent.trim();
+    const { fallbackSpriteClass, fallbackSrc, forceFallback } = this.dataset;
+    let { name, unicodeVersion } = this.dataset;
+
+    initEmojiMap()
+      .then(() => {
+        if (!unicodeVersion) {
+          const emojiInfo = getEmojiInfo(name);
+
+          if (emojiInfo) {
+            if (name !== emojiInfo.name) {
+              ({ name } = emojiInfo);
+              this.dataset.name = emojiInfo.name;
+            }
+            unicodeVersion = emojiInfo.u;
+            this.dataset.uni = unicodeVersion;
+
+            if (forceFallback === 'true' && !fallbackSpriteClass) {
+              this.innerHTML = emojiImageTag(name, emojiFallbackImageSrc(name));
+            } else {
+              emojiUnicode = emojiInfo.e;
+              this.innerHTML = emojiInfo.e;
+            }
+
+            this.title = emojiInfo.d;
+          }
+        }
 
         const isEmojiUnicode =
           this.childNodes &&
@@ -14,7 +40,11 @@ class GlEmoji extends HTMLElement {
         const hasImageFallback = fallbackSrc && fallbackSrc.length > 0;
         const hasCssSpriteFalback = fallbackSpriteClass && fallbackSpriteClass.length > 0;
 
-    if (emojiUnicode && isEmojiUnicode && !isEmojiUnicodeSupported(emojiUnicode, unicodeVersion)) {
+        if (
+          emojiUnicode &&
+          isEmojiUnicode &&
+          !isEmojiUnicodeSupported(emojiUnicode, unicodeVersion)
+        ) {
           // CSS sprite fallback takes precedence over image fallback
           if (hasCssSpriteFalback) {
             if (!gon.emoji_sprites_css_added && gon.emoji_sprites_css_path) {
@@ -27,22 +57,19 @@ class GlEmoji extends HTMLElement {
             // IE 11 doesn't like adding multiple at once :(
             this.classList.add('emoji-icon');
             this.classList.add(fallbackSpriteClass);
-      } else {
-        import(/* webpackChunkName: 'emoji' */ '../emoji')
-          .then(({ emojiImageTag, emojiFallbackImageSrc }) => {
-            if (hasImageFallback) {
+          } else if (hasImageFallback) {
             this.innerHTML = emojiImageTag(name, fallbackSrc);
           } else {
             const src = emojiFallbackImageSrc(name);
             this.innerHTML = emojiImageTag(name, src);
           }
+        }
       })
-          .catch(() => {
-            // do nothing
+      .catch(error => {
+        // Only reject is already handled in initEmojiMap
+        throw error;
       });
   }
-    }
-  }
 }
 
 export default function installGlEmojiElement() {

app/assets/javascripts/emoji/index.js

 import _ from 'underscore';
-import emojiMap from 'emojis/digests.json';
+import createFlash from '~/flash';
+import { s__ } from '~/locale';
 import emojiAliases from 'emojis/aliases.json';
+import axios from '../lib/utils/axios_utils';
 
-export const validEmojiNames = [...Object.keys(emojiMap), ...Object.keys(emojiAliases)];
+import AccessorUtilities from '../lib/utils/accessor';
+
+let emojiMap = null;
+let validEmojiNames = null;
+
+export const EMOJI_VERSION = '1';
+const EMOJI_VERSION_LOCALSTORAGE = `EMOJIS_${EMOJI_VERSION}`;
+
+const isLocalStorageAvailable = AccessorUtilities.isLocalStorageAccessSafe();
+
+export function initEmojiMap() {
+  return new Promise((resolve, reject) => {
+    if (emojiMap) {
+      resolve(emojiMap);
+    } else if (isLocalStorageAvailable && window.localStorage.getItem(EMOJI_VERSION_LOCALSTORAGE)) {
+      emojiMap = JSON.parse(window.localStorage.getItem(EMOJI_VERSION_LOCALSTORAGE));
+      validEmojiNames = [...Object.keys(emojiMap), ...Object.keys(emojiAliases)];
+      resolve(emojiMap);
+    } else {
+      // We load the JSON from server
+      axios
+        .get(
+          `${gon.asset_host || ''}${gon.relative_url_root ||
+            ''}/-/emojis/${EMOJI_VERSION}/emojis.json`,
+        )
+        .then(({ data }) => {
+          emojiMap = data;
+          validEmojiNames = [...Object.keys(emojiMap), ...Object.keys(emojiAliases)];
+          resolve(emojiMap);
+          if (isLocalStorageAvailable) {
+            window.localStorage.setItem(EMOJI_VERSION_LOCALSTORAGE, JSON.stringify(emojiMap));
+          }
+        })
+        .catch(err => {
+          createFlash(s__('Emojis|Something went wrong while loading emojis.'));
+          reject(err);
+        });
+    }
+  });
+}
 
 export function normalizeEmojiName(name) {
   return Object.prototype.hasOwnProperty.call(emojiAliases, name) ? emojiAliases[name] : name;
 }
 
+export function getValidEmojiNames() {
+  return validEmojiNames;
+}
+
 export function isEmojiNameValid(name) {
   return validEmojiNames.indexOf(name) >= 0;
 }
@@ -36,8 +81,8 @@ export function getEmojiCategoryMap() {
     };
     Object.keys(emojiMap).forEach(name => {
       const emoji = emojiMap[name];
-      if (emojiCategoryMap[emoji.category]) {
-        emojiCategoryMap[emoji.category].push(name);
+      if (emojiCategoryMap[emoji.c]) {
+        emojiCategoryMap[emoji.c].push(name);
       }
     });
   }
@@ -58,8 +103,9 @@ export function getEmojiInfo(query) {
 }
 
 export function emojiFallbackImageSrc(inputName) {
-  const { name, digest } = getEmojiInfo(inputName);
-  return `${gon.asset_host || ''}${gon.relative_url_root || ''}/assets/emoji/${name}-${digest}.png`;
+  const { name } = getEmojiInfo(inputName);
+  return `${gon.asset_host || ''}${gon.relative_url_root ||
+    ''}/-/emojis/${EMOJI_VERSION}/${name}.png`;
 }
 
 export function emojiImageTag(name, src) {
@@ -68,9 +114,8 @@ export function emojiImageTag(name, src) {
 
 export function glEmojiTag(inputName, options) {
   const opts = { sprite: false, forceFallback: false, ...options };
-  const { name, ...emojiInfo } = getEmojiInfo(inputName);
+  const name = normalizeEmojiName(inputName);
 
-  const fallbackImageSrc = emojiFallbackImageSrc(name);
   const fallbackSpriteClass = `emoji-${name}`;
 
   const classList = [];
@@ -79,24 +124,19 @@ export function glEmojiTag(inputName, options) {
     classList.push(fallbackSpriteClass);
   }
   const classAttribute = classList.length > 0 ? `class="${classList.join(' ')}"` : '';
+
   const fallbackSpriteAttribute = opts.sprite
     ? `data-fallback-sprite-class="${fallbackSpriteClass}"`
     : '';
-  let contents = emojiInfo.moji;
-  if (opts.forceFallback && !opts.sprite) {
-    contents = emojiImageTag(name, fallbackImageSrc);
-  }
+  const forceFallbackAttribute = opts.forceFallback ? 'data-force-fallback="true"' : '';
 
   return `
     <gl-emoji
       ${classAttribute}
       data-name="${name}"
-      data-fallback-src="${fallbackImageSrc}"
       ${fallbackSpriteAttribute}
-      data-unicode-version="${emojiInfo.unicodeVersion}"
-      title="${emojiInfo.description}"
+      ${forceFallbackAttribute}
     >
-      ${contents}
     </gl-emoji>
   `;
 }

app/assets/javascripts/emoji/support/index.js

@@ -5,6 +5,9 @@ import getUnicodeSupportMap from './unicode_support_map';
 let browserUnicodeSupportMap;
 
 export default function isEmojiUnicodeSupportedByBrowser(emojiUnicode, unicodeVersion) {
+  // Our Spec browser would fail producing emoji maps
+  if (/\bHeadlessChrome\//.test(navigator.userAgent)) return true;
+
   browserUnicodeSupportMap = browserUnicodeSupportMap || getUnicodeSupportMap();
   return isEmojiUnicodeSupported(browserUnicodeSupportMap, emojiUnicode, unicodeVersion);
 }

app/assets/javascripts/filtered_search/visual_token_value.js

@@ -102,6 +102,8 @@ export default class VisualTokenValue {
     return (
       import(/* webpackChunkName: 'emoji' */ '../emoji')
         .then(Emoji => {
+          Emoji.initEmojiMap()
+            .then(() => {
               if (!Emoji.isEmojiNameValid(value)) {
                 return;
               }
@@ -110,7 +112,14 @@ export default class VisualTokenValue {
               element.innerHTML = Emoji.glEmojiTag(value);
             })
             // ignore error and leave emoji name in the search bar
-        .catch(() => {})
+            .catch(err => {
+              throw err;
+            });
+        })
+        // ignore error and leave emoji name in the search bar
+        .catch(importError => {
+          throw importError;
+        })
     );
   }
 }

app/assets/javascripts/gfm_auto_complete.js

@@ -487,13 +487,19 @@ class GfmAutoComplete {
       this.loadData($input, at, this.cachedData[at]);
     } else if (GfmAutoComplete.atTypeMap[at] === 'emojis') {
       import(/* webpackChunkName: 'emoji' */ './emoji')
-        .then(({ validEmojiNames, glEmojiTag }) => {
-          this.loadData($input, at, validEmojiNames);
+        .then(({ initEmojiMap, getValidEmojiNames, glEmojiTag }) => {
+          initEmojiMap()
+            .then(() => {
+              this.loadData($input, at, getValidEmojiNames());
               GfmAutoComplete.glEmojiTag = glEmojiTag;
             })
             .catch(() => {
               this.isLoadingData[at] = false;
             });
+        })
+        .catch(() => {
+          this.isLoadingData[at] = false;
+        });
     } else if (dataSource) {
       AjaxCache.retrieve(dataSource, true)
         .then(data => {

app/assets/javascripts/pages/profiles/show/index.js

@@ -56,6 +56,8 @@ document.addEventListener('DOMContentLoaded', () => {
 
   import(/* webpackChunkName: 'emoji' */ '~/emoji')
     .then(Emoji => {
+      Emoji.initEmojiMap()
+        .then(() => {
           const emojiMenu = new EmojiMenu(
             Emoji,
             toggleEmojiMenuButtonSelector,
@@ -82,4 +84,6 @@ document.addEventListener('DOMContentLoaded', () => {
           });
         })
         .catch(() => createFlash('Failed to load emoji list.'));
+    })
+    .catch(() => createFlash('Failed to load emoji list.'));
 });

app/assets/javascripts/set_status_modal/set_status_modal_wrapper.vue

@@ -66,6 +66,8 @@ export default {
 
       import(/* webpackChunkName: 'emoji' */ '~/emoji')
         .then(Emoji => {
+          Emoji.initEmojiMap()
+            .then(() => {
               if (this.emoji) {
                 this.emojiTag = Emoji.glEmojiTag(this.emoji);
               }
@@ -81,6 +83,8 @@ export default {
               );
             })
             .catch(() => createFlash(__('Failed to load emoji list.')));
+        })
+        .catch(() => createFlash(__('Failed to load emoji list.')));
     },
     showEmojiMenu() {
       this.isEmojiMenuVisible = true;

lib/tasks/gemojione.rake

@@ -30,33 +30,28 @@ namespace :gemojione do
     # We don't have `node_modules` available in built versions of GitLab
     FileUtils.cp_r(Rails.root.join('node_modules', 'emoji-unicode-version', 'emoji-unicode-version-map.json'), File.join(Rails.root, 'fixtures', 'emojis'))
 
-    dir = Gemojione.images_path
     resultant_emoji_map = {}
 
     Gitlab::Emoji.emojis.each do |name, emoji_hash|
       # Ignore aliases
       unless Gitlab::Emoji.emojis_aliases.key?(name)
-        fpath = File.join(dir, "#{emoji_hash['unicode']}.png")
-        hash_digest = Digest::SHA256.file(fpath).hexdigest
-
         category = emoji_hash['category']
         if name == 'gay_pride_flag'
           category = 'flags'
         end
 
         entry = {
-          category: category,
-          moji: emoji_hash['moji'],
-          description: emoji_hash['description'],
-          unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name),
-          digest: hash_digest
+          c: category,
+          e: emoji_hash['moji'],
+          d: emoji_hash['description'],
+          u: Gitlab::Emoji.emoji_unicode_version(name)
         }
 
         resultant_emoji_map[name] = entry
       end
     end
 
-    out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
+    out = File.join(Rails.root, 'public', '-', 'emojis', '1', 'emojis.json')
     File.open(out, 'w') do |handle|
       handle.write(JSON.pretty_generate(resultant_emoji_map))
     end

locale/gitlab.pot

@@ -2968,6 +2968,9 @@ msgstr ""
 msgid "Embed"
 msgstr ""
 
+msgid "Emojis|Something went wrong while loading emojis."
+msgstr ""
+
 msgid "Empty file"
 msgstr ""