Merge branch 'tz-reorganise-digests-json' into 'master'

Reorganise Setup of Emojis and loading of digests.json

Closes #39000

See merge request gitlab-org/gitlab-ce!24543

app/assets/javascripts/awards_handler.js

@@ -615,10 +615,18 @@ export class AwardsHandler {
 let awardsHandlerPromise = null;
 export default function loadAwardsHandler(reload = false) {
   if (!awardsHandlerPromise || reload) {
-    awardsHandlerPromise = import(/* webpackChunkName: 'emoji' */ './emoji').then(Emoji => {
-      const awardsHandler = new AwardsHandler(Emoji);
-      awardsHandler.bindEvents();
-      return awardsHandler;
+    awardsHandlerPromise = new Promise((resolve, reject) => {
+      import(/* webpackChunkName: 'emoji' */ './emoji')
+        .then(Emoji => {
+          Emoji.initEmojiMap()
+            .then(() => {
+              const awardsHandler = new AwardsHandler(Emoji);
+              awardsHandler.bindEvents();
+              resolve(awardsHandler);
+            })
+            .catch(() => reject);
+        })
+        .catch(() => reject);
     });
   }
   return awardsHandlerPromise;

app/assets/javascripts/behaviors/gl_emoji.js

 import 'document-register-element';
 import isEmojiUnicodeSupported from '../emoji/support';
+import { initEmojiMap, getEmojiInfo, emojiFallbackImageSrc, emojiImageTag } from '../emoji';
 
 class GlEmoji extends HTMLElement {
   constructor() {
     super();
 
-    const emojiUnicode = this.textContent.trim();
-    const { name, unicodeVersion, fallbackSrc, fallbackSpriteClass } = this.dataset;
-
-    const isEmojiUnicode =
-      this.childNodes &&
-      Array.prototype.every.call(this.childNodes, childNode => childNode.nodeType === 3);
-    const hasImageFallback = fallbackSrc && fallbackSrc.length > 0;
-    const hasCssSpriteFalback = fallbackSpriteClass && fallbackSpriteClass.length > 0;
-
-    if (emojiUnicode && isEmojiUnicode && !isEmojiUnicodeSupported(emojiUnicode, unicodeVersion)) {
-      // CSS sprite fallback takes precedence over image fallback
-      if (hasCssSpriteFalback) {
-        if (!gon.emoji_sprites_css_added && gon.emoji_sprites_css_path) {
-          const emojiSpriteLinkTag = document.createElement('link');
-          emojiSpriteLinkTag.setAttribute('rel', 'stylesheet');
-          emojiSpriteLinkTag.setAttribute('href', gon.emoji_sprites_css_path);
-          document.head.appendChild(emojiSpriteLinkTag);
-          gon.emoji_sprites_css_added = true;
-        }
-        // IE 11 doesn't like adding multiple at once :(
-        this.classList.add('emoji-icon');
-        this.classList.add(fallbackSpriteClass);
-      } else {
-        import(/* webpackChunkName: 'emoji' */ '../emoji')
-          .then(({ emojiImageTag, emojiFallbackImageSrc }) => {
-            if (hasImageFallback) {
-              this.innerHTML = emojiImageTag(name, fallbackSrc);
+    let emojiUnicode = this.textContent.trim();
+    const { fallbackSpriteClass, fallbackSrc, forceFallback } = this.dataset;
+    let { name, unicodeVersion } = this.dataset;
+
+    initEmojiMap()
+      .then(() => {
+        if (!unicodeVersion) {
+          const emojiInfo = getEmojiInfo(name);
+
+          if (emojiInfo) {
+            if (name !== emojiInfo.name) {
+              ({ name } = emojiInfo);
+              this.dataset.name = emojiInfo.name;
+            }
+            unicodeVersion = emojiInfo.u;
+            this.dataset.uni = unicodeVersion;
+
+            if (forceFallback === 'true' && !fallbackSpriteClass) {
+              this.innerHTML = emojiImageTag(name, emojiFallbackImageSrc(name));
             } else {
-              const src = emojiFallbackImageSrc(name);
-              this.innerHTML = emojiImageTag(name, src);
+              emojiUnicode = emojiInfo.e;
+              this.innerHTML = emojiInfo.e;
             }
-          })
-          .catch(() => {
-            // do nothing
-          });
-      }
-    }
+
+            this.title = emojiInfo.d;
+          }
+        }
+
+        const isEmojiUnicode =
+          this.childNodes &&
+          Array.prototype.every.call(this.childNodes, childNode => childNode.nodeType === 3);
+        const hasImageFallback = fallbackSrc && fallbackSrc.length > 0;
+        const hasCssSpriteFalback = fallbackSpriteClass && fallbackSpriteClass.length > 0;
+
+        if (
+          emojiUnicode &&
+          isEmojiUnicode &&
+          !isEmojiUnicodeSupported(emojiUnicode, unicodeVersion)
+        ) {
+          // CSS sprite fallback takes precedence over image fallback
+          if (hasCssSpriteFalback) {
+            if (!gon.emoji_sprites_css_added && gon.emoji_sprites_css_path) {
+              const emojiSpriteLinkTag = document.createElement('link');
+              emojiSpriteLinkTag.setAttribute('rel', 'stylesheet');
+              emojiSpriteLinkTag.setAttribute('href', gon.emoji_sprites_css_path);
+              document.head.appendChild(emojiSpriteLinkTag);
+              gon.emoji_sprites_css_added = true;
+            }
+            // IE 11 doesn't like adding multiple at once :(
+            this.classList.add('emoji-icon');
+            this.classList.add(fallbackSpriteClass);
+          } else if (hasImageFallback) {
+            this.innerHTML = emojiImageTag(name, fallbackSrc);
+          } else {
+            const src = emojiFallbackImageSrc(name);
+            this.innerHTML = emojiImageTag(name, src);
+          }
+        }
+      })
+      .catch(error => {
+        // Only reject is already handled in initEmojiMap
+        throw error;
+      });
   }
 }
 

app/assets/javascripts/emoji/index.js

 import _ from 'underscore';
-import emojiMap from 'emojis/digests.json';
+import createFlash from '~/flash';
+import { s__ } from '~/locale';
 import emojiAliases from 'emojis/aliases.json';
+import axios from '../lib/utils/axios_utils';
 
-export const validEmojiNames = [...Object.keys(emojiMap), ...Object.keys(emojiAliases)];
+import AccessorUtilities from '../lib/utils/accessor';
+
+let emojiMap = null;
+let validEmojiNames = null;
+
+export const EMOJI_VERSION = '1';
+const EMOJI_VERSION_LOCALSTORAGE = `EMOJIS_${EMOJI_VERSION}`;
+
+const isLocalStorageAvailable = AccessorUtilities.isLocalStorageAccessSafe();
+
+export function initEmojiMap() {
+  return new Promise((resolve, reject) => {
+    if (emojiMap) {
+      resolve(emojiMap);
+    } else if (isLocalStorageAvailable && window.localStorage.getItem(EMOJI_VERSION_LOCALSTORAGE)) {
+      emojiMap = JSON.parse(window.localStorage.getItem(EMOJI_VERSION_LOCALSTORAGE));
+      validEmojiNames = [...Object.keys(emojiMap), ...Object.keys(emojiAliases)];
+      resolve(emojiMap);
+    } else {
+      // We load the JSON from server
+      axios
+        .get(
+          `${gon.asset_host || ''}${gon.relative_url_root ||
+            ''}/-/emojis/${EMOJI_VERSION}/emojis.json`,
+        )
+        .then(({ data }) => {
+          emojiMap = data;
+          validEmojiNames = [...Object.keys(emojiMap), ...Object.keys(emojiAliases)];
+          resolve(emojiMap);
+          if (isLocalStorageAvailable) {
+            window.localStorage.setItem(EMOJI_VERSION_LOCALSTORAGE, JSON.stringify(emojiMap));
+          }
+        })
+        .catch(err => {
+          createFlash(s__('Emojis|Something went wrong while loading emojis.'));
+          reject(err);
+        });
+    }
+  });
+}
 
 export function normalizeEmojiName(name) {
   return Object.prototype.hasOwnProperty.call(emojiAliases, name) ? emojiAliases[name] : name;
 }
 
+export function getValidEmojiNames() {
+  return validEmojiNames;
+}
+
 export function isEmojiNameValid(name) {
   return validEmojiNames.indexOf(name) >= 0;
 }
@@ -36,8 +81,8 @@ export function getEmojiCategoryMap() {
     };
     Object.keys(emojiMap).forEach(name => {
       const emoji = emojiMap[name];
-      if (emojiCategoryMap[emoji.category]) {
-        emojiCategoryMap[emoji.category].push(name);
+      if (emojiCategoryMap[emoji.c]) {
+        emojiCategoryMap[emoji.c].push(name);
       }
     });
   }
@@ -58,8 +103,9 @@ export function getEmojiInfo(query) {
 }
 
 export function emojiFallbackImageSrc(inputName) {
-  const { name, digest } = getEmojiInfo(inputName);
-  return `${gon.asset_host || ''}${gon.relative_url_root || ''}/assets/emoji/${name}-${digest}.png`;
+  const { name } = getEmojiInfo(inputName);
+  return `${gon.asset_host || ''}${gon.relative_url_root ||
+    ''}/-/emojis/${EMOJI_VERSION}/${name}.png`;
 }
 
 export function emojiImageTag(name, src) {
@@ -68,9 +114,8 @@ export function emojiImageTag(name, src) {
 
 export function glEmojiTag(inputName, options) {
   const opts = { sprite: false, forceFallback: false, ...options };
-  const { name, ...emojiInfo } = getEmojiInfo(inputName);
+  const name = normalizeEmojiName(inputName);
 
-  const fallbackImageSrc = emojiFallbackImageSrc(name);
   const fallbackSpriteClass = `emoji-${name}`;
 
   const classList = [];
@@ -79,24 +124,19 @@ export function glEmojiTag(inputName, options) {
     classList.push(fallbackSpriteClass);
   }
   const classAttribute = classList.length > 0 ? `class="${classList.join(' ')}"` : '';
+
   const fallbackSpriteAttribute = opts.sprite
     ? `data-fallback-sprite-class="${fallbackSpriteClass}"`
     : '';
-  let contents = emojiInfo.moji;
-  if (opts.forceFallback && !opts.sprite) {
-    contents = emojiImageTag(name, fallbackImageSrc);
-  }
+  const forceFallbackAttribute = opts.forceFallback ? 'data-force-fallback="true"' : '';
 
   return `
     <gl-emoji
       ${classAttribute}
       data-name="${name}"
-      data-fallback-src="${fallbackImageSrc}"
       ${fallbackSpriteAttribute}
-      data-unicode-version="${emojiInfo.unicodeVersion}"
-      title="${emojiInfo.description}"
+      ${forceFallbackAttribute}
     >
-      ${contents}
     </gl-emoji>
   `;
 }

app/assets/javascripts/emoji/support/index.js

@@ -5,6 +5,9 @@ import getUnicodeSupportMap from './unicode_support_map';
 let browserUnicodeSupportMap;
 
 export default function isEmojiUnicodeSupportedByBrowser(emojiUnicode, unicodeVersion) {
+  // Our Spec browser would fail producing emoji maps
+  if (/\bHeadlessChrome\//.test(navigator.userAgent)) return true;
+
   browserUnicodeSupportMap = browserUnicodeSupportMap || getUnicodeSupportMap();
   return isEmojiUnicodeSupported(browserUnicodeSupportMap, emojiUnicode, unicodeVersion);
 }

app/assets/javascripts/filtered_search/visual_token_value.js

@@ -102,15 +102,24 @@ export default class VisualTokenValue {
     return (
       import(/* webpackChunkName: 'emoji' */ '../emoji')
         .then(Emoji => {
-          if (!Emoji.isEmojiNameValid(value)) {
-            return;
-          }
-
-          container.dataset.originalValue = value;
-          element.innerHTML = Emoji.glEmojiTag(value);
+          Emoji.initEmojiMap()
+            .then(() => {
+              if (!Emoji.isEmojiNameValid(value)) {
+                return;
+              }
+
+              container.dataset.originalValue = value;
+              element.innerHTML = Emoji.glEmojiTag(value);
+            })
+            // ignore error and leave emoji name in the search bar
+            .catch(err => {
+              throw err;
+            });
         })
         // ignore error and leave emoji name in the search bar
-        .catch(() => {})
+        .catch(importError => {
+          throw importError;
+        })
     );
   }
 }

app/assets/javascripts/gfm_auto_complete.js

@@ -487,9 +487,15 @@ class GfmAutoComplete {
       this.loadData($input, at, this.cachedData[at]);
     } else if (GfmAutoComplete.atTypeMap[at] === 'emojis') {
       import(/* webpackChunkName: 'emoji' */ './emoji')
-        .then(({ validEmojiNames, glEmojiTag }) => {
-          this.loadData($input, at, validEmojiNames);
-          GfmAutoComplete.glEmojiTag = glEmojiTag;
+        .then(({ initEmojiMap, getValidEmojiNames, glEmojiTag }) => {
+          initEmojiMap()
+            .then(() => {
+              this.loadData($input, at, getValidEmojiNames());
+              GfmAutoComplete.glEmojiTag = glEmojiTag;
+            })
+            .catch(() => {
+              this.isLoadingData[at] = false;
+            });
         })
         .catch(() => {
           this.isLoadingData[at] = false;

app/assets/javascripts/pages/profiles/show/index.js

@@ -56,30 +56,34 @@ document.addEventListener('DOMContentLoaded', () => {
 
   import(/* webpackChunkName: 'emoji' */ '~/emoji')
     .then(Emoji => {
-      const emojiMenu = new EmojiMenu(
-        Emoji,
-        toggleEmojiMenuButtonSelector,
-        'js-status-emoji-menu',
-        selectEmojiCallback,
-      );
-      emojiMenu.bindEvents();
+      Emoji.initEmojiMap()
+        .then(() => {
+          const emojiMenu = new EmojiMenu(
+            Emoji,
+            toggleEmojiMenuButtonSelector,
+            'js-status-emoji-menu',
+            selectEmojiCallback,
+          );
+          emojiMenu.bindEvents();
 
-      const defaultEmojiTag = Emoji.glEmojiTag(defaultStatusEmoji);
-      statusMessageField.addEventListener('input', () => {
-        const hasStatusMessage = statusMessageField.value.trim() !== '';
-        const statusEmoji = findStatusEmoji();
-        if (hasStatusMessage && statusEmoji) {
-          return;
-        }
+          const defaultEmojiTag = Emoji.glEmojiTag(defaultStatusEmoji);
+          statusMessageField.addEventListener('input', () => {
+            const hasStatusMessage = statusMessageField.value.trim() !== '';
+            const statusEmoji = findStatusEmoji();
+            if (hasStatusMessage && statusEmoji) {
+              return;
+            }
 
-        if (hasStatusMessage) {
-          toggleNoEmojiPlaceholder(false);
-          toggleEmojiMenuButton.innerHTML += defaultEmojiTag;
-        } else if (statusEmoji.dataset.name === defaultStatusEmoji) {
-          toggleNoEmojiPlaceholder(true);
-          removeStatusEmoji();
-        }
-      });
+            if (hasStatusMessage) {
+              toggleNoEmojiPlaceholder(false);
+              toggleEmojiMenuButton.innerHTML += defaultEmojiTag;
+            } else if (statusEmoji.dataset.name === defaultStatusEmoji) {
+              toggleNoEmojiPlaceholder(true);
+              removeStatusEmoji();
+            }
+          });
+        })
+        .catch(() => createFlash('Failed to load emoji list.'));
     })
     .catch(() => createFlash('Failed to load emoji list.'));
 });

app/assets/javascripts/set_status_modal/set_status_modal_wrapper.vue

@@ -66,19 +66,23 @@ export default {
 
       import(/* webpackChunkName: 'emoji' */ '~/emoji')
         .then(Emoji => {
-          if (this.emoji) {
-            this.emojiTag = Emoji.glEmojiTag(this.emoji);
-          }
-          this.noEmoji = this.emoji === '';
-          this.defaultEmojiTag = Emoji.glEmojiTag('speech_balloon');
+          Emoji.initEmojiMap()
+            .then(() => {
+              if (this.emoji) {
+                this.emojiTag = Emoji.glEmojiTag(this.emoji);
+              }
+              this.noEmoji = this.emoji === '';
+              this.defaultEmojiTag = Emoji.glEmojiTag('speech_balloon');
 
-          this.emojiMenu = new EmojiMenuInModal(
-            Emoji,
-            toggleEmojiMenuButtonSelector,
-            emojiMenuClass,
-            this.setEmoji,
-            this.$refs.userStatusForm,
-          );
+              this.emojiMenu = new EmojiMenuInModal(
+                Emoji,
+                toggleEmojiMenuButtonSelector,
+                emojiMenuClass,
+                this.setEmoji,
+                this.$refs.userStatusForm,
+              );
+            })
+            .catch(() => createFlash(__('Failed to load emoji list.')));
         })
         .catch(() => createFlash(__('Failed to load emoji list.')));
     },

lib/tasks/gemojione.rake

@@ -30,33 +30,28 @@ namespace :gemojione do
     # We don't have `node_modules` available in built versions of GitLab
     FileUtils.cp_r(Rails.root.join('node_modules', 'emoji-unicode-version', 'emoji-unicode-version-map.json'), File.join(Rails.root, 'fixtures', 'emojis'))
 
-    dir = Gemojione.images_path
     resultant_emoji_map = {}
 
     Gitlab::Emoji.emojis.each do |name, emoji_hash|
       # Ignore aliases
       unless Gitlab::Emoji.emojis_aliases.key?(name)
-        fpath = File.join(dir, "#{emoji_hash['unicode']}.png")
-        hash_digest = Digest::SHA256.file(fpath).hexdigest
-
         category = emoji_hash['category']
         if name == 'gay_pride_flag'
           category = 'flags'
         end
 
         entry = {
-          category: category,
-          moji: emoji_hash['moji'],
-          description: emoji_hash['description'],
-          unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name),
-          digest: hash_digest
+          c: category,
+          e: emoji_hash['moji'],
+          d: emoji_hash['description'],
+          u: Gitlab::Emoji.emoji_unicode_version(name)
         }
 
         resultant_emoji_map[name] = entry
       end
     end
 
-    out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
+    out = File.join(Rails.root, 'public', '-', 'emojis', '1', 'emojis.json')
     File.open(out, 'w') do |handle|
       handle.write(JSON.pretty_generate(resultant_emoji_map))
     end

locale/gitlab.pot

@@ -2968,6 +2968,9 @@ msgstr ""
 msgid "Embed"
 msgstr ""
 
+msgid "Emojis|Something went wrong while loading emojis."
+msgstr ""
+
 msgid "Empty file"
 msgstr ""