Merge branch '26113-file-type-issue' into 'master'

Fix MIME / encoding related issues in Web IDE file uploader

See merge request gitlab-org/gitlab!26360

app/assets/javascripts/ide/components/new_dropdown/upload.vue

 <script>
 import ItemButton from './button.vue';
+import { isTextFile } from '~/ide/utils';
 
 export default {
   components: {
@@ -23,29 +24,11 @@ export default {
     },
   },
   methods: {
-    isText(content, fileType) {
-      const knownBinaryFileTypes = ['image/'];
-      const knownTextFileTypes = ['text/'];
-      const isKnownBinaryFileType = knownBinaryFileTypes.find(type => fileType.includes(type));
-      const isKnownTextFileType = knownTextFileTypes.find(type => fileType.includes(type));
-      const asciiRegex = /^[ -~\t\n\r]+$/; // tests whether a string contains ascii characters only (ranges from space to tilde, tabs and new lines)
-
-      if (isKnownBinaryFileType) {
-        return false;
-      }
-
-      if (isKnownTextFileType) {
-        return true;
-      }
-
-      // if it's not a known file type, determine the type by evaluating the file contents
-      return asciiRegex.test(content);
-    },
     createFile(target, file) {
       const { name } = file;
       const encodedContent = target.result.split('base64,')[1];
       const rawContent = encodedContent ? atob(encodedContent) : '';
-      const isText = this.isText(rawContent, file.type);
+      const isText = isTextFile(rawContent, file.type, name);
 
       const emitCreateEvent = content =>
         this.$emit('create', {

app/assets/javascripts/ide/utils.js

 import { commitItemIconMap } from './constants';
+import { languages } from 'monaco-editor';
+import { flatten } from 'lodash';
+
+const toLowerCase = x => x.toLowerCase();
+
+const monacoLanguages = languages.getLanguages();
+const monacoExtensions = new Set(
+  flatten(monacoLanguages.map(lang => lang.extensions?.map(toLowerCase) || [])),
+);
+const monacoMimetypes = new Set(
+  flatten(monacoLanguages.map(lang => lang.mimetypes?.map(toLowerCase) || [])),
+);
+const monacoFilenames = new Set(
+  flatten(monacoLanguages.map(lang => lang.filenames?.map(toLowerCase) || [])),
+);
+
+const KNOWN_TYPES = [
+  {
+    isText: false,
+    isMatch(mimeType) {
+      return mimeType.toLowerCase().includes('image/');
+    },
+  },
+  {
+    isText: true,
+    isMatch(mimeType) {
+      return mimeType.toLowerCase().includes('text/');
+    },
+  },
+  {
+    isText: true,
+    isMatch(mimeType, fileName) {
+      const fileExtension = fileName.includes('.') ? `.${fileName.split('.').pop()}` : '';
+
+      return (
+        monacoExtensions.has(fileExtension.toLowerCase()) ||
+        monacoMimetypes.has(mimeType.toLowerCase()) ||
+        monacoFilenames.has(fileName.toLowerCase())
+      );
+    },
+  },
+];
+
+export function isTextFile(content, mimeType, fileName) {
+  const knownType = KNOWN_TYPES.find(type => type.isMatch(mimeType, fileName));
+
+  if (knownType) return knownType.isText;
+
+  // does the string contain ascii characters only (ranges from space to tilde, tabs and new lines)
+  const asciiRegex = /^[ -~\t\n\r]+$/;
+  // for unknown types, determine the type by evaluating the file contents
+  return asciiRegex.test(content);
+}
 
 export const getCommitIconMap = file => {
   if (file.deleted) {

changelogs/unreleased/26113-file-type-issue.yml

+---
+title: Fix issues with non-ASCII plain text files being incorrectly uploaded as binary
+  in the Web IDE
+merge_request: 26360
+author:
+type: fixed

spec/frontend/ide/utils_spec.js

 import { commitItemIconMap } from '~/ide/constants';
-import { getCommitIconMap } from '~/ide/utils';
+import { getCommitIconMap, isTextFile } from '~/ide/utils';
 import { decorateData } from '~/ide/stores/utils';
 
 describe('WebIDE utils', () => {
+  describe('isTextFile', () => {
+    it('returns false for known binary types', () => {
+      expect(isTextFile('file content', 'image/png', 'my.png')).toBeFalsy();
+      // mime types are case insensitive
+      expect(isTextFile('file content', 'IMAGE/PNG', 'my.png')).toBeFalsy();
+    });
+
+    it('returns true for known text types', () => {
+      expect(isTextFile('file content', 'text/plain', 'my.txt')).toBeTruthy();
+      // mime types are case insensitive
+      expect(isTextFile('file content', 'TEXT/PLAIN', 'my.txt')).toBeTruthy();
+    });
+
+    it('returns true for file extensions that Monaco supports syntax highlighting for', () => {
+      // test based on both MIME and extension
+      expect(isTextFile('{"éêė":"value"}', 'application/json', 'my.json')).toBeTruthy();
+      expect(isTextFile('{"éêė":"value"}', 'application/json', '.tsconfig')).toBeTruthy();
+      expect(isTextFile('SELECT "éêė" from tablename', 'application/sql', 'my.sql')).toBeTruthy();
+    });
+
+    it('returns true even irrespective of whether the mimes, extensions or file names are lowercase or upper case', () => {
+      expect(isTextFile('{"éêė":"value"}', 'application/json', 'MY.JSON')).toBeTruthy();
+      expect(isTextFile('SELECT "éêė" from tablename', 'application/sql', 'MY.SQL')).toBeTruthy();
+      expect(
+        isTextFile('var code = "something"', 'application/javascript', 'Gruntfile'),
+      ).toBeTruthy();
+      expect(
+        isTextFile(
+          'MAINTAINER Александр "alexander11354322283@me.com"',
+          'application/octet-stream',
+          'dockerfile',
+        ),
+      ).toBeTruthy();
+    });
+
+    it('returns false if filename is same as the expected extension', () => {
+      expect(isTextFile('SELECT "éêė" from tablename', 'application/sql', 'sql')).toBeFalsy();
+    });
+
+    it('returns true for ASCII only content for unknown types', () => {
+      expect(isTextFile('plain text', 'application/x-new-type', 'hello.mytype')).toBeTruthy();
+    });
+
+    it('returns true for relevant filenames', () => {
+      expect(
+        isTextFile(
+          'MAINTAINER Александр "alexander11354322283@me.com"',
+          'application/octet-stream',
+          'Dockerfile',
+        ),
+      ).toBeTruthy();
+    });
+
+    it('returns false for non-ASCII content for unknown types', () => {
+      expect(isTextFile('{"éêė":"value"}', 'application/octet-stream', 'my.random')).toBeFalsy();
+    });
+  });
+
   const createFile = (name = 'name', id = name, type = '', parent = null) =>
     decorateData({
       id,