Commit 46f70b20 authored by Paul Slaughter's avatar Paul Slaughter

Merge branch '26113-file-type-issue' into 'master'

Fix MIME / encoding related issues in Web IDE file uploader

See merge request gitlab-org/gitlab!26360
parents c9af24c2 7bfa75d4
<script>
import ItemButton from './button.vue';
import { isTextFile } from '~/ide/utils';
export default {
components: {
......@@ -23,29 +24,11 @@ export default {
},
},
methods: {
isText(content, fileType) {
const knownBinaryFileTypes = ['image/'];
const knownTextFileTypes = ['text/'];
const isKnownBinaryFileType = knownBinaryFileTypes.find(type => fileType.includes(type));
const isKnownTextFileType = knownTextFileTypes.find(type => fileType.includes(type));
const asciiRegex = /^[ -~\t\n\r]+$/; // tests whether a string contains ascii characters only (ranges from space to tilde, tabs and new lines)
if (isKnownBinaryFileType) {
return false;
}
if (isKnownTextFileType) {
return true;
}
// if it's not a known file type, determine the type by evaluating the file contents
return asciiRegex.test(content);
},
createFile(target, file) {
const { name } = file;
const encodedContent = target.result.split('base64,')[1];
const rawContent = encodedContent ? atob(encodedContent) : '';
const isText = this.isText(rawContent, file.type);
const isText = isTextFile(rawContent, file.type, name);
const emitCreateEvent = content =>
this.$emit('create', {
......
import { commitItemIconMap } from './constants';
import { languages } from 'monaco-editor';
import { flatten } from 'lodash';
const toLowerCase = x => x.toLowerCase();
const monacoLanguages = languages.getLanguages();
const monacoExtensions = new Set(
flatten(monacoLanguages.map(lang => lang.extensions?.map(toLowerCase) || [])),
);
const monacoMimetypes = new Set(
flatten(monacoLanguages.map(lang => lang.mimetypes?.map(toLowerCase) || [])),
);
const monacoFilenames = new Set(
flatten(monacoLanguages.map(lang => lang.filenames?.map(toLowerCase) || [])),
);
const KNOWN_TYPES = [
{
isText: false,
isMatch(mimeType) {
return mimeType.toLowerCase().includes('image/');
},
},
{
isText: true,
isMatch(mimeType) {
return mimeType.toLowerCase().includes('text/');
},
},
{
isText: true,
isMatch(mimeType, fileName) {
const fileExtension = fileName.includes('.') ? `.${fileName.split('.').pop()}` : '';
return (
monacoExtensions.has(fileExtension.toLowerCase()) ||
monacoMimetypes.has(mimeType.toLowerCase()) ||
monacoFilenames.has(fileName.toLowerCase())
);
},
},
];
export function isTextFile(content, mimeType, fileName) {
const knownType = KNOWN_TYPES.find(type => type.isMatch(mimeType, fileName));
if (knownType) return knownType.isText;
// does the string contain ascii characters only (ranges from space to tilde, tabs and new lines)
const asciiRegex = /^[ -~\t\n\r]+$/;
// for unknown types, determine the type by evaluating the file contents
return asciiRegex.test(content);
}
export const getCommitIconMap = file => {
if (file.deleted) {
......
---
title: Fix issues with non-ASCII plain text files being incorrectly uploaded as binary
in the Web IDE
merge_request: 26360
author:
type: fixed
import { commitItemIconMap } from '~/ide/constants';
import { getCommitIconMap } from '~/ide/utils';
import { getCommitIconMap, isTextFile } from '~/ide/utils';
import { decorateData } from '~/ide/stores/utils';
describe('WebIDE utils', () => {
describe('isTextFile', () => {
it('returns false for known binary types', () => {
expect(isTextFile('file content', 'image/png', 'my.png')).toBeFalsy();
// mime types are case insensitive
expect(isTextFile('file content', 'IMAGE/PNG', 'my.png')).toBeFalsy();
});
it('returns true for known text types', () => {
expect(isTextFile('file content', 'text/plain', 'my.txt')).toBeTruthy();
// mime types are case insensitive
expect(isTextFile('file content', 'TEXT/PLAIN', 'my.txt')).toBeTruthy();
});
it('returns true for file extensions that Monaco supports syntax highlighting for', () => {
// test based on both MIME and extension
expect(isTextFile('{"éêė":"value"}', 'application/json', 'my.json')).toBeTruthy();
expect(isTextFile('{"éêė":"value"}', 'application/json', '.tsconfig')).toBeTruthy();
expect(isTextFile('SELECT "éêė" from tablename', 'application/sql', 'my.sql')).toBeTruthy();
});
it('returns true even irrespective of whether the mimes, extensions or file names are lowercase or upper case', () => {
expect(isTextFile('{"éêė":"value"}', 'application/json', 'MY.JSON')).toBeTruthy();
expect(isTextFile('SELECT "éêė" from tablename', 'application/sql', 'MY.SQL')).toBeTruthy();
expect(
isTextFile('var code = "something"', 'application/javascript', 'Gruntfile'),
).toBeTruthy();
expect(
isTextFile(
'MAINTAINER Александр "alexander11354322283@me.com"',
'application/octet-stream',
'dockerfile',
),
).toBeTruthy();
});
it('returns false if filename is same as the expected extension', () => {
expect(isTextFile('SELECT "éêė" from tablename', 'application/sql', 'sql')).toBeFalsy();
});
it('returns true for ASCII only content for unknown types', () => {
expect(isTextFile('plain text', 'application/x-new-type', 'hello.mytype')).toBeTruthy();
});
it('returns true for relevant filenames', () => {
expect(
isTextFile(
'MAINTAINER Александр "alexander11354322283@me.com"',
'application/octet-stream',
'Dockerfile',
),
).toBeTruthy();
});
it('returns false for non-ASCII content for unknown types', () => {
expect(isTextFile('{"éêė":"value"}', 'application/octet-stream', 'my.random')).toBeFalsy();
});
});
const createFile = (name = 'name', id = name, type = '', parent = null) =>
decorateData({
id,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment