encoding_helper_spec.rb 6.74 KB
Newer Older
Robert Speicher's avatar
Robert Speicher committed
1 2
require "spec_helper"

3 4
describe Gitlab::EncodingHelper do
  let(:ext_class) { Class.new { extend Gitlab::EncodingHelper } }
5
  let(:binary_string) { File.read(Rails.root + "spec/fixtures/dk.png") }
Robert Speicher's avatar
Robert Speicher committed
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21

  describe '#encode!' do
    [
      [
        'leaves ascii only string as is',
        'ascii only string',
        'ascii only string'
      ],
      [
        'leaves valid utf8 string as is',
        'multibyte string №∑∉',
        'multibyte string №∑∉'
      ],
      [
        'removes invalid bytes from ASCII-8bit encoded multibyte string. This can occur when a git diff match line truncates in the middle of a multibyte character. This occurs after the second word in this example. The test string is as short as we can get while still triggering the error condition when not looking at `detect[:confidence]`.',
        "mu ns\xC3\n Lorem ipsum dolor sit amet, consectetur adipisicing ut\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p\n {: .normal_pn}\n \n-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in\n# *Lorem ipsum\xC3\xB9l\xC3\xB9l\xC3\xA0 dolor\xC3\xB9k\xC3\xB9 sit\xC3\xA8b\xC3\xA8 N\xC3\xA8 amet b\xC3\xA0d\xC3\xAC*\n+# *consectetur\xC3\xB9l\xC3\xB9l\xC3\xA0 adipisicing\xC3\xB9k\xC3\xB9 elit\xC3\xA8b\xC3\xA8 N\xC3\xA8 sed do\xC3\xA0d\xC3\xAC*{: .italic .smcaps}\n \n \xEF\x9B\xA1 eiusmod tempor incididunt, ut\xC3\xAAn\xC3\xB9 labore et dolore. Tw\xC4\x83nj\xC3\xAC magna aliqua. Ut enim ad minim veniam\n {: .normal}\n@@ -9,5 +9,5 @@ quis nostrud\xC3\xAAt\xC3\xB9 exercitiation ullamco laboris m\xC3\xB9s\xC3\xB9k\xC3\xB9abc\xC3\xB9 nisi ".force_encoding('ASCII-8BIT'),
22 23
        "mu ns\n Lorem ipsum dolor sit amet, consectetur adipisicing ut\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p\n {: .normal_pn}\n \n-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in\n# *Lorem ipsum\xC3\xB9l\xC3\xB9l\xC3\xA0 dolor\xC3\xB9k\xC3\xB9 sit\xC3\xA8b\xC3\xA8 N\xC3\xA8 amet b\xC3\xA0d\xC3\xAC*\n+# *consectetur\xC3\xB9l\xC3\xB9l\xC3\xA0 adipisicing\xC3\xB9k\xC3\xB9 elit\xC3\xA8b\xC3\xA8 N\xC3\xA8 sed do\xC3\xA0d\xC3\xAC*{: .italic .smcaps}\n \n \xEF\x9B\xA1 eiusmod tempor incididunt, ut\xC3\xAAn\xC3\xB9 labore et dolore. Tw\xC4\x83nj\xC3\xAC magna aliqua. Ut enim ad minim veniam\n {: .normal}\n@@ -9,5 +9,5 @@ quis nostrud\xC3\xAAt\xC3\xB9 exercitiation ullamco laboris m\xC3\xB9s\xC3\xB9k\xC3\xB9abc\xC3\xB9 nisi "
      ]
Robert Speicher's avatar
Robert Speicher committed
24 25 26 27 28 29 30 31 32
    ].each do |description, test_string, xpect|
      it description do
        expect(ext_class.encode!(test_string)).to eq(xpect)
      end
    end

    it 'leaves binary string as is' do
      expect(ext_class.encode!(binary_string)).to eq(binary_string)
    end
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66

    context 'with corrupted diff' do
      let(:corrupted_diff) do
        with_empty_bare_repository do |repo|
          content = File.read(Rails.root.join(
            'spec/fixtures/encoding/Japanese.md').to_s)
          commit_a = commit(repo, 'Japanese.md', content)
          commit_b = commit(repo, 'Japanese.md',
            content.sub('[TODO: Link]', '[現在作業中です: Link]'))

          repo.diff(commit_a, commit_b).each_line.map(&:content).join
        end
      end

      let(:cleaned_diff) do
        corrupted_diff.dup.force_encoding('UTF-8')
          .encode!('UTF-8', invalid: :replace, replace: '')
      end

      let(:encoded_diff) do
        described_class.encode!(corrupted_diff.dup)
      end

      it 'does not corrupt data but remove invalid characters' do
        expect(encoded_diff).to eq(cleaned_diff)
      end

      def commit(repo, path, content)
        oid = repo.write(content, :blob)
        index = repo.index

        index.read_tree(repo.head.target.tree) unless repo.empty?

        index.add(path: path, oid: oid, mode: 0100644)
67
        user = { name: 'Test', email: 'test@example.com' }
68 69 70 71

        Rugged::Commit.create(
          repo,
          tree: index.write_tree(repo),
72 73
          author: user,
          committer: user,
74 75 76 77 78 79
          message: "Update #{path}",
          parents: repo.empty? ? [] : [repo.head.target].compact,
          update_ref: 'HEAD'
        )
      end
    end
Robert Speicher's avatar
Robert Speicher committed
80 81 82 83 84 85 86
  end

  describe '#encode_utf8' do
    [
      [
        "encodes valid utf8 encoded string to utf8",
        "λ, λ, λ".encode("UTF-8"),
87
        "λ, λ, λ".encode("UTF-8")
Robert Speicher's avatar
Robert Speicher committed
88 89 90 91
      ],
      [
        "encodes valid ASCII-8BIT encoded string to utf8",
        "ascii only".encode("ASCII-8BIT"),
92
        "ascii only".encode("UTF-8")
Robert Speicher's avatar
Robert Speicher committed
93 94 95 96
      ],
      [
        "encodes valid ISO-8859-1 encoded string to utf8",
        "Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann.".encode("ISO-8859-1", "UTF-8"),
97 98
        "Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann.".encode("UTF-8")
      ]
Robert Speicher's avatar
Robert Speicher committed
99 100 101 102 103 104 105
    ].each do |description, test_string, xpect|
      it description do
        r = ext_class.encode_utf8(test_string.force_encoding('UTF-8'))
        expect(r).to eq(xpect)
        expect(r.encoding.name).to eq('UTF-8')
      end
    end
106 107 108 109

    it 'returns empty string on conversion errors' do
      expect { ext_class.encode_utf8('') }.not_to raise_error(ArgumentError)
    end
Robert Speicher's avatar
Robert Speicher committed
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
  end

  describe '#clean' do
    [
      [
        'leaves ascii only string as is',
        'ascii only string',
        'ascii only string'
      ],
      [
        'leaves valid utf8 string as is',
        'multibyte string №∑∉',
        'multibyte string №∑∉'
      ],
      [
        'removes invalid bytes from ASCII-8bit encoded multibyte string.',
        "Lorem ipsum\xC3\n dolor sit amet, xy\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg".force_encoding('ASCII-8BIT'),
127 128
        "Lorem ipsum\n dolor sit amet, xyàyùabcdùefg"
      ]
Robert Speicher's avatar
Robert Speicher committed
129 130 131 132 133 134 135
    ].each do |description, test_string, xpect|
      it description do
        expect(ext_class.encode!(test_string)).to eq(xpect)
      end
    end
  end
end