Merge branch 'sh-force-encoding-utf8-build-logs-pipeline-view' into 'master'

Fix Error 500 viewing pipelines with invalid UTF-8 data

See merge request gitlab-org/gitlab!66686

lib/gitlab/ci/ansi2html.rb

@@ -33,6 +33,8 @@ module Gitlab
       Result = Struct.new(:html, :state, :append, :truncated, :offset, :size, :total, keyword_init: true) # rubocop:disable Lint/StructNewOverride
 
       class Converter
+        include EncodingHelper
+
         def on_0(_)
           reset
         end
@@ -256,6 +258,7 @@ module Gitlab
           start_offset = @offset
 
           stream.each_line do |line|
+            line = encode_utf8_no_detect(line)
             s = StringScanner.new(line)
 
             until s.eos?

lib/gitlab/ci/ansi2json/line.rb

@@ -9,6 +9,8 @@ module Gitlab
         # Line::Segment is a portion of a line that has its own style
         # and text. Multiple segments make the line content.
         class Segment
+          include EncodingHelper
+
           attr_accessor :text, :style
 
           def initialize(style:)
@@ -21,19 +23,15 @@ module Gitlab
           end
 
           def to_h
-            { text: encode_text(text) }.tap do |result|
+            # Without forcing the encoding to UTF-8 and then replacing
+            # invalid UTF-8 sequences we can get an error when serializing
+            # the Hash to JSON.
+            # Encoding::UndefinedConversionError:
+            #   "\xE2" from ASCII-8BIT to UTF-8
+            { text: encode_utf8_no_detect(text) }.tap do |result|
               result[:style] = style.to_s if style.set?
             end
           end
-
-          # Without forcing the encoding to UTF-8 and then dropping
-          # invalid UTF-8 sequences we can get an error when serializing
-          # the Hash to JSON.
-          # Encoding::UndefinedConversionError:
-          #   "\xE2" from ASCII-8BIT to UTF-8
-          def encode_text(text)
-            text.force_encoding(Encoding::UTF_8).encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
-          end
         end
 
         attr_reader :offset, :sections, :segments, :current_segment,

lib/gitlab/encoding_helper.rb

@@ -64,6 +64,15 @@ module Gitlab
       detect && detect[:type] == :binary
     end
 
+    # This is like encode_utf8 except we skip autodetection of the encoding. We
+    # assume the data must be interpreted as UTF-8.
+    def encode_utf8_no_detect(message)
+      message = force_encode_utf8(message)
+      return message if message.valid_encoding?
+
+      message.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+    end
+
     def encode_utf8(message, replace: "")
       message = force_encode_utf8(message)
       return message if message.valid_encoding?

spec/lib/gitlab/ci/ansi2html_spec.rb

@@ -150,6 +150,10 @@ RSpec.describe Gitlab::Ci::Ansi2html do
     expect(convert_html("\r\n")).to eq('<span><br/></span>')
   end
 
+  it 'replaces invalid UTF-8 data' do
+    expect(convert_html("UTF-8 dashes here: ───\n🐤🐤🐤🐤\xF0\x9F\x90\n")).to eq("<span>UTF-8 dashes here: ───<br/>🐤🐤🐤🐤�<br/></span>")
+  end
+
   describe "incremental update" do
     shared_examples 'stateable converter' do
       let(:pass1_stream) { StringIO.new(pre_text) }

spec/lib/gitlab/encoding_helper_spec.rb

@@ -3,6 +3,8 @@
 require "spec_helper"
 
 RSpec.describe Gitlab::EncodingHelper do
+  using RSpec::Parameterized::TableSyntax
+
   let(:ext_class) { Class.new { extend Gitlab::EncodingHelper } }
   let(:binary_string) { File.read(Rails.root + "spec/fixtures/dk.png") }
 
@@ -90,6 +92,22 @@ RSpec.describe Gitlab::EncodingHelper do
     end
   end
 
+  describe '#encode_utf8_no_detect' do
+    where(:input, :expected) do
+      "abcd" | "abcd"
+      "DzDzDz" | "DzDzDz"
+      "\xC7\xB2\xC7DzDzDz" | "Dz�DzDzDz"
+      "🐤🐤🐤🐤\xF0\x9F\x90" | "🐤🐤🐤🐤�"
+    end
+
+    with_them do
+      it 'drops invalid UTF-8' do
+        expect(ext_class.encode_utf8_no_detect(input.dup.force_encoding(Encoding::ASCII_8BIT))).to eq(expected)
+        expect(ext_class.encode_utf8_no_detect(input)).to eq(expected)
+      end
+    end
+  end
+
   describe '#encode_utf8' do
     [
       ["nil", nil, nil],