Merge branch 'send-incremental-build-log' into 'master'

Update build log incrementally

Proof of concept implementation of incremental sending of build log to browser.

cc @jschatz1 @vsizov @grzesiek @tmaczukin 


See merge request !3737

app/assets/javascripts/ci/build.coffee

 class CiBuild
   @interval: null
+  @state: null
 
-  constructor: (build_url, build_status) ->
+  constructor: (build_url, build_status, build_state) ->
     clearInterval(CiBuild.interval)
 
+    @state = build_state
+
     @initScrollButtonAffix()
 
     if build_status == "running" || build_status == "pending"
@@ -26,14 +29,18 @@ class CiBuild
       CiBuild.interval = setInterval =>
         if window.location.href.split("#").first() is build_url
           $.ajax
-            url: build_url
+            url: build_url + "/trace.json?state=" + encodeURIComponent(@state)
             dataType: "json"
-            success: (build) =>
-              if build.status == "running"
-                $('#build-trace code').html build.trace_html
+            success: (log) =>
+              @state = log.state
+              if log.status is "running"
+                if log.append
+                  $('.fa-refresh').before log.html
+                else
+                  $('#build-trace code').html log.html
                   $('#build-trace code').append '<i class="fa fa-refresh fa-spin"/>'
                 @checkAutoscroll()
-              else if build.status != build_status
+              else if log.status isnt build_status
                 Turbolinks.visit build_url
       , 4000
 

app/controllers/projects/builds_controller.rb

@@ -38,6 +38,14 @@ class Projects::BuildsController < Projects::ApplicationController
     end
   end
 
+  def trace
+    respond_to do |format|
+      format.json do
+        render json: @build.trace_with_state(params[:state]).merge!(id: @build.id, status: @build.status)
+      end
+    end
+  end
+
   def retry
     unless @build.retryable?
       return render_404

app/models/ci/build.rb

@@ -95,8 +95,12 @@ module Ci
     end
 
     def trace_html
-      html = Ci::Ansi2html::convert(trace) if trace.present?
-      html || ''
+      trace_with_state[:html] || ''
+    end
+
+    def trace_with_state(state = nil)
+      trace_with_state = Ci::Ansi2html::convert(trace, state) if trace.present?
+      trace_with_state || {}
     end
 
     def timeout

app/views/projects/builds/show.html.haml

 - page_title "#{@build.name} (##{@build.id})", "Builds"
 = render "header_title"
+- trace_with_state = @build.trace_with_state
 
 .build-page
   .row-content-block.top-block
@@ -85,7 +86,9 @@
         %pre.trace#build-trace
           %code.bash
             = preserve do
-              = raw @build.trace_html
+              = raw trace_with_state[:html]
+              - if @build.active?
+                %i{:class => "fa fa-refresh fa-spin"}
 
       %div#down-build-trace
 
@@ -216,4 +219,4 @@
 
 
   :javascript
-    new CiBuild("#{namespace_project_build_url(@project.namespace, @project, @build)}", "#{@build.status}")
+    new CiBuild("#{namespace_project_build_url(@project.namespace, @project, @build)}", "#{@build.status}", "#{trace_with_state[:state]}")

config/routes.rb

@@ -668,6 +668,7 @@ Rails.application.routes.draw do
             post :cancel
             post :retry
             post :erase
+            get :trace
             get :raw
           end
 

lib/ci/ansi2html.rb

@@ -23,8 +23,8 @@ module Ci
       cross:      0x10,
     }
 
-    def self.convert(ansi)
-      Converter.new().convert(ansi)
+    def self.convert(ansi, state = nil)
+      Converter.new.convert(ansi, state)
     end
 
     class Converter
@@ -84,22 +84,36 @@ module Ci
       def on_107(s) set_bg_color(7, 'l') end
       def on_109(s) set_bg_color(9, 'l') end
 
-      def convert(ansi)
-        @out = ""
-        @n_open_tags = 0
-        reset()
+      attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask
+
+      STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask]
+
+      def convert(raw, new_state)
+        reset_state
+        restore_state(raw, new_state) if new_state
+
+        start = @offset
+        ansi = raw[@offset..-1]
 
-        s = StringScanner.new(ansi.gsub("<", "&lt;"))
+        open_new_tag
+
+        s = StringScanner.new(ansi)
         while(!s.eos?)
           if s.scan(/\e([@-_])(.*?)([@-~])/)
             handle_sequence(s)
+          elsif s.scan(/\e(([@-_])(.*?)?)?$/)
+            break
+          elsif s.scan(/</)
+            @out << '&lt;'
           else
             @out << s.scan(/./m)
           end
+          @offset += s.matched_size
         end
 
         close_open_tags()
-        @out
+
+        { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 }
       end
 
       def handle_sequence(s)
@@ -121,6 +135,20 @@ module Ci
 
         evaluate_command_stack(commands)
 
+        open_new_tag
+      end
+
+      def evaluate_command_stack(stack)
+        return unless command = stack.shift()
+
+        if self.respond_to?("on_#{command}", true)
+          self.send("on_#{command}", stack)
+        end
+
+        evaluate_command_stack(stack)
+      end
+
+      def open_new_tag
         css_classes = []
 
         unless @fg_color.nil?
@@ -138,20 +166,8 @@ module Ci
           css_classes << "term-#{css_class}" if @style_mask & flag != 0
         end
 
-        open_new_tag(css_classes) if css_classes.length > 0
-      end
-
-      def evaluate_command_stack(stack)
-        return unless command = stack.shift()
-
-        if self.respond_to?("on_#{command}", true)
-          self.send("on_#{command}", stack)
-        end
-
-        evaluate_command_stack(stack)
-      end
+        return if css_classes.empty?
 
-      def open_new_tag(css_classes)
         @out << %{<span class="#{css_classes.join(' ')}">}
         @n_open_tags += 1
       end
@@ -163,6 +179,31 @@ module Ci
         end
       end
 
+      def reset_state
+        @offset = 0
+        @n_open_tags = 0
+        @out = ''
+        reset
+      end
+
+      def state
+        state = STATE_PARAMS.inject({}) do |h, param|
+          h[param] = send(param)
+          h
+        end
+        Base64.urlsafe_encode64(state.to_json)
+      end
+
+      def restore_state(raw, new_state)
+        state = Base64.urlsafe_decode64(new_state)
+        state = JSON.parse(state, symbolize_names: true)
+        return if state[:offset].to_i > raw.length
+
+        STATE_PARAMS.each do |param|
+          send("#{param}=".to_sym, state[param])
+        end
+      end
+
       def reset
         @fg_color = nil
         @bg_color = nil

spec/lib/ci/ansi2html_spec.rb

@@ -4,131 +4,176 @@ describe Ci::Ansi2html, lib: true do
   subject { Ci::Ansi2html }
 
   it "prints non-ansi as-is" do
-    expect(subject.convert("Hello")).to eq('Hello')
+    expect(subject.convert("Hello")[:html]).to eq('Hello')
   end
 
   it "strips non-color-changing controll sequences" do
-    expect(subject.convert("Hello \e[2Kworld")).to eq('Hello world')
+    expect(subject.convert("Hello \e[2Kworld")[:html]).to eq('Hello world')
   end
 
   it "prints simply red" do
-    expect(subject.convert("\e[31mHello\e[0m")).to eq('<span class="term-fg-red">Hello</span>')
+    expect(subject.convert("\e[31mHello\e[0m")[:html]).to eq('<span class="term-fg-red">Hello</span>')
   end
 
   it "prints simply red without trailing reset" do
-    expect(subject.convert("\e[31mHello")).to eq('<span class="term-fg-red">Hello</span>')
+    expect(subject.convert("\e[31mHello")[:html]).to eq('<span class="term-fg-red">Hello</span>')
   end
 
   it "prints simply yellow" do
-    expect(subject.convert("\e[33mHello\e[0m")).to eq('<span class="term-fg-yellow">Hello</span>')
+    expect(subject.convert("\e[33mHello\e[0m")[:html]).to eq('<span class="term-fg-yellow">Hello</span>')
   end
 
   it "prints default on blue" do
-    expect(subject.convert("\e[39;44mHello")).to eq('<span class="term-bg-blue">Hello</span>')
+    expect(subject.convert("\e[39;44mHello")[:html]).to eq('<span class="term-bg-blue">Hello</span>')
   end
 
   it "prints red on blue" do
-    expect(subject.convert("\e[31;44mHello")).to eq('<span class="term-fg-red term-bg-blue">Hello</span>')
+    expect(subject.convert("\e[31;44mHello")[:html]).to eq('<span class="term-fg-red term-bg-blue">Hello</span>')
   end
 
   it "resets colors after red on blue" do
-    expect(subject.convert("\e[31;44mHello\e[0m world")).to eq('<span class="term-fg-red term-bg-blue">Hello</span> world')
+    expect(subject.convert("\e[31;44mHello\e[0m world")[:html]).to eq('<span class="term-fg-red term-bg-blue">Hello</span> world')
   end
 
   it "performs color change from red/blue to yellow/blue" do
-    expect(subject.convert("\e[31;44mHello \e[33mworld")).to eq('<span class="term-fg-red term-bg-blue">Hello </span><span class="term-fg-yellow term-bg-blue">world</span>')
+    expect(subject.convert("\e[31;44mHello \e[33mworld")[:html]).to eq('<span class="term-fg-red term-bg-blue">Hello </span><span class="term-fg-yellow term-bg-blue">world</span>')
   end
 
   it "performs color change from red/blue to yellow/green" do
-    expect(subject.convert("\e[31;44mHello \e[33;42mworld")).to eq('<span class="term-fg-red term-bg-blue">Hello </span><span class="term-fg-yellow term-bg-green">world</span>')
+    expect(subject.convert("\e[31;44mHello \e[33;42mworld")[:html]).to eq('<span class="term-fg-red term-bg-blue">Hello </span><span class="term-fg-yellow term-bg-green">world</span>')
   end
 
   it "performs color change from red/blue to reset to yellow/green" do
-    expect(subject.convert("\e[31;44mHello\e[0m \e[33;42mworld")).to eq('<span class="term-fg-red term-bg-blue">Hello</span> <span class="term-fg-yellow term-bg-green">world</span>')
+    expect(subject.convert("\e[31;44mHello\e[0m \e[33;42mworld")[:html]).to eq('<span class="term-fg-red term-bg-blue">Hello</span> <span class="term-fg-yellow term-bg-green">world</span>')
   end
 
   it "ignores unsupported codes" do
-    expect(subject.convert("\e[51mHello\e[0m")).to eq('Hello')
+    expect(subject.convert("\e[51mHello\e[0m")[:html]).to eq('Hello')
   end
 
   it "prints light red" do
-    expect(subject.convert("\e[91mHello\e[0m")).to eq('<span class="term-fg-l-red">Hello</span>')
+    expect(subject.convert("\e[91mHello\e[0m")[:html]).to eq('<span class="term-fg-l-red">Hello</span>')
   end
 
   it "prints default on light red" do
-    expect(subject.convert("\e[101mHello\e[0m")).to eq('<span class="term-bg-l-red">Hello</span>')
+    expect(subject.convert("\e[101mHello\e[0m")[:html]).to eq('<span class="term-bg-l-red">Hello</span>')
   end
 
   it "performs color change from red/blue to default/blue" do
-    expect(subject.convert("\e[31;44mHello \e[39mworld")).to eq('<span class="term-fg-red term-bg-blue">Hello </span><span class="term-bg-blue">world</span>')
+    expect(subject.convert("\e[31;44mHello \e[39mworld")[:html]).to eq('<span class="term-fg-red term-bg-blue">Hello </span><span class="term-bg-blue">world</span>')
   end
 
   it "performs color change from light red/blue to default/blue" do
-    expect(subject.convert("\e[91;44mHello \e[39mworld")).to eq('<span class="term-fg-l-red term-bg-blue">Hello </span><span class="term-bg-blue">world</span>')
+    expect(subject.convert("\e[91;44mHello \e[39mworld")[:html]).to eq('<span class="term-fg-l-red term-bg-blue">Hello </span><span class="term-bg-blue">world</span>')
   end
 
   it "prints bold text" do
-    expect(subject.convert("\e[1mHello")).to eq('<span class="term-bold">Hello</span>')
+    expect(subject.convert("\e[1mHello")[:html]).to eq('<span class="term-bold">Hello</span>')
   end
 
   it "resets bold text" do
-    expect(subject.convert("\e[1mHello\e[21m world")).to eq('<span class="term-bold">Hello</span> world')
-    expect(subject.convert("\e[1mHello\e[22m world")).to eq('<span class="term-bold">Hello</span> world')
+    expect(subject.convert("\e[1mHello\e[21m world")[:html]).to eq('<span class="term-bold">Hello</span> world')
+    expect(subject.convert("\e[1mHello\e[22m world")[:html]).to eq('<span class="term-bold">Hello</span> world')
   end
 
   it "prints italic text" do
-    expect(subject.convert("\e[3mHello")).to eq('<span class="term-italic">Hello</span>')
+    expect(subject.convert("\e[3mHello")[:html]).to eq('<span class="term-italic">Hello</span>')
   end
 
   it "resets italic text" do
-    expect(subject.convert("\e[3mHello\e[23m world")).to eq('<span class="term-italic">Hello</span> world')
+    expect(subject.convert("\e[3mHello\e[23m world")[:html]).to eq('<span class="term-italic">Hello</span> world')
   end
 
   it "prints underlined text" do
-    expect(subject.convert("\e[4mHello")).to eq('<span class="term-underline">Hello</span>')
+    expect(subject.convert("\e[4mHello")[:html]).to eq('<span class="term-underline">Hello</span>')
   end
 
   it "resets underlined text" do
-    expect(subject.convert("\e[4mHello\e[24m world")).to eq('<span class="term-underline">Hello</span> world')
+    expect(subject.convert("\e[4mHello\e[24m world")[:html]).to eq('<span class="term-underline">Hello</span> world')
   end
 
   it "prints concealed text" do
-    expect(subject.convert("\e[8mHello")).to eq('<span class="term-conceal">Hello</span>')
+    expect(subject.convert("\e[8mHello")[:html]).to eq('<span class="term-conceal">Hello</span>')
   end
 
   it "resets concealed text" do
-    expect(subject.convert("\e[8mHello\e[28m world")).to eq('<span class="term-conceal">Hello</span> world')
+    expect(subject.convert("\e[8mHello\e[28m world")[:html]).to eq('<span class="term-conceal">Hello</span> world')
   end
 
   it "prints crossed-out text" do
-    expect(subject.convert("\e[9mHello")).to eq('<span class="term-cross">Hello</span>')
+    expect(subject.convert("\e[9mHello")[:html]).to eq('<span class="term-cross">Hello</span>')
   end
 
   it "resets crossed-out text" do
-    expect(subject.convert("\e[9mHello\e[29m world")).to eq('<span class="term-cross">Hello</span> world')
+    expect(subject.convert("\e[9mHello\e[29m world")[:html]).to eq('<span class="term-cross">Hello</span> world')
   end
 
   it "can print 256 xterm fg colors" do
-    expect(subject.convert("\e[38;5;16mHello")).to eq('<span class="xterm-fg-16">Hello</span>')
+    expect(subject.convert("\e[38;5;16mHello")[:html]).to eq('<span class="xterm-fg-16">Hello</span>')
   end
 
   it "can print 256 xterm fg colors on normal magenta background" do
-    expect(subject.convert("\e[38;5;16;45mHello")).to eq('<span class="xterm-fg-16 term-bg-magenta">Hello</span>')
+    expect(subject.convert("\e[38;5;16;45mHello")[:html]).to eq('<span class="xterm-fg-16 term-bg-magenta">Hello</span>')
   end
 
   it "can print 256 xterm bg colors" do
-    expect(subject.convert("\e[48;5;240mHello")).to eq('<span class="xterm-bg-240">Hello</span>')
+    expect(subject.convert("\e[48;5;240mHello")[:html]).to eq('<span class="xterm-bg-240">Hello</span>')
   end
 
   it "can print 256 xterm bg colors on normal magenta foreground" do
-    expect(subject.convert("\e[48;5;16;35mHello")).to eq('<span class="term-fg-magenta xterm-bg-16">Hello</span>')
+    expect(subject.convert("\e[48;5;16;35mHello")[:html]).to eq('<span class="term-fg-magenta xterm-bg-16">Hello</span>')
   end
 
   it "prints bold colored text vividly" do
-    expect(subject.convert("\e[1;31mHello\e[0m")).to eq('<span class="term-fg-l-red term-bold">Hello</span>')
+    expect(subject.convert("\e[1;31mHello\e[0m")[:html]).to eq('<span class="term-fg-l-red term-bold">Hello</span>')
   end
 
   it "prints bold light colored text correctly" do
-    expect(subject.convert("\e[1;91mHello\e[0m")).to eq('<span class="term-fg-l-red term-bold">Hello</span>')
+    expect(subject.convert("\e[1;91mHello\e[0m")[:html]).to eq('<span class="term-fg-l-red term-bold">Hello</span>')
+  end
+
+  it "prints &lt;" do
+    expect(subject.convert("<")[:html]).to eq('&lt;')
+  end
+
+  describe "incremental update" do
+    shared_examples 'stateable converter' do
+      let(:pass1) { subject.convert(pre_text) }
+      let(:pass2) { subject.convert(pre_text + text, pass1[:state]) }
+
+      it "to returns html to append" do
+        expect(pass2[:append]).to be_truthy
+        expect(pass2[:html]).to eq(html)
+        expect(pass1[:text] + pass2[:text]).to eq(pre_text + text)
+        expect(pass1[:html] + pass2[:html]).to eq(pre_html + html)
+      end
+    end
+
+    context "with split word" do
+      let(:pre_text) { "\e[1mHello" }
+      let(:pre_html) { "<span class=\"term-bold\">Hello</span>" }
+      let(:text) { "\e[1mWorld" }
+      let(:html) { "<span class=\"term-bold\"></span><span class=\"term-bold\">World</span>" }
+
+      it_behaves_like 'stateable converter'
+    end
+
+    context "with split sequence" do
+      let(:pre_text) { "\e[1m" }
+      let(:pre_html) { "<span class=\"term-bold\"></span>" }
+      let(:text) { "Hello" }
+      let(:html) { "<span class=\"term-bold\">Hello</span>" }
+
+      it_behaves_like 'stateable converter'
+    end
+
+    context "with partial sequence" do
+      let(:pre_text) { "Hello\e" }
+      let(:pre_html) { "Hello" }
+      let(:text) { "[1m World" }
+      let(:html) { "<span class=\"term-bold\"> World</span>" }
+
+      it_behaves_like 'stateable converter'
+    end
   end
 end