Merge branch '198413-CI-Pre-Collapsed-Sections' into 'master'

Pre-Collapsed Sections in CI Job Logs

See merge request gitlab-org/gitlab!42231

app/assets/javascripts/jobs/store/utils.js

+import { parseBoolean } from '../../lib/utils/common_utils';
+
 /**
  * Adds the line number property
  * @param Object line
@@ -17,7 +19,7 @@ export const parseLine = (line = {}, lineNumber) => ({
  * @param Number lineNumber
  */
 export const parseHeaderLine = (line = {}, lineNumber) => ({
-  isClosed: false,
+  isClosed: parseBoolean(line.section_options?.collapsed),
   isHeader: true,
   line: parseLine(line, lineNumber),
   lines: [],

changelogs/unreleased/198413-CI-Pre-Collapsed-Sections.yml

+---
+title: Pre-Collapsed Sections in CI Job Logs
+merge_request: 42231
+author: Kev @KevSlashNull
+type: added

doc/ci/pipelines/index.md

@@ -461,6 +461,28 @@ this line should be hidden when collapsed
 section_end:1560896353:my_first_section\r\e[0K
 ```
 
+#### Pre-collapse sections
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198413) in GitLab 13.5.
+
+You can make the job log automatically collapse collapsible sections by adding the `collapsed` option to the section start.
+Add `[collapsed=true]` after the section name and before the `\r`. The section end marker
+remains unchanged:
+
+- Section start marker with `[collapsed=true]`: `section_start:UNIX_TIMESTAMP:SECTION_NAME[collapsed=true]\r\e[0K` + `TEXT_OF_SECTION_HEADER`
+- Section end marker: `section_end:UNIX_TIMESTAMP:SECTION_NAME\r\e[0K`
+
+Add the updated section start text to the CI configuration. For example,
+using `echo`:
+
+```yaml
+job1:
+  script:
+    - echo -e "section_start:`date +%s`:my_first_section[collapsed=true]\r\e[0KHeader of the 1st collapsible section"
+    - echo 'this line should be hidden automatically after loading the job log'
+    - echo -e "section_end:`date +%s`:my_first_section\r\e[0K"
+```
+
 ## Visualize pipelines
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5742) in GitLab 8.11.

lib/gitlab/ci/ansi2json/converter.rb

@@ -104,23 +104,24 @@ module Gitlab
           action = scanner[1]
           timestamp = scanner[2]
           section = scanner[3]
+          options = parse_section_options(scanner[4])
 
           section_name = sanitize_section_name(section)
 
-          if action == "start"
-            handle_section_start(scanner, section_name, timestamp)
-          elsif action == "end"
+          if action == 'start'
+            handle_section_start(scanner, section_name, timestamp, options)
+          elsif action == 'end'
             handle_section_end(scanner, section_name, timestamp)
           else
             raise 'unsupported action'
           end
         end
 
-        def handle_section_start(scanner, section, timestamp)
+        def handle_section_start(scanner, section, timestamp, options)
           # We make a new line for new section
           flush_current_line
 
-          @state.open_section(section, timestamp)
+          @state.open_section(section, timestamp, options)
 
           # we need to consume match after handling
           # the open of section, as we want the section
@@ -157,6 +158,18 @@ module Gitlab
         def sanitize_section_name(section)
           section.to_s.downcase.gsub(/[^a-z0-9]/, '-')
         end
+
+        def parse_section_options(raw_options)
+          return unless raw_options
+
+          # We need to remove the square brackets and split
+          # by comma to get a list of the options
+          options = raw_options[1...-1].split ','
+
+          # Now split each option by equals to separate
+          # each in the format [key, value]
+          options.to_h { |option| option.split '=' }
+        end
       end
     end
   end

lib/gitlab/ci/ansi2json/line.rb

@@ -32,7 +32,7 @@ module Gitlab
         end
 
         attr_reader :offset, :sections, :segments, :current_segment,
-                    :section_header, :section_duration
+                    :section_header, :section_duration, :section_options
 
         def initialize(offset:, style:, sections: [])
           @offset = offset
@@ -68,6 +68,10 @@ module Gitlab
           @sections << section
         end
 
+        def set_section_options(options)
+          @section_options = options
+        end
+
         def set_as_section_header
           @section_header = true
         end
@@ -90,6 +94,7 @@ module Gitlab
             result[:section] = sections.last if sections.any?
             result[:section_header] = true if @section_header
             result[:section_duration] = @section_duration if @section_duration
+            result[:section_options] = @section_options if @section_options
           end
         end
       end

lib/gitlab/ci/ansi2json/state.rb

@@ -26,10 +26,11 @@ module Gitlab
           Base64.urlsafe_encode64(state.to_json)
         end
 
-        def open_section(section, timestamp)
+        def open_section(section, timestamp, options)
           @open_sections[section] = timestamp
 
           @current_line.add_section(section)
+          @current_line.set_section_options(options)
           @current_line.set_as_section_header
         end
 

lib/gitlab/regex.rb

@@ -220,8 +220,27 @@ module Gitlab
       "Must start with a letter, and cannot end with '-'"
     end
 
+    # The section start, e.g. section_start:12345678:NAME
+    def logs_section_prefix_regex
+      /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)/
+    end
+
+    # The optional section options, e.g. [collapsed=true]
+    def logs_section_options_regex
+      /(\[(?:\w+=\w+)(?:, ?(?:\w+=\w+))*\])?/
+    end
+
+    # The region end, always: \r\e\[0K
+    def logs_section_suffix_regex
+      /\r\033\[0K/
+    end
+
     def build_trace_section_regex
-      @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze
+      @build_trace_section_regexp ||= %r{
+        #{logs_section_prefix_regex}
+        #{logs_section_options_regex}
+        #{logs_section_suffix_regex}
+      }x.freeze
     end
 
     def markdown_code_or_html_blocks

spec/frontend/jobs/store/utils_spec.js

@@ -35,6 +35,14 @@ describe('Jobs Store Utils', () => {
         lines: [],
       });
     });
+
+    it('pre-closes a section when specified in options', () => {
+      const headerLine = { content: [{ text: 'foo' }], section_options: { collapsed: 'true' } };
+
+      const parsedHeaderLine = parseHeaderLine(headerLine, 2);
+
+      expect(parsedHeaderLine.isClosed).toBe(true);
+    });
   });
 
   describe('parseLine', () => {

spec/lib/gitlab/ci/ansi2json/line_spec.rb

@@ -58,6 +58,15 @@ RSpec.describe Gitlab::Ci::Ansi2json::Line do
     end
   end
 
+  describe '#set_section_options' do
+    it 'sets the current section\'s options' do
+      options = { collapsed: true }
+      subject.set_section_options(options)
+
+      expect(subject.to_h[:section_options]).to eq(options)
+    end
+  end
+
   describe '#set_as_section_header' do
     it 'change the section_header to true' do
       expect { subject.set_as_section_header }

spec/lib/gitlab/ci/ansi2json_spec.rb

@@ -229,7 +229,7 @@ RSpec.describe Gitlab::Ci::Ansi2json do
         expect(convert_json(trace)).to eq([
           {
             offset: 0,
-            content: [{ text: "section_end:1:2<div>hello</div>" }],
+            content: [{ text: 'section_end:1:2<div>hello</div>' }],
             section: 'prepare-script',
             section_header: true
           },
@@ -329,6 +329,32 @@ RSpec.describe Gitlab::Ci::Ansi2json do
             ])
         end
       end
+
+      context 'with section options' do
+        let(:option_section_start) { "section_start:#{section_start_time.to_i}:#{section_name}[collapsed=true,unused_option=123]\r\033[0K"}
+
+        it 'provides section options when set' do
+          trace = "#{option_section_start}hello#{section_end}"
+          expect(convert_json(trace)).to eq([
+            {
+              offset: 0,
+              content: [{ text: 'hello' }],
+              section: 'prepare-script',
+              section_header: true,
+              section_options: {
+                'collapsed' => 'true',
+                'unused_option' => '123'
+              }
+            },
+            {
+              offset: 83,
+              content: [],
+              section: 'prepare-script',
+              section_duration: '01:03'
+            }
+          ])
+        end
+      end
     end
 
     describe 'incremental updates' do
@@ -339,7 +365,7 @@ RSpec.describe Gitlab::Ci::Ansi2json do
 
       context 'with split word' do
         let(:pre_text) { "\e[1mHello " }
-        let(:text) { "World" }
+        let(:text) { 'World' }
 
         let(:lines) do
           [
@@ -355,7 +381,7 @@ RSpec.describe Gitlab::Ci::Ansi2json do
 
       context 'with split word on second line' do
         let(:pre_text) { "Good\nmorning " }
-        let(:text) { "World" }
+        let(:text) { 'World' }
 
         let(:lines) do
           [
@@ -514,7 +540,7 @@ RSpec.describe Gitlab::Ci::Ansi2json do
     end
 
     describe 'trucates' do
-      let(:text) { "Hello World" }
+      let(:text) { 'Hello World' }
       let(:stream) { StringIO.new(text) }
       let(:subject) { described_class.convert(stream) }
 
@@ -522,11 +548,11 @@ RSpec.describe Gitlab::Ci::Ansi2json do
         stream.seek(3, IO::SEEK_SET)
       end
 
-      it "returns truncated output" do
+      it 'returns truncated output' do
         expect(subject.truncated).to be_truthy
       end
 
-      it "does not append output" do
+      it 'does not append output' do
         expect(subject.append).to be_falsey
       end
     end

spec/lib/gitlab/regex_spec.rb

@@ -99,6 +99,36 @@ RSpec.describe Gitlab::Regex do
     it { is_expected.not_to match('foo-') }
   end
 
+  describe '.build_trace_section_regex' do
+    subject { described_class.build_trace_section_regex }
+
+    context 'without options' do
+      example = "section_start:1600445393032:NAME\r\033\[0K"
+
+      it { is_expected.to match(example) }
+      it { is_expected.to match("section_end:12345678:aBcDeFg1234\r\033\[0K") }
+      it { is_expected.to match("section_start:0:sect_for_alpha-v1.0\r\033\[0K") }
+      it { is_expected.not_to match("section_start:section:0\r\033\[0K") }
+      it { is_expected.not_to match("section_:1600445393032:NAME\r\033\[0K") }
+      it { is_expected.not_to match(example.upcase) }
+    end
+
+    context 'with options' do
+      it { is_expected.to match("section_start:1600445393032:NAME[collapsed=true]\r\033\[0K") }
+      it { is_expected.to match("section_start:1600445393032:NAME[collapsed=true, example_option=false]\r\033\[0K") }
+      it { is_expected.to match("section_start:1600445393032:NAME[collapsed=true,example_option=false]\r\033\[0K") }
+      it { is_expected.to match("section_start:1600445393032:NAME[numeric_option=1234567]\r\033\[0K") }
+      # Without splitting the regex in one for start and one for end,
+      # this is possible, however, it is ignored for section_end.
+      it { is_expected.to match("section_end:1600445393032:NAME[collapsed=true]\r\033\[0K") }
+      it { is_expected.not_to match("section_start:1600445393032:NAME[collapsed=[]]]\r\033\[0K") }
+      it { is_expected.not_to match("section_start:1600445393032:NAME[collapsed = true]\r\033\[0K") }
+      it { is_expected.not_to match("section_start:1600445393032:NAME[collapsed = true, example_option=false]\r\033\[0K") }
+      it { is_expected.not_to match("section_start:1600445393032:NAME[collapsed=true,  example_option=false]\r\033\[0K") }
+      it { is_expected.not_to match("section_start:1600445393032:NAME[]\r\033\[0K") }
+    end
+  end
+
   describe '.container_repository_name_regex' do
     subject { described_class.container_repository_name_regex }