Merge branch 'mc/feature/flatten-ci-scripts' into 'master'

Flatten CI scripts

Closes #23005

See merge request gitlab-org/gitlab!18849

changelogs/unreleased/mc-feature-flatten-ci-scripts.yml

+---
+title: Add support for YAML anchors in CI scripts.
+merge_request: 18849
+author:
+type: changed

doc/ci/yaml/README.md

@@ -181,6 +181,25 @@ that the YAML parser knows to interpret the whole thing as a string rather than
 a "key: value" pair. Be careful when using special characters:
 `:`, `{`, `}`, `[`, `]`, `,`, `&`, `*`, `#`, `?`, `|`, `-`, `<`, `>`, `=`, `!`, `%`, `@`, `` ` ``.
 
+#### YAML anchors for `script`
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/23005) in GitLab 12.5.
+
+You can use [YAML anchors](#anchors) with scripts, which makes it possible to
+include a predefined list of commands in multiple jobs.
+
+Example:
+
+```yaml
+.something: &something
+- echo 'something'
+
+job_name:
+  script:
+    - *something
+    - echo 'this is the script'
+```
+
 ### `image`
 
 Used to specify [a Docker image](../docker/using_docker_images.md#what-is-an-image) to use for the job.
@@ -284,6 +303,33 @@ job:
     - execute this after my script
 ```
 
+#### YAML anchors for `before_script` and `after_script`
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/23005) in GitLab 12.5.
+
+You can use [YAML anchors](#anchors) with `before_script` and `after_script`,
+which makes it possible to include a predefined list of commands in multiple
+jobs.
+
+Example:
+
+```yaml
+.something_before: &something_before
+- echo 'something before'
+
+.something_after: &something_after
+- echo 'something after'
+
+
+job_name:
+  before_script:
+    - *something_before
+  script:
+    - echo 'this is the script'
+  after_script:
+    - *something_after
+```
+
 ### `stages`
 
 `stages` is used to define stages that can be used by jobs and is defined

ee/spec/lib/gitlab/web_ide/config/entry/global_spec.rb

@@ -109,7 +109,7 @@ describe Gitlab::WebIde::Config::Entry::Global do
       describe '#errors' do
         it 'reports errors about missing script' do
           expect(global.errors)
-            .to include "terminal:before_script config should be an array of strings"
+            .to include "terminal:before_script config should be an array containing strings and arrays of strings"
         end
       end
     end

lib/gitlab/ci/config/entry/commands.rb

@@ -11,11 +11,11 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
 
           validations do
-            validates :config, array_of_strings_or_string: true
+            validates :config, string_or_nested_array_of_strings: true
           end
 
           def value
-            Array(@config)
+            Array(@config).flatten(1)
           end
         end
       end

lib/gitlab/ci/config/entry/script.rb

@@ -11,7 +11,11 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
 
           validations do
-            validates :config, array_of_strings: true
+            validates :config, nested_array_of_strings: true
+          end
+
+          def value
+            config.flatten(1)
           end
         end
       end

lib/gitlab/config/entry/validators.rb

@@ -228,6 +228,34 @@ module Gitlab
           end
         end
 
+        class NestedArrayOfStringsValidator < ArrayOfStringsOrStringValidator
+          def validate_each(record, attribute, value)
+            unless validate_nested_array_of_strings(value)
+              record.errors.add(attribute, 'should be an array containing strings and arrays of strings')
+            end
+          end
+
+          private
+
+          def validate_nested_array_of_strings(values)
+            values.is_a?(Array) && values.all? { |element| validate_array_of_strings_or_string(element) }
+          end
+        end
+
+        class StringOrNestedArrayOfStringsValidator < NestedArrayOfStringsValidator
+          def validate_each(record, attribute, value)
+            unless validate_string_or_nested_array_of_strings(value)
+              record.errors.add(attribute, 'should be a string or an array containing strings and arrays of strings')
+            end
+          end
+
+          private
+
+          def validate_string_or_nested_array_of_strings(values)
+            validate_string(values) || validate_nested_array_of_strings(values)
+          end
+        end
+
         class TypeValidator < ActiveModel::EachValidator
           def validate_each(record, attribute, value)
             type = options[:with]

spec/lib/gitlab/ci/config/entry/commands_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 describe Gitlab::Ci::Config::Entry::Commands do
   let(:entry) { described_class.new(config) }
 
-  context 'when entry config value is an array' do
+  context 'when entry config value is an array of strings' do
     let(:config) { %w(ls pwd) }
 
     describe '#value' do
@@ -37,13 +37,74 @@ describe Gitlab::Ci::Config::Entry::Commands do
     end
   end
 
-  context 'when entry value is not valid' do
+  context 'when entry config value is array of arrays of strings' do
+    let(:config) { [['ls'], ['pwd', 'echo 1']] }
+
+    describe '#value' do
+      it 'returns array of strings' do
+        expect(entry.value).to eq ['ls', 'pwd', 'echo 1']
+      end
+    end
+
+    describe '#errors' do
+      it 'does not append errors' do
+        expect(entry.errors).to be_empty
+      end
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+  end
+
+  context 'when entry config value is array of strings and arrays of strings' do
+    let(:config) { ['ls', ['pwd', 'echo 1']] }
+
+    describe '#value' do
+      it 'returns array of strings' do
+        expect(entry.value).to eq ['ls', 'pwd', 'echo 1']
+      end
+    end
+
+    describe '#errors' do
+      it 'does not append errors' do
+        expect(entry.errors).to be_empty
+      end
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+  end
+
+  context 'when entry value is integer' do
     let(:config) { 1 }
 
     describe '#errors' do
       it 'saves errors' do
         expect(entry.errors)
-          .to include 'commands config should be an array of strings or a string'
+          .to include 'commands config should be a string or an array containing strings and arrays of strings'
+      end
+    end
+  end
+
+  context 'when entry value is multi-level nested array' do
+    let(:config) { [['ls', ['echo 1']], 'pwd'] }
+
+    describe '#errors' do
+      it 'saves errors' do
+        expect(entry.errors)
+          .to include 'commands config should be a string or an array containing strings and arrays of strings'
+      end
+    end
+
+    describe '#valid?' do
+      it 'is not valid' do
+        expect(entry).not_to be_valid
       end
     end
   end

spec/lib/gitlab/ci/config/entry/root_spec.rb

@@ -298,7 +298,7 @@ describe Gitlab::Ci::Config::Entry::Root do
       describe '#errors' do
         it 'reports errors from child nodes' do
           expect(root.errors)
-            .to include 'before_script config should be an array of strings'
+            .to include 'before_script config should be an array containing strings and arrays of strings'
         end
       end
     end

spec/lib/gitlab/ci/config/entry/script_spec.rb

@@ -6,7 +6,7 @@ describe Gitlab::Ci::Config::Entry::Script do
   let(:entry) { described_class.new(config) }
 
   describe 'validations' do
-    context 'when entry config value is correct' do
+    context 'when entry config value is array of strings' do
       let(:config) { %w(ls pwd) }
 
       describe '#value' do
@@ -28,13 +28,74 @@ describe Gitlab::Ci::Config::Entry::Script do
       end
     end
 
-    context 'when entry value is not correct' do
+    context 'when entry config value is array of arrays of strings' do
+      let(:config) { [['ls'], ['pwd', 'echo 1']] }
+
+      describe '#value' do
+        it 'returns array of strings' do
+          expect(entry.value).to eq ['ls', 'pwd', 'echo 1']
+        end
+      end
+
+      describe '#errors' do
+        it 'does not append errors' do
+          expect(entry.errors).to be_empty
+        end
+      end
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+    end
+
+    context 'when entry config value is array containing strings and arrays of strings' do
+      let(:config) { ['ls', ['pwd', 'echo 1']] }
+
+      describe '#value' do
+        it 'returns array of strings' do
+          expect(entry.value).to eq ['ls', 'pwd', 'echo 1']
+        end
+      end
+
+      describe '#errors' do
+        it 'does not append errors' do
+          expect(entry.errors).to be_empty
+        end
+      end
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+    end
+
+    context 'when entry value is string' do
       let(:config) { 'ls' }
 
       describe '#errors' do
         it 'saves errors' do
           expect(entry.errors)
-            .to include 'script config should be an array of strings'
+            .to include 'script config should be an array containing strings and arrays of strings'
+        end
+      end
+
+      describe '#valid?' do
+        it 'is not valid' do
+          expect(entry).not_to be_valid
+        end
+      end
+    end
+
+    context 'when entry value is multi-level nested array' do
+      let(:config) { [['ls', ['echo 1']], 'pwd'] }
+
+      describe '#errors' do
+        it 'saves errors' do
+          expect(entry.errors)
+            .to include 'script config should be an array containing strings and arrays of strings'
         end
       end
 

spec/lib/gitlab/ci/pipeline/chain/validate/config_spec.rb

@@ -94,7 +94,7 @@ describe Gitlab::Ci::Pipeline::Chain::Validate::Config do
 
     it 'appends configuration validation errors to pipeline errors' do
       expect(pipeline.errors.to_a)
-        .to include "jobs:rspec:before_script config should be an array of strings"
+        .to include "jobs:rspec:before_script config should be an array containing strings and arrays of strings"
     end
 
     it 'breaks the chain' do

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -330,7 +330,7 @@ module Gitlab
               }
             end
 
-            it "return commands with scripts concencaced" do
+            it "return commands with scripts concatenated" do
               expect(subject[:options][:before_script]).to eq(["global script"])
             end
           end
@@ -343,7 +343,7 @@ module Gitlab
               }
             end
 
-            it "return commands with scripts concencaced" do
+            it "return commands with scripts concatenated" do
               expect(subject[:options][:before_script]).to eq(["global script"])
             end
           end
@@ -356,21 +356,48 @@ module Gitlab
               }
             end
 
-            it "return commands with scripts concencaced" do
+            it "return commands with scripts concatenated" do
               expect(subject[:options][:before_script]).to eq(["local script"])
             end
           end
+
+          context 'when script is array of arrays of strings' do
+            let(:config) do
+              {
+                before_script: [["global script", "echo 1"], ["ls"], "pwd"],
+                test: { script: ["script"] }
+              }
+            end
+
+            it "return commands with scripts concatenated" do
+              expect(subject[:options][:before_script]).to eq(["global script", "echo 1", "ls", "pwd"])
+            end
+          end
         end
 
         describe "script" do
-          let(:config) do
-            {
-              test: { script: ["script"] }
-            }
+          context 'when script is array of strings' do
+            let(:config) do
+              {
+                test: { script: ["script"] }
+              }
+            end
+
+            it "return commands with scripts concatenated" do
+              expect(subject[:options][:script]).to eq(["script"])
+            end
           end
 
-          it "return commands with scripts concencaced" do
-            expect(subject[:options][:script]).to eq(["script"])
+          context 'when script is array of arrays of strings' do
+            let(:config) do
+              {
+                test: { script: [["script"], ["echo 1"], "ls"] }
+              }
+            end
+
+            it "return commands with scripts concatenated" do
+              expect(subject[:options][:script]).to eq(["script", "echo 1", "ls"])
+            end
           end
         end
 
@@ -413,6 +440,19 @@ module Gitlab
               expect(subject[:options][:after_script]).to eq(["local after_script"])
             end
           end
+
+          context 'when script is array of arrays of strings' do
+            let(:config) do
+              {
+                after_script: [["global script", "echo 1"], ["ls"], "pwd"],
+                test: { script: ["script"] }
+              }
+            end
+
+            it "return after_script in options" do
+              expect(subject[:options][:after_script]).to eq(["global script", "echo 1", "ls", "pwd"])
+            end
+          end
         end
       end
 
@@ -1536,28 +1576,42 @@ module Gitlab
           config = YAML.dump({ before_script: "bundle update", rspec: { script: "test" } })
           expect do
             Gitlab::Ci::YamlProcessor.new(config)
-          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "before_script config should be an array of strings")
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "before_script config should be an array containing strings and arrays of strings")
         end
 
         it "returns errors if job before_script parameter is not an array of strings" do
           config = YAML.dump({ rspec: { script: "test", before_script: [10, "test"] } })
           expect do
             Gitlab::Ci::YamlProcessor.new(config)
-          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec:before_script config should be an array of strings")
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec:before_script config should be an array containing strings and arrays of strings")
+        end
+
+        it "returns errors if job before_script parameter is multi-level nested array of strings" do
+          config = YAML.dump({ rspec: { script: "test", before_script: [["ls", ["pwd"]], "test"] } })
+          expect do
+            Gitlab::Ci::YamlProcessor.new(config)
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec:before_script config should be an array containing strings and arrays of strings")
         end
 
         it "returns errors if after_script parameter is invalid" do
           config = YAML.dump({ after_script: "bundle update", rspec: { script: "test" } })
           expect do
             Gitlab::Ci::YamlProcessor.new(config)
-          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "after_script config should be an array of strings")
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "after_script config should be an array containing strings and arrays of strings")
         end
 
         it "returns errors if job after_script parameter is not an array of strings" do
           config = YAML.dump({ rspec: { script: "test", after_script: [10, "test"] } })
           expect do
             Gitlab::Ci::YamlProcessor.new(config)
-          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec:after_script config should be an array of strings")
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec:after_script config should be an array containing strings and arrays of strings")
+        end
+
+        it "returns errors if job after_script parameter is multi-level nested array of strings" do
+          config = YAML.dump({ rspec: { script: "test", after_script: [["ls", ["pwd"]], "test"] } })
+          expect do
+            Gitlab::Ci::YamlProcessor.new(config)
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec:after_script config should be an array containing strings and arrays of strings")
         end
 
         it "returns errors if image parameter is invalid" do