Merge branch '270957-parallel-trigger' into 'master'

Add trigger support for matrix jobs

See merge request gitlab-org/gitlab!55348

changelogs/unreleased/270957-parallel-trigger.yml

+---
+title: Add trigger support for matrix jobs
+merge_request: 55348
+author:
+type: added

doc/ci/yaml/README.md

@@ -3662,6 +3662,40 @@ deploystacks:
       - PROVIDER: [aws, ovh, gcp, vultr]
 ```
 
+##### Parallel `matrix` trigger jobs
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/270957) in GitLab 13.10.
+
+Use `matrix:` to run a [trigger](#trigger) job multiple times in parallel in a single pipeline,
+but with different variable values for each instance of the job.
+
+```yaml
+deploystacks:
+  stage: deploy
+  trigger:
+    include: path/to/child-pipeline.yml
+  parallel:
+    matrix:
+      - PROVIDER: aws
+        STACK: [monitoring, app1]
+      - PROVIDER: ovh
+        STACK: [monitoring, backup]
+      - PROVIDER: [gcp, vultr]
+        STACK: [data]
+```
+
+This example generates 6 parallel `deploystacks` trigger jobs, each with different values
+for `PROVIDER` and `STACK`, and they create 6 different child pipelines with those variables.
+
+```plaintext
+deploystacks: [aws, monitoring]
+deploystacks: [aws, app1]
+deploystacks: [ovh, monitoring]
+deploystacks: [ovh, backup]
+deploystacks: [gcp, data]
+deploystacks: [vultr, data]
+```
+
 ### `trigger`
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8997) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8.

lib/gitlab/ci/config/entry/bridge.rb

@@ -12,7 +12,7 @@ module Gitlab
           include ::Gitlab::Ci::Config::Entry::Processable
 
           ALLOWED_WHEN = %w[on_success on_failure always manual].freeze
-          ALLOWED_KEYS = %i[trigger].freeze
+          ALLOWED_KEYS = %i[trigger parallel].freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS
@@ -48,7 +48,12 @@ module Gitlab
             inherit: false,
             metadata: { allowed_needs: %i[job bridge] }
 
-          attributes :when, :allow_failure
+          entry :parallel, Entry::Product::Parallel,
+            description: 'Parallel configuration for this job.',
+            inherit: false,
+            metadata: { allowed_strategies: %i(matrix) }
+
+          attributes :when, :allow_failure, :parallel
 
           def self.matching?(name, config)
             !name.to_s.start_with?('.') &&
@@ -66,7 +71,8 @@ module Gitlab
               needs: (needs_value if needs_defined?),
               ignore: ignored?,
               when: self.when,
-              scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage
+              scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage,
+              parallel: has_parallel? ? parallel_value : nil
             ).compact
           end
 

lib/gitlab/ci/config/entry/product/parallel.rb

@@ -22,6 +22,13 @@ module Gitlab
                                                    greater_than_or_equal_to: 2,
                                                    less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT },
                                    allow_nil: true
+
+                validate do
+                  next unless opt(:allowed_strategies)
+                  next if opt(:allowed_strategies).include?(:numeric)
+
+                  errors.add(:config, 'cannot use "parallel: <number>".')
+                end
               end
 
               def value
@@ -38,6 +45,13 @@ module Gitlab
               validations do
                 validates :config, allowed_keys: PERMITTED_KEYS
                 validates :config, required_keys: PERMITTED_KEYS
+
+                validate do
+                  next unless opt(:allowed_strategies)
+                  next if opt(:allowed_strategies).include?(:matrix)
+
+                  errors.add(:config, 'cannot use "parallel: matrix".')
+                end
               end
 
               entry :matrix, Entry::Product::Matrix,

spec/lib/gitlab/ci/config/entry/bridge_spec.rb

@@ -244,6 +244,52 @@ RSpec.describe Gitlab::Ci::Config::Entry::Bridge do
         end
       end
     end
+
+    context 'when bridge config contains parallel' do
+      let(:config) { { trigger: 'some/project', parallel: parallel_config } }
+
+      context 'when parallel config is a number' do
+        let(:parallel_config) { 2 }
+
+        describe '#valid?' do
+          it { is_expected.not_to be_valid }
+        end
+
+        describe '#errors' do
+          it 'returns an error message' do
+            expect(subject.errors)
+              .to include(/cannot use "parallel: <number>"/)
+          end
+        end
+      end
+
+      context 'when parallel config is a matrix' do
+        let(:parallel_config) do
+          { matrix: [{ PROVIDER: 'aws', STACK: %w[monitoring app1] },
+                     { PROVIDER: 'gcp', STACK: %w[data] }] }
+        end
+
+        describe '#valid?' do
+          it { is_expected.to be_valid }
+        end
+
+        describe '#value' do
+          it 'is returns a bridge job configuration' do
+            expect(subject.value).to eq(
+              name: :my_bridge,
+              trigger: { project: 'some/project' },
+              ignore: false,
+              stage: 'test',
+              only: { refs: %w[branches tags] },
+              parallel: { matrix: [{ 'PROVIDER' => ['aws'], 'STACK' => %w(monitoring app1) },
+                                   { 'PROVIDER' => ['gcp'], 'STACK' => %w(data) }] },
+              variables: {},
+              scheduling_type: :stage
+            )
+          end
+        end
+      end
+    end
   end
 
   describe '#manual_action?' do

spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb

@@ -4,21 +4,23 @@ require 'fast_spec_helper'
 require_dependency 'active_model'
 
 RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do
-  subject(:parallel) { described_class.new(config) }
+  let(:metadata) { {} }
 
-  context 'with invalid config' do
-    shared_examples 'invalid config' do |error_message|
-      describe '#valid?' do
-        it { is_expected.not_to be_valid }
-      end
+  subject(:parallel) { described_class.new(config, **metadata) }
 
-      describe '#errors' do
-        it 'returns error about invalid type' do
-          expect(parallel.errors).to match(a_collection_including(error_message))
-        end
+  shared_examples 'invalid config' do |error_message|
+    describe '#valid?' do
+      it { is_expected.not_to be_valid }
+    end
+
+    describe '#errors' do
+      it 'returns error about invalid type' do
+        expect(parallel.errors).to match(a_collection_including(error_message))
       end
     end
+  end
 
+  context 'with invalid config' do
     context 'when it is not a numeric value' do
       let(:config) { true }
 
@@ -63,6 +65,12 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do
           expect(parallel.value).to match(number: config)
         end
       end
+
+      context 'when :numeric is not allowed' do
+        let(:metadata) { { allowed_strategies: [:matrix] } }
+
+        it_behaves_like 'invalid config', /cannot use "parallel: <number>"/
+      end
     end
   end
 
@@ -89,6 +97,12 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do
           ])
         end
       end
+
+      context 'when :matrix is not allowed' do
+        let(:metadata) { { allowed_strategies: [:numeric] } }
+
+        it_behaves_like 'invalid config', /cannot use "parallel: matrix"/
+      end
     end
   end
 end

spec/services/ci/create_pipeline_service/parallel_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Ci::CreatePipelineService do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user)    { project.owner }
+
+  let(:service)  { described_class.new(project, user, { ref: 'master' }) }
+  let(:pipeline) { service.execute(:push) }
+
+  before do
+    stub_ci_pipeline_yaml_file(config)
+  end
+
+  context 'job:parallel' do
+    context 'numeric' do
+      let(:config) do
+        <<-EOY
+        job:
+          script: "echo job"
+          parallel: 3
+        EOY
+      end
+
+      it 'creates the pipeline' do
+        expect(pipeline).to be_created_successfully
+      end
+
+      it 'creates 3 jobs' do
+        expect(pipeline.processables.pluck(:name)).to contain_exactly(
+          'job 1/3', 'job 2/3', 'job 3/3'
+        )
+      end
+    end
+
+    context 'matrix' do
+      let(:config) do
+        <<-EOY
+        job:
+          script: "echo job"
+          parallel:
+            matrix:
+              - PROVIDER: ovh
+                STACK: [monitoring, app]
+              - PROVIDER: [gcp, vultr]
+                STACK: [data]
+        EOY
+      end
+
+      it 'creates the pipeline' do
+        expect(pipeline).to be_created_successfully
+      end
+
+      it 'creates 4 builds with the corresponding matrix variables' do
+        expect(pipeline.processables.pluck(:name)).to contain_exactly(
+          'job: [gcp, data]', 'job: [ovh, app]', 'job: [ovh, monitoring]', 'job: [vultr, data]'
+        )
+
+        job1 = find_job('job: [gcp, data]')
+        job2 = find_job('job: [ovh, app]')
+        job3 = find_job('job: [ovh, monitoring]')
+        job4 = find_job('job: [vultr, data]')
+
+        expect(job1.scoped_variables.to_hash).to include('PROVIDER' => 'gcp', 'STACK' => 'data')
+        expect(job2.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'app')
+        expect(job3.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'monitoring')
+        expect(job4.scoped_variables.to_hash).to include('PROVIDER' => 'vultr', 'STACK' => 'data')
+      end
+
+      context 'when a bridge is using parallel:matrix' do
+        let(:config) do
+          <<-EOY
+          job:
+            stage: test
+            script: "echo job"
+
+          deploy:
+            stage: deploy
+            trigger:
+              include: child.yml
+            parallel:
+              matrix:
+                - PROVIDER: ovh
+                  STACK: [monitoring, app]
+                - PROVIDER: [gcp, vultr]
+                  STACK: [data]
+          EOY
+        end
+
+        it 'creates the pipeline' do
+          expect(pipeline).to be_created_successfully
+        end
+
+        it 'creates 1 build and 4 bridges with the corresponding matrix variables' do
+          expect(pipeline.processables.pluck(:name)).to contain_exactly(
+            'job', 'deploy: [gcp, data]', 'deploy: [ovh, app]', 'deploy: [ovh, monitoring]', 'deploy: [vultr, data]'
+          )
+
+          bridge1 = find_job('deploy: [gcp, data]')
+          bridge2 = find_job('deploy: [ovh, app]')
+          bridge3 = find_job('deploy: [ovh, monitoring]')
+          bridge4 = find_job('deploy: [vultr, data]')
+
+          expect(bridge1.scoped_variables.to_hash).to include('PROVIDER' => 'gcp', 'STACK' => 'data')
+          expect(bridge2.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'app')
+          expect(bridge3.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'monitoring')
+          expect(bridge4.scoped_variables.to_hash).to include('PROVIDER' => 'vultr', 'STACK' => 'data')
+        end
+      end
+    end
+  end
+
+  private
+
+  def find_job(name)
+    pipeline.processables.find { |job| job.name == name }
+  end
+end

spec/services/ci/pipeline_processing/shared_processing_service.rb

 # frozen_string_literal: true
 
 RSpec.shared_examples 'Pipeline Processing Service' do
-  let(:user) { create(:user) }
-  let(:project) { create(:project) }
+  let(:project) { create(:project, :repository) }
+  let(:user)    { project.owner }
 
   let(:pipeline) do
     create(:ci_empty_pipeline, ref: 'master', project: project)
   end
 
-  before do
-    stub_ci_pipeline_to_return_yaml_file
-
-    stub_not_protect_default_branch
-
-    project.add_developer(user)
-  end
-
   context 'when simple pipeline is defined' do
     before do
       create_build('linux', stage_idx: 0)
@@ -866,10 +858,74 @@ RSpec.shared_examples 'Pipeline Processing Service' do
     end
   end
 
+  context 'when a bridge job has parallel:matrix config', :sidekiq_inline do
+    let(:parent_config) do
+      <<-EOY
+      test:
+        stage: test
+        script: echo test
+
+      deploy:
+        stage: deploy
+        trigger:
+          include: .child.yml
+        parallel:
+          matrix:
+            - PROVIDER: ovh
+              STACK: [monitoring, app]
+      EOY
+    end
+
+    let(:child_config) do
+      <<-EOY
+      test:
+        stage: test
+        script: echo test
+      EOY
+    end
+
+    let(:pipeline) do
+      Ci::CreatePipelineService.new(project, user, { ref: 'master' }).execute(:push)
+    end
+
+    before do
+      allow_next_instance_of(Repository) do |repository|
+        allow(repository)
+          .to receive(:blob_data_at)
+          .with(an_instance_of(String), '.gitlab-ci.yml')
+          .and_return(parent_config)
+
+        allow(repository)
+          .to receive(:blob_data_at)
+          .with(an_instance_of(String), '.child.yml')
+          .and_return(child_config)
+      end
+    end
+
+    it 'creates pipeline with bridges, then passes the matrix variables to downstream jobs' do
+      expect(all_builds_names).to contain_exactly('test', 'deploy: [ovh, monitoring]', 'deploy: [ovh, app]')
+      expect(all_builds_statuses).to contain_exactly('pending', 'created', 'created')
+
+      succeed_pending
+
+      # bridge jobs directly transition to success
+      expect(all_builds_statuses).to contain_exactly('success', 'success', 'success')
+
+      bridge1 = all_builds.find_by(name: 'deploy: [ovh, monitoring]')
+      bridge2 = all_builds.find_by(name: 'deploy: [ovh, app]')
+
+      downstream_job1 = bridge1.downstream_pipeline.processables.first
+      downstream_job2 = bridge2.downstream_pipeline.processables.first
+
+      expect(downstream_job1.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'monitoring')
+      expect(downstream_job2.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'app')
+    end
+  end
+
   private
 
   def all_builds
-    pipeline.builds.order(:stage_idx, :id)
+    pipeline.processables.order(:stage_idx, :id)
   end
 
   def builds