Merge branch '53134-multiple-extendes-for-a-job' into 'master'

Add multiple extends support

Closes #53134

See merge request gitlab-org/gitlab-ce!26801

changelogs/unreleased/53134-multiple-extendes-for-a-job.yml

+---
+title: Add support for multiple job parents in GitLab CI YAML.
+merge_request: 26801
+author: Wolphin (Nikita)
+type: added

doc/ci/yaml/README.md

@@ -108,7 +108,7 @@ The following table lists available parameters for jobs:
 | [`parallel`](#parallel)                            | How many instances of a job should be run in parallel.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | [`trigger`](#trigger-premium)                      | Defines a downstream pipeline trigger.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | [`include`](#include)                              | Allows this job to include external YAML files. Also available: `include:local`, `include:file`, `include:template`, and `include:remote`.                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| [`extends`](#extends)                              | Configuration entry that this job is going to inherit from.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| [`extends`](#extends)                              | Configuration entries that this job is going to inherit from.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | [`pages`](#pages)                                  | Upload the result of a job to use with GitLab Pages.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 | [`variables`](#variables)                          | Define job variables on a job level.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 
@@ -2117,7 +2117,7 @@ docker-test:
 
 > Introduced in GitLab 11.3.
 
-`extends` defines an entry name that a job that uses `extends` is going to
+`extends` defines entry names that a job that uses `extends` is going to
 inherit from.
 
 It is an alternative to using [YAML anchors](#anchors) and is a little
@@ -2194,6 +2194,46 @@ spinach:
   script: rake spinach
 ```
 
+It's also possible to use multiple parents for `extends`.
+The algorithm used for merge is "closest scope wins", so keys
+from the last member will always shadow anything defined on other levels.
+For example:
+
+```yaml
+.only-important:
+  only:
+    - master
+    - stable
+  tags:
+    - production
+
+.in-docker:
+  tags:
+    - docker
+  image: alpine
+
+rspec:
+  extends:
+    - .only-important
+    - .in-docker
+  script:
+    - rake rspec
+```
+
+This results in the following `rspec` job:
+
+```yaml
+rspec:
+  only:
+    - master
+    - stable
+  tags:
+    - docker
+  image: alpine
+  script:
+    - rake rspec
+```
+
 ### Using `extends` and `include` together
 
 `extends` works across configuration files combined with `include`.

lib/gitlab/ci/config/entry/job.rb

@@ -34,7 +34,7 @@ module Gitlab
                              message: 'should be on_success, on_failure, ' \
                                       'always, manual or delayed' }
               validates :dependencies, array_of_strings: true
-              validates :extends, type: String
+              validates :extends, array_of_strings_or_string: true
             end
 
             validates :start_in, duration: { limit: '1 day' }, if: :delayed?

lib/gitlab/ci/config/extendable/entry.rb

@@ -5,6 +5,8 @@ module Gitlab
     class Config
       class Extendable
         class Entry
+          include Gitlab::Utils::StrongMemoize
+
           InvalidExtensionError = Class.new(Extendable::ExtensionError)
           CircularDependencyError = Class.new(Extendable::ExtensionError)
           NestingTooDeepError = Class.new(Extendable::ExtensionError)
@@ -28,34 +30,46 @@ module Gitlab
           end
 
           def value
-            @value ||= @context.fetch(@key)
+            strong_memoize(:value) do
+              @context.fetch(@key)
+            end
           end
 
-          def base_hash!
-            @base ||= Extendable::Entry
-              .new(extends_key, @context, self)
-              .extend!
+          def base_hashes!
+            strong_memoize(:base_hashes) do
+              extends_keys.map do |key|
+                Extendable::Entry
+                  .new(key, @context, self)
+                  .extend!
+              end
+            end
           end
 
-          def extends_key
-            value.fetch(:extends).to_s.to_sym if extensible?
+          def extends_keys
+            strong_memoize(:extends_keys) do
+              next unless extensible?
+
+              Array(value.fetch(:extends)).map(&:to_s).map(&:to_sym)
+            end
           end
 
           def ancestors
-            @ancestors ||= Array(@parent&.ancestors) + Array(@parent&.key)
+            strong_memoize(:ancestors) do
+              Array(@parent&.ancestors) + Array(@parent&.key)
+            end
           end
 
           def extend!
             return value unless extensible?
 
-            if unknown_extension?
+            if unknown_extensions.any?
               raise Entry::InvalidExtensionError,
-                    "#{key}: unknown key in `extends`"
+                    "#{key}: unknown keys in `extends` (#{show_keys(unknown_extensions)})"
             end
 
-            if invalid_base?
+            if invalid_bases.any?
               raise Entry::InvalidExtensionError,
-                    "#{key}: invalid base hash in `extends`"
+                    "#{key}: invalid base hashes in `extends` (#{show_keys(invalid_bases)})"
             end
 
             if nesting_too_deep?
@@ -68,11 +82,18 @@ module Gitlab
                     "#{key}: circular dependency detected in `extends`"
             end
 
-            @context[key] = base_hash!.deep_merge(value)
+            merged = {}
+            base_hashes!.each { |h| merged.deep_merge!(h) }
+
+            @context[key] = merged.deep_merge!(value)
           end
 
           private
 
+          def show_keys(keys)
+            keys.join(', ')
+          end
+
           def nesting_too_deep?
             ancestors.count > MAX_NESTING_LEVELS
           end
@@ -81,12 +102,16 @@ module Gitlab
             ancestors.include?(key)
           end
 
-          def unknown_extension?
-            !@context.key?(extends_key)
+          def unknown_extensions
+            strong_memoize(:unknown_extensions) do
+              extends_keys.reject { |key| @context.key?(key) }
+            end
           end
 
-          def invalid_base?
-            !@context[extends_key].is_a?(Hash)
+          def invalid_bases
+            strong_memoize(:invalid_bases) do
+              extends_keys.reject { |key| @context[key].is_a?(Hash) }
+            end
           end
         end
       end

spec/lib/gitlab/ci/config/entry/job_spec.rb

@@ -94,7 +94,7 @@ describe Gitlab::Ci::Config::Entry::Job do
 
         it 'returns error about wrong value type' do
           expect(entry).not_to be_valid
-          expect(entry.errors).to include "job extends should be a string"
+          expect(entry.errors).to include "job extends should be an array of strings or a string"
         end
       end
 

spec/lib/gitlab/ci/config/extendable/entry_spec.rb

@@ -44,12 +44,12 @@ describe Gitlab::Ci::Config::Extendable::Entry do
     end
   end
 
-  describe '#extends_key' do
+  describe '#extends_keys' do
     context 'when entry is extensible' do
       it 'returns symbolized extends key value' do
         entry = described_class.new(:test, test: { extends: 'something' })
 
-        expect(entry.extends_key).to eq :something
+        expect(entry.extends_keys).to eq [:something]
       end
     end
 
@@ -57,7 +57,7 @@ describe Gitlab::Ci::Config::Extendable::Entry do
       it 'returns nil' do
         entry = described_class.new(:test, test: 'something')
 
-        expect(entry.extends_key).to be_nil
+        expect(entry.extends_keys).to be_nil
       end
     end
   end
@@ -76,7 +76,7 @@ describe Gitlab::Ci::Config::Extendable::Entry do
     end
   end
 
-  describe '#base_hash!' do
+  describe '#base_hashes!' do
     subject { described_class.new(:test, hash) }
 
     context 'when base hash is not extensible' do
@@ -87,8 +87,8 @@ describe Gitlab::Ci::Config::Extendable::Entry do
         }
       end
 
-      it 'returns unchanged base hash' do
-        expect(subject.base_hash!).to eq(script: 'rspec')
+      it 'returns unchanged base hashes' do
+        expect(subject.base_hashes!).to eq([{ script: 'rspec' }])
       end
     end
 
@@ -101,12 +101,12 @@ describe Gitlab::Ci::Config::Extendable::Entry do
         }
       end
 
-      it 'extends the base hash first' do
-        expect(subject.base_hash!).to eq(extends: 'first', script: 'rspec')
+      it 'extends the base hashes first' do
+        expect(subject.base_hashes!).to eq([{ extends: 'first', script: 'rspec' }])
       end
 
       it 'mutates original context' do
-        subject.base_hash!
+        subject.base_hashes!
 
         expect(hash.fetch(:second)).to eq(extends: 'first', script: 'rspec')
       end
@@ -171,6 +171,34 @@ describe Gitlab::Ci::Config::Extendable::Entry do
       end
     end
 
+    context 'when extending multiple hashes correctly' do
+      let(:hash) do
+        {
+          first: { script: 'my value', image: 'ubuntu' },
+          second: { image: 'alpine' },
+          test: { extends: %w(first second) }
+        }
+      end
+
+      let(:result) do
+        {
+          first: { script: 'my value', image: 'ubuntu' },
+          second: { image: 'alpine' },
+          test: { extends: %w(first second), script: 'my value', image: 'alpine' }
+        }
+      end
+
+      it 'returns extended part of the hash' do
+        expect(subject.extend!).to eq result[:test]
+      end
+
+      it 'mutates original context' do
+        subject.extend!
+
+        expect(hash).to eq result
+      end
+    end
+
     context 'when hash is not extensible' do
       let(:hash) do
         {

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -1470,7 +1470,7 @@ module Gitlab
 
           expect { Gitlab::Ci::YamlProcessor.new(config) }
             .to raise_error(Gitlab::Ci::YamlProcessor::ValidationError,
-                            'rspec: unknown key in `extends`')
+                            'rspec: unknown keys in `extends` (something)')
         end
       end