Merge branch 'feature/gb/kubernetes-only-pipeline-jobs' into 'master'

Check if Kubernetes is required when creating pipeline jobs

Closes #34785

See merge request !13849

app/models/ci/pipeline.rb

@@ -305,6 +305,10 @@ module Ci
       @stage_seeds ||= config_processor.stage_seeds(self)
     end
 
+    def has_kubernetes_active?
+      project.kubernetes_service&.active?
+    end
+
     def has_stage_seeds?
       stage_seeds.any?
     end

app/views/ci/lints/_create.html.haml

@@ -22,10 +22,10 @@
                 %b Tag list:
                 = build[:tag_list].to_a.join(", ")
                 %br
-                %b Refs only:
+                %b Only policy:
                 = @jobs[build[:name].to_sym][:only].to_a.join(", ")
                 %br
-                %b Refs except:
+                %b Except policy:
                 = @jobs[build[:name].to_sym][:except].to_a.join(", ")
                 %br
                 %b Environment:

changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml

+---
+title: Add CI/CD active kubernetes job policy
+merge_request: 13849
+author:
+type: added

doc/ci/yaml/README.md

@@ -427,16 +427,16 @@ a "key: value" pair. Be careful when using special characters:
 are executed in `parallel`. For more info about the use of `stage` please check
 [stages](#stages).
 
-### only and except
+### only and except (simplified)
 
-`only` and `except` are two parameters that set a refs policy to limit when
-jobs are built:
+`only` and `except` are two parameters that set a job policy to limit when
+jobs are created:
 
 1. `only` defines the names of branches and tags for which the job will run.
 2. `except` defines the names of branches and tags for which the job will
     **not** run.
 
-There are a few rules that apply to the usage of refs policy:
+There are a few rules that apply to the usage of job policy:
 
 * `only` and `except` are inclusive. If both `only` and `except` are defined
    in a job specification, the ref is filtered by `only` and `except`.
@@ -497,6 +497,36 @@ job:
 The above example will run `job` for all branches on `gitlab-org/gitlab-ce`,
 except master.
 
+### only and except (complex)
+
+> Introduced in GitLab 10.0
+
+> This an _alpha_ feature, and it it subject to change at any time without
+  prior notice!
+
+Since GitLab 10.0 it is possible to define a more elaborate only/except job
+policy configuration.
+
+GitLab now supports both, simple and complex strategies, so it is possible to
+use an array and a hash configuration scheme.
+
+Two keys are now available: `refs` and `kubernetes`. Refs strategy equals to
+simplified only/except configuration, whereas kubernetes strategy accepts only
+`active` keyword.
+
+See the example below. Job is going to be created only when pipeline has been
+scheduled or runs for a `master` branch, and only if kubernetes service is
+active in the project.
+
+```yaml
+job:
+  only:
+    refs:
+      - master
+      - schedules
+    kubernetes: active
+```
+
 ### Job variables
 
 It is possible to define job variables using a `variables` keyword on a job

lib/ci/gitlab_ci_yaml_processor.rb

@@ -20,24 +20,6 @@ module Ci
       raise ValidationError, e.message
     end
 
-    def jobs_for_ref(ref, tag = false, source = nil)
-      @jobs.select do |_, job|
-        process?(job[:only], job[:except], ref, tag, source)
-      end
-    end
-
-    def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
-      jobs_for_ref(ref, tag, source).select do |_, job|
-        job[:stage] == stage
-      end
-    end
-
-    def builds_for_ref(ref, tag = false, source = nil)
-      jobs_for_ref(ref, tag, source).map do |name, _|
-        build_attributes(name)
-      end
-    end
-
     def builds_for_stage_and_ref(stage, ref, tag = false, source = nil)
       jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _|
         build_attributes(name)
@@ -52,8 +34,7 @@ module Ci
 
     def stage_seeds(pipeline)
       seeds = @stages.uniq.map do |stage|
-        builds = builds_for_stage_and_ref(
-          stage, pipeline.ref, pipeline.tag?, pipeline.source)
+        builds = pipeline_stage_builds(stage, pipeline)
 
         Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
       end
@@ -101,6 +82,34 @@ module Ci
 
     private
 
+    def pipeline_stage_builds(stage, pipeline)
+      builds = builds_for_stage_and_ref(
+        stage, pipeline.ref, pipeline.tag?, pipeline.source)
+
+      builds.select do |build|
+        job = @jobs[build.fetch(:name).to_sym]
+        has_kubernetes = pipeline.has_kubernetes_active?
+        only_kubernetes = job.dig(:only, :kubernetes)
+        except_kubernetes = job.dig(:except, :kubernetes)
+
+        [!only_kubernetes && !except_kubernetes,
+         only_kubernetes && has_kubernetes,
+         except_kubernetes && !has_kubernetes].any?
+      end
+    end
+
+    def jobs_for_ref(ref, tag = false, source = nil)
+      @jobs.select do |_, job|
+        process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source)
+      end
+    end
+
+    def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
+      jobs_for_ref(ref, tag, source).select do |_, job|
+        job[:stage] == stage
+      end
+    end
+
     def initial_parsing
       ##
       # Global config

lib/gitlab/ci/config/entry/policy.rb

@@ -7,6 +7,7 @@ module Gitlab
         #
         class Policy < Simplifiable
           strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) }
+          strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) }
 
           class RefsPolicy < Entry::Node
             include Entry::Validatable
@@ -14,6 +15,27 @@ module Gitlab
             validations do
               validates :config, array_of_strings_or_regexps: true
             end
+
+            def value
+              { refs: @config }
+            end
+          end
+
+          class ComplexPolicy < Entry::Node
+            include Entry::Validatable
+            include Entry::Attributable
+
+            attributes :refs, :kubernetes
+
+            validations do
+              validates :config, presence: true
+              validates :config, allowed_keys: %i[refs kubernetes]
+
+              with_options allow_nil: true do
+                validates :refs, array_of_strings_or_regexps: true
+                validates :kubernetes, allowed_values: %w[active]
+              end
+            end
           end
 
           class UnknownStrategy < Entry::Node

lib/gitlab/ci/config/entry/validators.rb

@@ -14,6 +14,14 @@ module Gitlab
             end
           end
 
+          class AllowedValuesValidator < ActiveModel::EachValidator
+            def validate_each(record, attribute, value)
+              unless options[:in].include?(value.to_s)
+                record.errors.add(attribute, "unknown value: #{value}")
+              end
+            end
+          end
+
           class ArrayOfStringsValidator < ActiveModel::EachValidator
             include LegacyValidationHelpers
 

spec/lib/ci/gitlab_ci_yaml_processor_spec.rb

@@ -164,9 +164,46 @@ module Ci
           expect(seeds.first.builds.dig(0, :name)).to eq 'spinach'
         end
       end
+
+      context 'when kubernetes policy is specified' do
+        let(:pipeline) { create(:ci_empty_pipeline) }
+
+        let(:config) do
+          YAML.dump(
+            spinach: { stage: 'test', script: 'spinach' },
+            production: {
+              stage: 'deploy',
+              script: 'cap',
+              only: { kubernetes: 'active' }
+            }
+          )
+        end
+
+        context 'when kubernetes is active' do
+          let(:project) { create(:kubernetes_project) }
+          let(:pipeline) { create(:ci_empty_pipeline, project: project) }
+
+          it 'returns seeds for kubernetes dependent job' do
+            seeds = subject.stage_seeds(pipeline)
+
+            expect(seeds.size).to eq 2
+            expect(seeds.first.builds.dig(0, :name)).to eq 'spinach'
+            expect(seeds.second.builds.dig(0, :name)).to eq 'production'
+          end
+        end
+
+        context 'when kubernetes is not active' do
+          it 'does not return seeds for kubernetes dependent job' do
+            seeds = subject.stage_seeds(pipeline)
+
+            expect(seeds.size).to eq 1
+            expect(seeds.first.builds.dig(0, :name)).to eq 'spinach'
+          end
+        end
+      end
     end
 
-    describe "#builds_for_ref" do
+    describe "#builds_for_stage_and_ref" do
       let(:type) { 'test' }
 
       it "returns builds if no branch specified" do

spec/lib/gitlab/ci/config/entry/policy_spec.rb

@@ -16,8 +16,8 @@ describe Gitlab::Ci::Config::Entry::Policy do
           end
 
           describe '#value' do
-            it 'returns key value' do
-              expect(entry.value).to eq config
+            it 'returns refs hash' do
+              expect(entry.value).to eq(refs: config)
             end
           end
         end
@@ -56,6 +56,50 @@ describe Gitlab::Ci::Config::Entry::Policy do
     end
   end
 
+  context 'when using complex policy' do
+    context 'when specifiying refs policy' do
+      let(:config) { { refs: ['master'] } }
+
+      it 'is a correct configuraton' do
+        expect(entry).to be_valid
+        expect(entry.value).to eq(refs: %w[master])
+      end
+    end
+
+    context 'when specifying kubernetes policy' do
+      let(:config) { { kubernetes: 'active' } }
+
+      it 'is a correct configuraton' do
+        expect(entry).to be_valid
+        expect(entry.value).to eq(kubernetes: 'active')
+      end
+    end
+
+    context 'when specifying invalid kubernetes policy' do
+      let(:config) { { kubernetes: 'something' } }
+
+      it 'reports an error about invalid policy' do
+        expect(entry.errors).to include /unknown value: something/
+      end
+    end
+
+    context 'when specifying unknown policy' do
+      let(:config) { { refs: ['master'], invalid: :something } }
+
+      it 'returns error about invalid key' do
+        expect(entry.errors).to include /unknown keys: invalid/
+      end
+    end
+
+    context 'when policy is empty' do
+      let(:config) { {} }
+
+      it 'is not a valid configuration' do
+        expect(entry.errors).to include /can't be blank/
+      end
+    end
+  end
+
   context 'when policy strategy does not match' do
     let(:config) { 'string strategy' }
 

spec/models/ci/pipeline_spec.rb

@@ -546,6 +546,22 @@ describe Ci::Pipeline, :mailer do
     end
   end
 
+  describe '#has_kubernetes_active?' do
+    context 'when kubernetes is active' do
+      let(:project) { create(:kubernetes_project) }
+
+      it 'returns true' do
+        expect(pipeline).to have_kubernetes_active
+      end
+    end
+
+    context 'when kubernetes is not active' do
+      it 'returns false' do
+        expect(pipeline).not_to have_kubernetes_active
+      end
+    end
+  end
+
   describe '#has_stage_seeds?' do
     context 'when pipeline has stage seeds' do
       subject { build(:ci_pipeline_with_one_job) }

spec/models/project_services/kubernetes_service_spec.rb

@@ -203,18 +203,13 @@ describe KubernetesService, :use_clean_rails_memory_store_caching do
 
   describe '#predefined_variables' do
     let(:kubeconfig) do
-      config =
-        YAML.load(File.read(expand_fixture_path('config/kubeconfig.yml')))
-
-      config.dig('users', 0, 'user')['token'] =
-        'token'
-
+      config_file = expand_fixture_path('config/kubeconfig.yml')
+      config = YAML.load(File.read(config_file))
+      config.dig('users', 0, 'user')['token'] = 'token'
+      config.dig('contexts', 0, 'context')['namespace'] = namespace
       config.dig('clusters', 0, 'cluster')['certificate-authority-data'] =
         Base64.encode64('CA PEM DATA')
 
-      config.dig('contexts', 0, 'context')['namespace'] =
-        namespace
-
       YAML.dump(config)
     end
 

spec/views/ci/lints/show.html.haml_spec.rb

@@ -73,8 +73,8 @@ describe 'ci/lints/show' do
       render
 
       expect(rendered).to have_content('Tag list: dotnet')
-      expect(rendered).to have_content('Refs only: test@dude/repo')
-      expect(rendered).to have_content('Refs except: deploy')
+      expect(rendered).to have_content('Only policy: refs, test@dude/repo')
+      expect(rendered).to have_content('Except policy: refs, deploy')
       expect(rendered).to have_content('Environment: testing')
       expect(rendered).to have_content('When: on_success')
     end