Merge branch '324275-agent-config-ci-variable' into 'master'

Inject kubectl-compatible config file into the CI job

See merge request gitlab-org/gitlab!67089

app/models/ci/build.rb

@@ -554,6 +554,7 @@ module Ci
           .concat(persisted_variables)
           .concat(dependency_proxy_variables)
           .concat(job_jwt_variables)
+          .concat(kubernetes_variables)
           .concat(scoped_variables)
           .concat(job_variables)
           .concat(persisted_environment_variables)
@@ -1172,6 +1173,10 @@ module Ci
       end
     end
 
+    def kubernetes_variables
+      [] # Overridden in EE
+    end
+
     def conditionally_allow_failure!(exit_code)
       return unless exit_code
 

config/feature_flags/development/agent_kubeconfig_ci_variable.yml

+---
+name: agent_kubeconfig_ci_variable
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67089
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/337164
+milestone: '14.2'
+type: development
+group: group::configure
+default_enabled: false

ee/app/models/ee/ci/build.rb

@@ -225,6 +225,25 @@ module EE
         end
       end
 
+      override :kubernetes_variables
+      def kubernetes_variables
+        ::Gitlab::Ci::Variables::Collection.new.tap do |collection|
+          break collection unless ::Feature.enabled?(:agent_kubeconfig_ci_variable, project, default_enabled: :yaml)
+
+          # A cluster deployemnt may also define a KUBECONFIG variable, so to keep existing
+          # configurations working we shouldn't overwrite it here.
+          # This check will be removed when Cluster and Agent configurations are
+          # merged in https://gitlab.com/gitlab-org/gitlab/-/issues/335089
+          break collection if deployment&.deployment_cluster
+
+          template = ::Ci::GenerateKubeconfigService.new(self).execute
+
+          if template.valid?
+            collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true)
+          end
+        end
+      end
+
       def parse_security_artifact_blob(security_report, blob)
         report_clone = security_report.clone_as_blank
         parse_raw_security_artifact_blob(report_clone, blob)

ee/app/models/ee/ci/pipeline.rb

@@ -177,6 +177,12 @@ module EE
         ondemand_dast_scan? && parameter_source?
       end
 
+      def authorized_cluster_agents
+        strong_memoize(:authorized_cluster_agents) do
+          ::Clusters::DeployableAgentsFinder.new(project).execute
+        end
+      end
+
       private
 
       def has_security_reports?

ee/app/services/ci/generate_kubeconfig_service.rb

+# frozen_string_literal: true
+
+module Ci
+  class GenerateKubeconfigService
+    def initialize(build)
+      @build = build
+      @project = build.project
+      @template = Gitlab::Kubernetes::Kubeconfig::Template.new
+    end
+
+    def execute
+      template.add_cluster(
+        name: cluster_name,
+        url: Gitlab::Kas.tunnel_url
+      )
+
+      agents.each do |agent|
+        user = user_name(agent)
+
+        template.add_user(
+          name: user,
+          token: agent_token(agent)
+        )
+
+        template.add_context(
+          name: context_name(agent),
+          cluster: cluster_name,
+          user: user
+        )
+      end
+
+      template
+    end
+
+    private
+
+    attr_reader :build, :project, :template
+
+    def agents
+      build.pipeline.authorized_cluster_agents
+    end
+
+    def cluster_name
+      'gitlab'
+    end
+
+    def user_name(agent)
+      ['agent', agent.id].join(delimiter)
+    end
+
+    def context_name(agent)
+      [project.full_path, agent.name].join(delimiter)
+    end
+
+    def agent_token(agent)
+      ['ci', agent.id, build.token].join(delimiter)
+    end
+
+    def delimiter
+      ':'
+    end
+  end
+end

ee/spec/models/ci/build_spec.rb

@@ -325,6 +325,39 @@ RSpec.describe Ci::Build do
         expect(requirement_variable).to be_nil
       end
     end
+
+    describe 'kubernetes variables' do
+      let(:service) { double(execute: template) }
+      let(:template) { double(to_yaml: 'example-kubeconfig', valid?: template_valid) }
+      let(:template_valid) { true }
+
+      before do
+        allow(::Ci::GenerateKubeconfigService).to receive(:new).with(job).and_return(service)
+      end
+
+      it { is_expected.to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+
+      context 'feature flag is disabled' do
+        before do
+          stub_feature_flags(agent_kubeconfig_ci_variable: false)
+        end
+
+        it { is_expected.not_to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+      end
+
+      context 'job is deploying to a cluster' do
+        let(:deployment) { create(:deployment, deployment_cluster: create(:deployment_cluster)) }
+        let(:job) { create(:ci_build, pipeline: pipeline, deployment: deployment) }
+
+        it { is_expected.not_to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+      end
+
+      context 'generated config is invalid' do
+        let(:template_valid) { false }
+
+        it { is_expected.not_to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+      end
+    end
   end
 
   describe '#has_security_reports?' do

ee/spec/models/ci/pipeline_spec.rb

@@ -634,4 +634,18 @@ RSpec.describe Ci::Pipeline do
       it { is_expected.to eq(true) }
     end
   end
+
+  describe '#authorized_cluster_agents' do
+    let(:finder) { double(execute: agents) }
+    let(:agents) { double }
+
+    it 'retrieves agent records from the finder and caches the result' do
+      expect(Clusters::DeployableAgentsFinder).to receive(:new).once
+        .with(pipeline.project)
+        .and_return(finder)
+
+      expect(pipeline.authorized_cluster_agents).to eq(agents)
+      expect(pipeline.authorized_cluster_agents).to eq(agents) # cached
+    end
+  end
 end

ee/spec/services/ci/generate_kubeconfig_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::GenerateKubeconfigService do
+  describe '#execute' do
+    let(:project) { create(:project) }
+    let(:build) { create(:ci_build, project: project) }
+    let(:agent1) { create(:cluster_agent, project: project) }
+    let(:agent2) { create(:cluster_agent, project: project) }
+
+    let(:template) { instance_double(Gitlab::Kubernetes::Kubeconfig::Template) }
+
+    subject { described_class.new(build).execute }
+
+    before do
+      expect(Gitlab::Kubernetes::Kubeconfig::Template).to receive(:new).and_return(template)
+      expect(build.pipeline).to receive(:authorized_cluster_agents).and_return([agent1, agent2])
+    end
+
+    it 'adds a cluster, and a user and context for each available agent' do
+      expect(template).to receive(:add_cluster).with(
+        name: 'gitlab',
+        url: Gitlab::Kas.tunnel_url
+      ).once
+
+      expect(template).to receive(:add_user).with(
+        name: "agent:#{agent1.id}",
+        token: "ci:#{agent1.id}:#{build.token}"
+      )
+      expect(template).to receive(:add_user).with(
+        name: "agent:#{agent2.id}",
+        token: "ci:#{agent2.id}:#{build.token}"
+      )
+
+      expect(template).to receive(:add_context).with(
+        name: "#{project.full_path}:#{agent1.name}",
+        cluster: 'gitlab',
+        user: "agent:#{agent1.id}"
+      )
+      expect(template).to receive(:add_context).with(
+        name: "#{project.full_path}:#{agent2.name}",
+        cluster: 'gitlab',
+        user: "agent:#{agent2.id}"
+      )
+
+      expect(subject).to eq(template)
+    end
+  end
+end

lib/gitlab/kas.rb

@@ -5,6 +5,7 @@ module Gitlab
     INTERNAL_API_REQUEST_HEADER = 'Gitlab-Kas-Api-Request'
     VERSION_FILE = 'GITLAB_KAS_VERSION'
     JWT_ISSUER = 'gitlab-kas'
+    K8S_PROXY_PATH = 'k8s-proxy'
 
     include JwtAuthenticatable
 
@@ -39,6 +40,10 @@ module Gitlab
         Gitlab.config.gitlab_kas.external_url
       end
 
+      def tunnel_url
+        URI.join(external_url, K8S_PROXY_PATH).to_s
+      end
+
       # Return GitLab KAS internal_url
       #
       # @return [String] internal_url

lib/gitlab/kubernetes/kubeconfig/entry/cluster.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Kubeconfig
+      module Entry
+        class Cluster
+          attr_reader :name
+
+          def initialize(name:, url:, ca_pem: nil)
+            @name = name
+            @url = url
+            @ca_pem = ca_pem
+          end
+
+          def to_h
+            {
+              name: name,
+              cluster: cluster
+            }
+          end
+
+          private
+
+          attr_reader :url, :ca_pem
+
+          def cluster
+            {
+              server: url,
+              'certificate-authority-data': certificate_authority_data
+            }.compact
+          end
+
+          def certificate_authority_data
+            return unless ca_pem.present?
+
+            Base64.strict_encode64(ca_pem)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/kubernetes/kubeconfig/entry/context.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Kubeconfig
+      module Entry
+        class Context
+          attr_reader :name
+
+          def initialize(name:, cluster:, user:, namespace: nil)
+            @name = name
+            @cluster = cluster
+            @user = user
+            @namespace = namespace
+          end
+
+          def to_h
+            {
+              name: name,
+              context: context
+            }
+          end
+
+          private
+
+          attr_reader :cluster, :user, :namespace
+
+          def context
+            {
+              cluster: cluster,
+              namespace: namespace,
+              user: user
+            }.compact
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/kubernetes/kubeconfig/entry/user.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Kubeconfig
+      module Entry
+        class User
+          attr_reader :name
+
+          def initialize(name:, token:)
+            @name = name
+            @token = token
+          end
+
+          def to_h
+            {
+              name: name,
+              user: { token: token }
+            }
+          end
+
+          private
+
+          attr_reader :token
+        end
+      end
+    end
+  end
+end

lib/gitlab/kubernetes/kubeconfig/template.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Kubeconfig
+      class Template
+        ENTRIES = {
+          cluster: Gitlab::Kubernetes::Kubeconfig::Entry::Cluster,
+          user: Gitlab::Kubernetes::Kubeconfig::Entry::User,
+          context: Gitlab::Kubernetes::Kubeconfig::Entry::Context
+        }.freeze
+
+        def initialize
+          @clusters = []
+          @users = []
+          @contexts = []
+        end
+
+        def valid?
+          contexts.present?
+        end
+
+        def add_cluster(**args)
+          clusters << new_entry(:cluster, **args)
+        end
+
+        def add_user(**args)
+          users << new_entry(:user, **args)
+        end
+
+        def add_context(**args)
+          contexts << new_entry(:context, **args)
+        end
+
+        def to_h
+          {
+            apiVersion: 'v1',
+            kind: 'Config',
+            clusters: clusters.map(&:to_h),
+            users: users.map(&:to_h),
+            contexts: contexts.map(&:to_h)
+          }
+        end
+
+        def to_yaml
+          YAML.dump(to_h.deep_stringify_keys)
+        end
+
+        private
+
+        attr_reader :clusters, :users, :contexts
+
+        def new_entry(entry, **args)
+          ENTRIES.fetch(entry).new(**args)
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/kas_spec.rb

@@ -65,6 +65,12 @@ RSpec.describe Gitlab::Kas do
     end
   end
 
+  describe '.tunnel_url' do
+    it 'returns gitlab_kas external_url with proxy path appended' do
+      expect(described_class.tunnel_url).to eq(Gitlab.config.gitlab_kas.external_url + '/k8s-proxy')
+    end
+  end
+
   describe '.internal_url' do
     it 'returns gitlab_kas internal_url config' do
       expect(described_class.internal_url).to eq(Gitlab.config.gitlab_kas.internal_url)

spec/lib/gitlab/kubernetes/kubeconfig/entry/cluster_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Kubernetes::Kubeconfig::Entry::Cluster do
+  describe '#to_h' do
+    let(:name) { 'name' }
+    let(:url) { 'url' }
+
+    subject { described_class.new(name: name, url: url).to_h }
+
+    it { is_expected.to eq({ name: name, cluster: { server: url } }) }
+
+    context 'with a certificate' do
+      let(:cert) { 'certificate' }
+      let(:cert_encoded) { Base64.strict_encode64(cert) }
+
+      subject { described_class.new(name: name, url: url, ca_pem: cert).to_h }
+
+      it { is_expected.to eq({ name: name, cluster: { server: url, 'certificate-authority-data': cert_encoded } }) }
+    end
+  end
+end

spec/lib/gitlab/kubernetes/kubeconfig/entry/context_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Kubernetes::Kubeconfig::Entry::Context do
+  describe '#to_h' do
+    let(:name) { 'name' }
+    let(:user) { 'user' }
+    let(:cluster) { 'cluster' }
+
+    subject { described_class.new(name: name, user: user, cluster: cluster).to_h }
+
+    it { is_expected.to eq({ name: name, context: { cluster: cluster, user: user } }) }
+
+    context 'with a namespace' do
+      let(:namespace) { 'namespace' }
+
+      subject { described_class.new(name: name, user: user, cluster: cluster, namespace: namespace).to_h }
+
+      it { is_expected.to eq({ name: name, context: { cluster: cluster, user: user, namespace: namespace } }) }
+    end
+  end
+end

spec/lib/gitlab/kubernetes/kubeconfig/entry/user_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Kubernetes::Kubeconfig::Entry::User do
+  describe '#to_h' do
+    let(:name) { 'name' }
+    let(:token) { 'token' }
+
+    subject { described_class.new(name: name, token: token).to_h }
+
+    it { is_expected.to eq({ name: name, user: { token: token } }) }
+  end
+end

spec/lib/gitlab/kubernetes/kubeconfig/template_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Kubernetes::Kubeconfig::Template do
+  let(:template) { described_class.new }
+
+  describe '#valid?' do
+    subject { template.valid? }
+
+    it { is_expected.to be_falsey }
+
+    context 'with configuration added' do
+      before do
+        template.add_context(name: 'name', cluster: 'cluster', user: 'user')
+      end
+
+      it { is_expected.to be_truthy }
+    end
+  end
+
+  describe '#to_h' do
+    subject { described_class.new.to_h }
+
+    it do
+      is_expected.to eq(
+        apiVersion: 'v1',
+        kind: 'Config',
+        clusters: [],
+        users: [],
+        contexts: []
+      )
+    end
+  end
+
+  describe '#to_yaml' do
+    subject { template.to_yaml }
+
+    it { is_expected.to eq(YAML.dump(template.to_h.deep_stringify_keys)) }
+  end
+
+  describe 'adding entries' do
+    let(:entry) { instance_double(entry_class, to_h: attributes) }
+    let(:attributes) do
+      { name: 'name', other: 'other' }
+    end
+
+    subject { template.to_h }
+
+    before do
+      expect(entry_class).to receive(:new).with(attributes).and_return(entry)
+    end
+
+    describe '#add_cluster' do
+      let(:entry_class) { Gitlab::Kubernetes::Kubeconfig::Entry::Cluster }
+
+      before do
+        template.add_cluster(**attributes)
+      end
+
+      it { is_expected.to include(clusters: [attributes]) }
+    end
+
+    describe '#add_user' do
+      let(:entry_class) { Gitlab::Kubernetes::Kubeconfig::Entry::User }
+
+      before do
+        template.add_user(**attributes)
+      end
+
+      it { is_expected.to include(users: [attributes]) }
+    end
+
+    describe '#add_context' do
+      let(:entry_class) { Gitlab::Kubernetes::Kubeconfig::Entry::Context }
+
+      before do
+        template.add_context(**attributes)
+      end
+
+      it { is_expected.to include(contexts: [attributes]) }
+    end
+  end
+end