Merge branch 'cluster_agent' into 'master'

Add internal Kubernetes Agent API

See merge request gitlab-org/gitlab!33228

app/models/clusters/agent.rb

+# frozen_string_literal: true
+
+module Clusters
+  class Agent < ApplicationRecord
+    self.table_name = 'cluster_agents'
+
+    belongs_to :project, class_name: '::Project' # Otherwise, it will load ::Clusters::Project
+
+    has_many :agent_tokens, class_name: 'Clusters::AgentToken'
+
+    validates :name, presence: true, length: { maximum: 255 }, uniqueness: { scope: :project_id }
+  end
+end

app/models/clusters/agent_token.rb

+# frozen_string_literal: true
+
+module Clusters
+  class AgentToken < ApplicationRecord
+    include TokenAuthenticatable
+    add_authentication_token_field :token, encrypted: :required
+
+    self.table_name = 'cluster_agent_tokens'
+
+    belongs_to :agent, class_name: 'Clusters::Agent'
+
+    before_save :ensure_token
+  end
+end

app/models/project.rb

@@ -261,6 +261,7 @@ class Project < ApplicationRecord
   has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster'
   has_many :kubernetes_namespaces, class_name: 'Clusters::KubernetesNamespace'
   has_many :management_clusters, class_name: 'Clusters::Cluster', foreign_key: :management_project_id, inverse_of: :management_project
+  has_many :cluster_agents, class_name: 'Clusters::Agent'
 
   has_many :prometheus_metrics
   has_many :prometheus_alerts, inverse_of: :project

changelogs/unreleased/cluster_agent.yml

+---
+title: Adds models and tables for cluster agent and cluster agent tokens
+merge_request: 33228
+author:
+type: other

db/migrate/20200607223047_create_cluster_agents.rb

+# frozen_string_literal: true
+
+class CreateClusterAgents < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:cluster_agents)
+      with_lock_retries do
+        create_table :cluster_agents do |t|
+          t.timestamps_with_timezone null: false
+          t.belongs_to(:project, null: false, index: true, foreign_key: { on_delete: :cascade })
+          t.text :name, null: false
+
+          t.index [:project_id, :name], unique: true
+        end
+      end
+    end
+
+    add_text_limit :cluster_agents, :name, 255
+  end
+
+  def down
+    with_lock_retries do
+      drop_table :cluster_agents
+    end
+  end
+end

db/migrate/20200607235435_create_cluster_agent_tokens.rb

+# frozen_string_literal: true
+
+class CreateClusterAgentTokens < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:cluster_agent_tokens)
+      create_table :cluster_agent_tokens do |t|
+        t.timestamps_with_timezone null: false
+        t.belongs_to :agent, null: false, index: true, foreign_key: { to_table: :cluster_agents, on_delete: :cascade }
+        t.text :token_encrypted, null: false
+
+        t.index :token_encrypted, unique: true
+      end
+    end
+
+    add_text_limit :cluster_agent_tokens, :token_encrypted, 255
+  end
+
+  def down
+    drop_table :cluster_agent_tokens
+  end
+end

db/structure.sql

@@ -10374,6 +10374,42 @@ CREATE SEQUENCE public.ci_variables_id_seq
 
 ALTER SEQUENCE public.ci_variables_id_seq OWNED BY public.ci_variables.id;
 
+CREATE TABLE public.cluster_agent_tokens (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    agent_id bigint NOT NULL,
+    token_encrypted text NOT NULL,
+    CONSTRAINT check_c60daed227 CHECK ((char_length(token_encrypted) <= 255))
+);
+
+CREATE SEQUENCE public.cluster_agent_tokens_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.cluster_agent_tokens_id_seq OWNED BY public.cluster_agent_tokens.id;
+
+CREATE TABLE public.cluster_agents (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    project_id bigint NOT NULL,
+    name text NOT NULL,
+    CONSTRAINT check_3498369510 CHECK ((char_length(name) <= 255))
+);
+
+CREATE SEQUENCE public.cluster_agents_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.cluster_agents_id_seq OWNED BY public.cluster_agents.id;
+
 CREATE TABLE public.cluster_groups (
     id integer NOT NULL,
     cluster_id integer NOT NULL,
@@ -16572,6 +16608,10 @@ ALTER TABLE ONLY public.ci_triggers ALTER COLUMN id SET DEFAULT nextval('public.
 
 ALTER TABLE ONLY public.ci_variables ALTER COLUMN id SET DEFAULT nextval('public.ci_variables_id_seq'::regclass);
 
+ALTER TABLE ONLY public.cluster_agent_tokens ALTER COLUMN id SET DEFAULT nextval('public.cluster_agent_tokens_id_seq'::regclass);
+
+ALTER TABLE ONLY public.cluster_agents ALTER COLUMN id SET DEFAULT nextval('public.cluster_agents_id_seq'::regclass);
+
 ALTER TABLE ONLY public.cluster_groups ALTER COLUMN id SET DEFAULT nextval('public.cluster_groups_id_seq'::regclass);
 
 ALTER TABLE ONLY public.cluster_platforms_kubernetes ALTER COLUMN id SET DEFAULT nextval('public.cluster_platforms_kubernetes_id_seq'::regclass);
@@ -17519,6 +17559,12 @@ ALTER TABLE ONLY public.ci_triggers
 ALTER TABLE ONLY public.ci_variables
     ADD CONSTRAINT ci_variables_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY public.cluster_agent_tokens
+    ADD CONSTRAINT cluster_agent_tokens_pkey PRIMARY KEY (id);
+
+ALTER TABLE ONLY public.cluster_agents
+    ADD CONSTRAINT cluster_agents_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY public.cluster_groups
     ADD CONSTRAINT cluster_groups_pkey PRIMARY KEY (id);
 
@@ -19021,6 +19067,14 @@ CREATE INDEX index_ci_triggers_on_project_id ON public.ci_triggers USING btree (
 
 CREATE UNIQUE INDEX index_ci_variables_on_project_id_and_key_and_environment_scope ON public.ci_variables USING btree (project_id, key, environment_scope);
 
+CREATE INDEX index_cluster_agent_tokens_on_agent_id ON public.cluster_agent_tokens USING btree (agent_id);
+
+CREATE UNIQUE INDEX index_cluster_agent_tokens_on_token_encrypted ON public.cluster_agent_tokens USING btree (token_encrypted);
+
+CREATE INDEX index_cluster_agents_on_project_id ON public.cluster_agents USING btree (project_id);
+
+CREATE UNIQUE INDEX index_cluster_agents_on_project_id_and_name ON public.cluster_agents USING btree (project_id, name);
+
 CREATE UNIQUE INDEX index_cluster_groups_on_cluster_id_and_group_id ON public.cluster_groups USING btree (cluster_id, group_id);
 
 CREATE INDEX index_cluster_groups_on_group_id ON public.cluster_groups USING btree (group_id);
@@ -21712,6 +21766,9 @@ ALTER TABLE ONLY public.group_custom_attributes
 ALTER TABLE ONLY public.requirements_management_test_reports
     ADD CONSTRAINT fk_rails_24cecc1e68 FOREIGN KEY (pipeline_id) REFERENCES public.ci_pipelines(id) ON DELETE SET NULL;
 
+ALTER TABLE ONLY public.cluster_agents
+    ADD CONSTRAINT fk_rails_25e9fc2d5d FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY public.group_wiki_repositories
     ADD CONSTRAINT fk_rails_26f867598c FOREIGN KEY (group_id) REFERENCES public.namespaces(id) ON DELETE CASCADE;
 
@@ -22510,6 +22567,9 @@ ALTER TABLE ONLY public.subscriptions
 ALTER TABLE ONLY public.operations_strategies
     ADD CONSTRAINT fk_rails_d183b6e6dd FOREIGN KEY (feature_flag_id) REFERENCES public.operations_feature_flags(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY public.cluster_agent_tokens
+    ADD CONSTRAINT fk_rails_d1d26abc25 FOREIGN KEY (agent_id) REFERENCES public.cluster_agents(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY public.requirements_management_test_reports
     ADD CONSTRAINT fk_rails_d1e8b498bf FOREIGN KEY (author_id) REFERENCES public.users(id) ON DELETE SET NULL;
 
@@ -23756,6 +23816,8 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200605160806
 20200605160836
 20200605160851
+20200607223047
+20200607235435
 20200608072931
 20200608075553
 20200608195222

lib/api/api.rb

@@ -237,6 +237,7 @@ module API
 
     mount ::API::Internal::Base
     mount ::API::Internal::Pages
+    mount ::API::Internal::Kubernetes
 
     route :any, '*path' do
       error!('404 Not Found', 404)

lib/api/internal/kubernetes.rb

+# frozen_string_literal: true
+
+module API
+  # Kubernetes Internal API
+  module Internal
+    class Kubernetes < Grape::API::Instance
+      helpers do
+        def repo_type
+          Gitlab::GlRepository::PROJECT
+        end
+
+        def gl_repository(project)
+          repo_type.identifier_for_container(project)
+        end
+
+        def gl_repository_path(project)
+          repo_type.repository_for(project).full_path
+        end
+
+        def check_feature_enabled
+          not_found! unless Feature.enabled?(:kubernetes_agent_internal_api)
+        end
+      end
+
+      namespace 'internal' do
+        namespace 'kubernetes' do
+          desc 'Gets agent info' do
+            detail 'Retrieves agent info for the given token'
+          end
+          route_setting :authentication, cluster_agent_token_allowed: true
+          get '/agent_info' do
+            check_feature_enabled
+
+            agent_token = cluster_agent_token_from_authorization_token
+
+            if agent_token
+              agent = agent_token.agent
+              project = agent.project
+              @gl_project_string = "project-#{project.id}"
+
+              status 200
+              {
+                project_id: project.id,
+                agent_id: agent.id,
+                agent_name: agent.name,
+                storage_name: project.repository_storage,
+                relative_path: project.disk_path + '.git',
+                gl_repository: gl_repository(project),
+                gl_project_path: gl_repository_path(project)
+              }
+            else
+              status 403
+            end
+          end
+
+          desc 'Gets project info' do
+            detail 'Retrieves project info (if authorized)'
+          end
+          route_setting :authentication, cluster_agent_token_allowed: true
+          get '/project_info' do
+            check_feature_enabled
+
+            agent_token = cluster_agent_token_from_authorization_token
+
+            if agent_token
+              project = find_project(params[:id])
+
+              # TODO sort out authorization for real
+              # https://gitlab.com/gitlab-org/gitlab/-/issues/220912
+              if !project || !project.public?
+                not_found!
+              end
+
+              @gl_project_string = "project-#{project.id}"
+
+              status 200
+              {
+                project_id: project.id,
+                storage_name: project.repository_storage,
+                relative_path: project.disk_path + '.git',
+                gl_repository: gl_repository(project),
+                gl_project_path: gl_repository_path(project)
+              }
+            else
+              status 403
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/auth/auth_finders.rb

@@ -20,6 +20,7 @@ module Gitlab
     module AuthFinders
       include Gitlab::Utils::StrongMemoize
       include ActionController::HttpAuthentication::Basic
+      include ActionController::HttpAuthentication::Token
 
       PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'
       PRIVATE_TOKEN_PARAM = :private_token
@@ -131,6 +132,15 @@ module Gitlab
         deploy_token
       end
 
+      def cluster_agent_token_from_authorization_token
+        return unless route_authentication_setting[:cluster_agent_token_allowed]
+        return unless current_request.authorization.present?
+
+        authorization_token, _options = token_and_options(current_request)
+
+        ::Clusters::AgentToken.find_by_token(authorization_token)
+      end
+
       def find_runner_from_token
         return unless api_request?
 

spec/factories/clusters/agent_tokens.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :cluster_agent_token, class: 'Clusters::AgentToken' do
+    association :agent, factory: :cluster_agent
+
+    token_encrypted { Gitlab::CryptoHelper.aes256_gcm_encrypt(SecureRandom.hex(50)) }
+  end
+end

spec/factories/clusters/agents.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :cluster_agent, class: 'Clusters::Agent' do
+    project
+
+    sequence(:name) { |n| "agent_#{n}" }
+  end
+end

spec/lib/gitlab/auth/auth_finders_spec.rb

@@ -744,6 +744,56 @@ RSpec.describe Gitlab::Auth::AuthFinders do
     end
   end
 
+  describe '#cluster_agent_token_from_authorization_token' do
+    let_it_be(:agent_token) { create(:cluster_agent_token) }
+
+    context 'when route_setting is empty' do
+      it 'returns nil' do
+        expect(cluster_agent_token_from_authorization_token).to be_nil
+      end
+    end
+
+    context 'when route_setting allows cluster agent token' do
+      let(:route_authentication_setting) { { cluster_agent_token_allowed: true } }
+
+      context 'Authorization header is empty' do
+        it 'returns nil' do
+          expect(cluster_agent_token_from_authorization_token).to be_nil
+        end
+      end
+
+      context 'Authorization header is incorrect' do
+        before do
+          request.headers['Authorization'] = 'Bearer ABCD'
+        end
+
+        it 'returns nil' do
+          expect(cluster_agent_token_from_authorization_token).to be_nil
+        end
+      end
+
+      context 'Authorization header is malformed' do
+        before do
+          request.headers['Authorization'] = 'Bearer'
+        end
+
+        it 'returns nil' do
+          expect(cluster_agent_token_from_authorization_token).to be_nil
+        end
+      end
+
+      context 'Authorization header matches agent token' do
+        before do
+          request.headers['Authorization'] = "Bearer #{agent_token.token}"
+        end
+
+        it 'returns the agent token' do
+          expect(cluster_agent_token_from_authorization_token).to eq(agent_token)
+        end
+      end
+    end
+  end
+
   describe '#find_runner_from_token' do
     let(:runner) { create(:ci_runner) }
 

spec/lib/gitlab/import_export/all_models.yml

@@ -312,6 +312,7 @@ project:
 - chat_services
 - cluster
 - clusters
+- cluster_agents
 - cluster_project
 - creator
 - cycle_analytics_stages

spec/models/clusters/agent_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Clusters::Agent do
+  subject { create(:cluster_agent) }
+
+  it { is_expected.to belong_to(:project).class_name('::Project') }
+  it { is_expected.to have_many(:agent_tokens).class_name('Clusters::AgentToken') }
+
+  it { is_expected.to validate_presence_of(:name) }
+  it { is_expected.to validate_length_of(:name).is_at_most(255) }
+  it { is_expected.to validate_uniqueness_of(:name).scoped_to(:project_id) }
+end

spec/models/clusters/agent_token_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Clusters::AgentToken do
+  it { is_expected.to belong_to(:agent).class_name('Clusters::Agent') }
+
+  describe '#token' do
+    it 'is generated on save' do
+      agent_token = build(:cluster_agent_token, token_encrypted: nil)
+      expect(agent_token.token).to be_nil
+
+      agent_token.save!
+
+      expect(agent_token.token).to be_present
+    end
+  end
+end

spec/models/project_spec.rb

@@ -104,6 +104,7 @@ RSpec.describe Project do
     it { is_expected.to have_many(:clusters) }
     it { is_expected.to have_many(:management_clusters).class_name('Clusters::Cluster') }
     it { is_expected.to have_many(:kubernetes_namespaces) }
+    it { is_expected.to have_many(:cluster_agents).class_name('Clusters::Agent') }
     it { is_expected.to have_many(:custom_attributes).class_name('ProjectCustomAttribute') }
     it { is_expected.to have_many(:project_badges).class_name('ProjectBadge') }
     it { is_expected.to have_many(:lfs_file_locks) }

spec/requests/api/internal/kubernetes_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::Internal::Kubernetes do
+  describe "GET /internal/kubernetes/agent_info" do
+    context 'kubernetes_agent_internal_api feature flag disabled' do
+      before do
+        stub_feature_flags(kubernetes_agent_internal_api: false)
+      end
+
+      it 'returns 404' do
+        get api('/internal/kubernetes/agent_info')
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    it 'returns 403 if Authorization header not sent' do
+      get api('/internal/kubernetes/agent_info')
+
+      expect(response).to have_gitlab_http_status(:forbidden)
+    end
+
+    context 'an agent is found' do
+      let!(:agent_token) { create(:cluster_agent_token) }
+
+      let(:agent) { agent_token.agent }
+      let(:project) { agent.project }
+
+      it 'returns expected data', :aggregate_failures do
+        get api('/internal/kubernetes/agent_info'), headers: { 'Authorization' => "Bearer #{agent_token.token}" }
+
+        expect(response).to have_gitlab_http_status(:success)
+
+        expect(json_response['project_id']).to eq(project.id)
+        expect(json_response['agent_id']).to eq(agent.id)
+        expect(json_response['agent_name']).to eq(agent.name)
+        expect(json_response['storage_name']).to eq(project.repository_storage)
+        expect(json_response['relative_path']).to eq(project.disk_path + '.git')
+        expect(json_response['gl_repository']).to eq("project-#{project.id}")
+        expect(json_response['gl_project_path']).to eq(project.full_path)
+      end
+    end
+
+    context 'no such agent exists' do
+      it 'returns 404' do
+        get api('/internal/kubernetes/agent_info'), headers: { 'Authorization' => 'Bearer ABCD' }
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+  end
+
+  describe 'GET /internal/kubernetes/project_info' do
+    context 'kubernetes_agent_internal_api feature flag disabled' do
+      before do
+        stub_feature_flags(kubernetes_agent_internal_api: false)
+      end
+
+      it 'returns 404' do
+        get api('/internal/kubernetes/project_info')
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    it 'returns 403 if Authorization header not sent' do
+      get api('/internal/kubernetes/project_info')
+
+      expect(response).to have_gitlab_http_status(:forbidden)
+    end
+
+    context 'no such agent exists' do
+      it 'returns 404' do
+        get api('/internal/kubernetes/project_info'), headers: { 'Authorization' => 'Bearer ABCD' }
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+
+    context 'an agent is found' do
+      let!(:agent_token) { create(:cluster_agent_token) }
+
+      let(:agent) { agent_token.agent }
+
+      context 'project is public' do
+        let(:project) { create(:project, :public) }
+
+        it 'returns expected data', :aggregate_failures do
+          get api('/internal/kubernetes/project_info'), params: { id: project.id }, headers: { 'Authorization' => "Bearer #{agent_token.token}" }
+
+          expect(response).to have_gitlab_http_status(:success)
+
+          expect(json_response['project_id']).to eq(project.id)
+          expect(json_response['storage_name']).to eq(project.repository_storage)
+          expect(json_response['relative_path']).to eq(project.disk_path + '.git')
+          expect(json_response['gl_repository']).to eq("project-#{project.id}")
+          expect(json_response['gl_project_path']).to eq(project.full_path)
+        end
+      end
+
+      context 'project is private' do
+        let(:project) { create(:project, :private) }
+
+        it 'returns 404' do
+          get api('/internal/kubernetes/project_info'), params: { id: project.id }, headers: { 'Authorization' => "Bearer #{agent_token.token}" }
+
+          expect(response).to have_gitlab_http_status(:not_found)
+        end
+      end
+
+      context 'project is internal' do
+        let(:project) { create(:project, :internal) }
+
+        it 'returns 404' do
+          get api('/internal/kubernetes/project_info'), params: { id: project.id }, headers: { 'Authorization' => "Bearer #{agent_token.token}" }
+
+          expect(response).to have_gitlab_http_status(:not_found)
+        end
+      end
+
+      context 'project does not exist' do
+        it 'returns 404' do
+          get api('/internal/kubernetes/project_info'), params: { id: 0 }, headers: { 'Authorization' => "Bearer #{agent_token.token}" }
+
+          expect(response).to have_gitlab_http_status(:not_found)
+        end
+      end
+    end
+  end
+end