Add gitlab-managed option to clusters form

When this option is enabled, GitLab will create namespaces and service
accounts as usual. When disabled, GitLab wont create any project
specific kubernetes resources

Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/56557

app/controllers/clusters/clusters_controller.rb

@@ -156,6 +156,7 @@ class Clusters::ClustersController < Clusters::BaseController
       :enabled,
       :name,
       :environment_scope,
+      :managed,
       provider_gcp_attributes: [
         :gcp_project_id,
         :zone,
@@ -174,6 +175,7 @@ class Clusters::ClustersController < Clusters::BaseController
       :enabled,
       :name,
       :environment_scope,
+      :managed,
       platform_kubernetes_attributes: [
         :namespace,
         :api_url,

app/models/clusters/cluster.rb

@@ -94,6 +94,7 @@ module Clusters
     scope :user_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:user]) }
     scope :gcp_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:gcp]) }
     scope :gcp_installed, -> { gcp_provided.includes(:provider_gcp).where(cluster_providers_gcp: { status: ::Clusters::Providers::Gcp.state_machines[:status].states[:created].value }) }
+    scope :managed, -> { where(managed: true) }
 
     scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) }
 

app/models/clusters/platforms/kubernetes.rb

@@ -92,11 +92,12 @@ module Clusters
 
           if kubernetes_namespace = cluster.kubernetes_namespaces.has_service_account_token.find_by(project: project)
             variables.concat(kubernetes_namespace.predefined_variables)
-          elsif cluster.project_type?
-            # From 11.5, every Clusters::Project should have at least one
-            # Clusters::KubernetesNamespace, so once migration has been completed,
-            # this 'else' branch will be removed. For more information, please see
-            # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22433
+          elsif cluster.project_type? || !cluster.managed?
+            # As of 11.11 a user can create a cluster that they manage themselves,
+            # which replicates the existing project-level cluster behaviour.
+            # Once we have marked all project-level clusters that make use of this
+            # behaviour as "unmanaged", we can remove the `cluster.project_type?`
+            # check here.
             variables
               .append(key: 'KUBE_URL', value: api_url)
               .append(key: 'KUBE_TOKEN', value: token, public: false, masked: true)

app/services/clusters/refresh_service.rb

@@ -22,9 +22,9 @@ module Clusters
 
     def self.clusters_with_missing_kubernetes_namespaces_for_project(project)
       if Feature.enabled?(:ci_preparing_state, default_enabled: true)
-        project.clusters.missing_kubernetes_namespace(project.kubernetes_namespaces)
+        project.clusters.managed.missing_kubernetes_namespace(project.kubernetes_namespaces)
       else
-        project.all_clusters.missing_kubernetes_namespace(project.kubernetes_namespaces)
+        project.all_clusters.managed.missing_kubernetes_namespace(project.kubernetes_namespaces)
       end
     end
 

app/views/clusters/clusters/gcp/_form.html.haml

@@ -74,6 +74,13 @@
         = link_to _('More information'), help_page_path('user/project/clusters/index.md',
           anchor: 'role-based-access-control-rbac-core-only'), target: '_blank'
 
+  .form-group
+    = field.check_box :managed, { label: s_('ClusterIntegration|GitLab-managed cluster'),
+      label_class: 'label-bold' }
+    .form-text.text-muted
+      = s_('ClusterIntegration|Allow GitLab to manage namespace and service accounts for this cluster.')
+      = link_to _('More information'), help_page_path('user/project/clusters/index.md', anchor: 'gitlab-managed-clusters'), target: '_blank'
+
   .form-group
     = field.submit s_('ClusterIntegration|Create Kubernetes cluster'),
       class: 'js-gke-cluster-creation-submit btn btn-success', disabled: true

app/views/clusters/clusters/user/_form.html.haml

@@ -44,5 +44,12 @@
         { class: 'qa-rbac-checkbox', label: s_('ClusterIntegration|RBAC-enabled cluster'),
         label_class: 'label-bold', inline: true }, 'rbac', 'abac'
 
+  .form-group
+    = field.check_box :managed, { label: s_('ClusterIntegration|GitLab-managed cluster'),
+      label_class: 'label-bold' }
+    .form-text.text-muted
+      = s_('ClusterIntegration|Allow GitLab to manage namespace and service accounts for this cluster.')
+      = link_to _('More information'), help_page_path('user/project/clusters/index.md', anchor: 'gitlab-managed-clusters'), target: '_blank'
+
   .form-group
     = field.submit s_('ClusterIntegration|Add Kubernetes cluster'), class: 'btn btn-success'

app/views/clusters/platforms/kubernetes/_form.html.haml

@@ -47,5 +47,12 @@
         = s_('ClusterIntegration|Enable this setting if using role-based access control (RBAC).')
         = s_('ClusterIntegration|This option will allow you to install applications on RBAC clusters.')
 
+    .form-group
+      = field.check_box :managed, { disabled: true, label: s_('ClusterIntegration|GitLab-managed cluster'),
+        label_class: 'label-bold' }
+      .form-text.text-muted
+        = s_('ClusterIntegration|Allow GitLab to manage namespace and service accounts for this cluster.')
+        = link_to _('More information'), help_page_path('user/project/clusters/index.md', anchor: 'gitlab-managed-clusters'), target: '_blank'
+
   .form-group
     = field.submit s_('ClusterIntegration|Save changes'), class: 'btn btn-success'

app/workers/cluster_configure_worker.rb

@@ -5,7 +5,7 @@ class ClusterConfigureWorker
   include ClusterQueue
 
   def perform(cluster_id)
-    Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
+    Clusters::Cluster.managed.find_by_id(cluster_id).try do |cluster|
       if cluster.project_type? || Feature.disabled?(:ci_preparing_state, default_enabled: true)
         Clusters::RefreshService.create_or_update_namespaces_for_cluster(cluster)
       end

changelogs/unreleased/56557-disable-kubernetes-namespace-service-account-backend.yml

+---
+title: Disables kubernetes resources creation if a cluster is not managed
+merge_request: 26565
+author:
+type: added

doc/api/project_clusters.md

@@ -161,6 +161,7 @@ Parameters:
 | `name` | String | yes | The name of the cluster |
 | `domain` | String | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster |
 | `enabled` | Boolean | no | Determines if cluster is active or not, defaults to true |
+| `managed` | Boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true |
 | `platform_kubernetes_attributes[api_url]` | String | yes | The URL to access the Kubernetes API |
 | `platform_kubernetes_attributes[token]` | String | yes | The token to authenticate against Kubernetes |
 | `platform_kubernetes_attributes[ca_cert]` | String | no | TLS certificate (needed if API is using a self-signed TLS certificate |

doc/user/group/clusters/index.md

@@ -72,6 +72,29 @@ Add another cluster similar to the first one and make sure to
 [set an environment scope](#environment-scopes-premium) that will
 differentiate the new cluster from the rest.
 
+## Gitlab-managed clusters
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22011) in GitLab 11.5.
+> Became [optional](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/26565) in GitLab 11.11.
+
+NOTE: **Note:**
+Only available when creating clusters. Existing clusters not managed by GitLab
+cannot become GitLab-managed later.
+
+You can choose to allow GitLab to manage your cluster for you. If your cluster is
+managed by GitLab, resources for your projects will be automatically created. See the
+[Access controls](../../project/clusters/index.md#access-controls) section for details on which resources will
+be created.
+
+If you choose to manage your own cluster, project-specific resources will not be created
+automatically. If you are using [Auto DevOps](../../../topics/autodevops/index.md), you will
+need to explicitly provide the `KUBE_NAMESPACE` [deployment variable](../../project/clusters/index.md#deployment-variables)
+that will be used by your deployment jobs.
+
+NOTE: **Note:**
+If you [install applications](#installing-applications) on your cluster, GitLab will create
+the resources required to run these even if you have chosen to manage your own cluster.
+
 ## Base domain
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/24580) in GitLab 11.8.

doc/user/project/clusters/index.md

@@ -70,6 +70,7 @@ new Kubernetes cluster to your project:
    - **Machine type** - The [machine type](https://cloud.google.com/compute/docs/machine-types)
      of the Virtual Machine instance that the cluster will be based on.
    - **RBAC-enabled cluster** - Leave this checked if using default GKE creation options, see the [RBAC section](#role-based-access-control-rbac) for more information.
+   - **GitLab-managed cluster** - Leave this checked if you want GitLab to manage namespaces and service accounts for this cluster. See the [Managed clusters section](#gitlab-managed-clusters) for more information.
 1. Finally, click the **Create Kubernetes cluster** button.
 
 After a couple of minutes, your cluster will be ready to go. You can now proceed
@@ -188,6 +189,9 @@ To add an existing Kubernetes cluster to your project:
       role binding. You can follow the [Google Cloud
       documentation](https://cloud.google.com/iam/docs/granting-changing-revoking-access)
       to grant access.
+
+    - **GitLab-managed cluster** - Leave this checked if you want GitLab to manage namespaces and service accounts for this cluster. See the [Managed clusters section](#gitlab-managed-clusters) for more information.
+
     - **Project namespace** (optional) - You don't have to fill it in; by leaving
       it blank, GitLab will create one for you. Also:
        - Each project should have a unique namespace.
@@ -214,6 +218,29 @@ functionalities needed to successfully build and deploy a containerized
 application. Bear in mind that the same credentials are used for all the
 applications running on the cluster.
 
+## Gitlab-managed clusters
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22011) in GitLab 11.5.
+> Became [optional](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/26565) in GitLab 11.11.
+
+NOTE: **Note:**
+Only available when creating clusters. Existing clusters not managed by GitLab
+cannot become GitLab-managed later.
+
+You can choose to allow GitLab to manage your cluster for you. If your cluster is
+managed by GitLab, resources for your projects will be automatically created. See the
+[Access controls](#access-controls) section for details on which resources will
+be created.
+
+If you choose to manage your own cluster, project-specific resources will not be created
+automatically. If you are using [Auto DevOps](../../../topics/autodevops/index.md), you will
+need to explicitly provide the `KUBE_NAMESPACE` [deployment variable](#deployment-variables)
+that will be used by your deployment jobs, otherwise a namespace will be created for you.
+
+NOTE: **Note:**
+If you [install applications](#installing-applications) on your cluster, GitLab will create
+the resources required to run these even if you have chosen to manage your own cluster.
+
 ## Base domain
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/24580) in GitLab 11.8.
@@ -278,8 +305,8 @@ The following sections summarize which resources will be created on ABAC/RBAC cl
 | `gitlab-token`    | `Secret`             | Token for `gitlab` ServiceAccount | Creating a new GKE Cluster        |
 | `tiller`          | `ServiceAccount`     | `gitlab-managed-apps` namespace   | Installing Helm Tiller            |
 | `tiller-admin`    | `ClusterRoleBinding` | `cluster-admin` roleRef           | Installing Helm Tiller            |
-| Project namespace | `ServiceAccount`     | Uses namespace of Project         | Creating/Adding a new GKE Cluster |
-| Project namespace | `Secret`             | Token for project ServiceAccount  | Creating/Adding a new GKE Cluster |
+| Project namespace | `ServiceAccount`     | Uses namespace of Project         | Deploying to a cluster |
+| Project namespace | `Secret`             | Token for project ServiceAccount  | Deploying to a cluster |
 
 ### Role-based access control (RBAC)
 
@@ -290,9 +317,12 @@ The following sections summarize which resources will be created on ABAC/RBAC cl
 | `gitlab-token`      | `Secret`             | Token for `gitlab` ServiceAccount | Creating a new GKE Cluster        |
 | `tiller`            | `ServiceAccount`     | `gitlab-managed-apps` namespace   | Installing Helm Tiller            |
 | `tiller-admin`      | `ClusterRoleBinding` | `cluster-admin` roleRef           | Installing Helm Tiller            |
-| Project namespace   | `ServiceAccount`     | Uses namespace of Project         | Creating/Adding a new GKE Cluster |
-| Project namespace   | `Secret`             | Token for project ServiceAccount  | Creating/Adding a new GKE Cluster |
-| Project namespace   | `RoleBinding`        | [`edit`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) roleRef                  | Creating/Adding a new GKE Cluster |
+| Project namespace   | `ServiceAccount`     | Uses namespace of Project         | Deploying to a cluster |
+| Project namespace   | `Secret`             | Token for project ServiceAccount  | Deploying to a cluster |
+| Project namespace   | `RoleBinding`        | [`edit`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) roleRef                  | Deploying to a cluster |
+
+NOTE: **Note:**
+Project-specific resources are only created if your cluster is [managed by GitLab](#gitlab-managed-clusters).
 
 ### Security of GitLab Runners
 

lib/api/project_clusters.rb

@@ -54,6 +54,7 @@ module API
         requires :name, type: String, desc: 'Cluster name'
         optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true'
         optional :domain, type: String, desc: 'Cluster base domain'
+        optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true'
         requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do
           requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API'
           requires :token, type: String, desc: 'Token to authenticate against Kubernetes'

lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb

@@ -7,6 +7,7 @@ module Gitlab
         class KubernetesNamespace < Base
           def unmet?
             deployment_cluster.present? &&
+              deployment_cluster.managed? &&
               !deployment_cluster.project_type? &&
               kubernetes_namespace.new_record?
           end

locale/gitlab.pot

@@ -2017,6 +2017,9 @@ msgstr ""
 msgid "ClusterIntegration|All data will be deleted and cannot be restored."
 msgstr ""
 
+msgid "ClusterIntegration|Allow GitLab to manage namespace and service accounts for this cluster."
+msgstr ""
+
 msgid "ClusterIntegration|Alternatively"
 msgstr ""
 
@@ -2140,6 +2143,9 @@ msgstr ""
 msgid "ClusterIntegration|GitLab Runner connects to the repository and executes CI/CD jobs, pushing results back and deploying applications to production."
 msgstr ""
 
+msgid "ClusterIntegration|GitLab-managed cluster"
+msgstr ""
+
 msgid "ClusterIntegration|Google Cloud Platform project"
 msgstr ""
 

spec/controllers/groups/clusters_controller_spec.rb

@@ -189,6 +189,7 @@ describe Groups::ClustersController do
       {
         cluster: {
           name: 'new-cluster',
+          managed: '1',
           provider_gcp_attributes: {
             gcp_project_id: 'gcp-project-12345',
             legacy_abac: legacy_abac_param
@@ -218,6 +219,7 @@ describe Groups::ClustersController do
           expect(cluster).to be_gcp
           expect(cluster).to be_kubernetes
           expect(cluster.provider_gcp).to be_legacy_abac
+          expect(cluster).to be_managed
         end
 
         context 'when legacy_abac param is false' do
@@ -278,6 +280,7 @@ describe Groups::ClustersController do
       {
         cluster: {
           name: 'new-cluster',
+          managed: '1',
           platform_kubernetes_attributes: {
             api_url: 'http://my-url',
             token: 'test'
@@ -303,6 +306,7 @@ describe Groups::ClustersController do
           expect(response).to redirect_to(group_cluster_path(group, cluster))
           expect(cluster).to be_user
           expect(cluster).to be_kubernetes
+          expect(cluster).to be_managed
         end
       end
 
@@ -334,6 +338,29 @@ describe Groups::ClustersController do
           expect(cluster).to be_platform_kubernetes_rbac
         end
       end
+
+      context 'when creates a user-managed cluster' do
+        let(:params) do
+          {
+            cluster: {
+              name: 'new-cluster',
+              managed: '0',
+              platform_kubernetes_attributes: {
+                api_url: 'http://my-url',
+                token: 'test',
+                authorization_type: 'rbac'
+              }
+            }
+          }
+        end
+
+        it 'creates a new user-managed cluster' do
+          go
+
+          cluster = group.clusters.first
+          expect(cluster.managed?).to be_falsy
+        end
+      end
     end
 
     describe 'security' do

spec/controllers/projects/clusters_controller_spec.rb

@@ -165,6 +165,7 @@ describe Projects::ClustersController do
       {
         cluster: {
           name: 'new-cluster',
+          managed: '1',
           provider_gcp_attributes: {
             gcp_project_id: 'gcp-project-12345',
             legacy_abac: legacy_abac_param
@@ -191,6 +192,7 @@ describe Projects::ClustersController do
           expect(project.clusters.first).to be_gcp
           expect(project.clusters.first).to be_kubernetes
           expect(project.clusters.first.provider_gcp).to be_legacy_abac
+          expect(project.clusters.first.managed?).to be_truthy
         end
 
         context 'when legacy_abac param is false' do
@@ -251,6 +253,7 @@ describe Projects::ClustersController do
       {
         cluster: {
           name: 'new-cluster',
+          managed: '1',
           platform_kubernetes_attributes: {
             api_url: 'http://my-url',
             token: 'test',
@@ -302,9 +305,35 @@ describe Projects::ClustersController do
 
           expect(response).to redirect_to(project_cluster_path(project, project.clusters.first))
 
-          expect(project.clusters.first).to be_user
-          expect(project.clusters.first).to be_kubernetes
-          expect(project.clusters.first).to be_platform_kubernetes_rbac
+          cluster = project.clusters.first
+
+          expect(cluster).to be_user
+          expect(cluster).to be_kubernetes
+          expect(cluster).to be_platform_kubernetes_rbac
+        end
+      end
+
+      context 'when creates a user-managed cluster' do
+        let(:params) do
+          {
+            cluster: {
+              name: 'new-cluster',
+              managed: '0',
+              platform_kubernetes_attributes: {
+                api_url: 'http://my-url',
+                token: 'test',
+                namespace: 'aaa',
+                authorization_type: 'rbac'
+              }
+            }
+          }
+        end
+
+        it 'creates a new user-managed cluster' do
+          go
+          cluster = project.clusters.first
+
+          expect(cluster.managed?).to be_falsy
         end
       end
     end

spec/factories/clusters/clusters.rb

@@ -65,7 +65,7 @@ FactoryBot.define do
       domain 'example.com'
     end
 
-    trait :user_managed do
+    trait :not_managed do
       managed false
     end
   end

spec/lib/gitlab/ci/build/prerequisite/kubernetes_namespace_spec.rb

@@ -28,6 +28,12 @@ describe Gitlab::Ci::Build::Prerequisite::KubernetesNamespace do
 
         it { is_expected.to be_truthy }
 
+        context 'and the cluster is not managed' do
+          let(:cluster) { create(:cluster, :not_managed, projects: [build.project]) }
+
+          it { is_expected.to be_falsey }
+        end
+
         context 'and a namespace is already created for this project' do
           let!(:kubernetes_namespace) { create(:cluster_kubernetes_namespace, cluster: cluster, project: build.project) }
 

spec/models/clusters/cluster_spec.rb

@@ -95,6 +95,24 @@ describe Clusters::Cluster do
     it { is_expected.to contain_exactly(cluster) }
   end
 
+  describe '.managed' do
+    subject do
+      described_class.managed
+    end
+
+    context 'cluster is not managed' do
+      let!(:cluster) { create(:cluster, :not_managed) }
+
+      it { is_expected.not_to include(cluster) }
+    end
+
+    context 'cluster is managed' do
+      let!(:cluster) { create(:cluster) }
+
+      it { is_expected.to include(cluster) }
+    end
+  end
+
   describe '.missing_kubernetes_namespace' do
     let!(:cluster) { create(:cluster, :provided_by_gcp, :project) }
     let(:project) { cluster.project }

spec/models/clusters/platforms/kubernetes_spec.rb

@@ -331,6 +331,18 @@ describe Clusters::Platforms::Kubernetes, :use_clean_rails_memory_store_caching
             { key: 'KUBE_TOKEN', value: kubernetes.token, public: false }
           )
         end
+
+        context 'the cluster is not managed' do
+          let!(:cluster) { create(:cluster, :group, :not_managed, platform_kubernetes: kubernetes) }
+
+          it_behaves_like 'setting variables'
+
+          it 'sets KUBE_TOKEN' do
+            expect(subject).to include(
+              { key: 'KUBE_TOKEN', value: kubernetes.token, public: false, masked: true }
+            )
+          end
+        end
       end
 
       context 'kubernetes namespace exists for the project' do

spec/requests/api/project_clusters_spec.rb

@@ -189,6 +189,7 @@ describe API::ProjectClusters do
       {
         name: 'test-cluster',
         domain: 'domain.example.com',
+        managed: false,
         platform_kubernetes_attributes: platform_kubernetes_attributes
       }
     end
@@ -220,6 +221,7 @@ describe API::ProjectClusters do
           expect(cluster_result.project).to eq(project)
           expect(cluster_result.name).to eq('test-cluster')
           expect(cluster_result.domain).to eq('domain.example.com')
+          expect(cluster_result.managed).to be_falsy
           expect(platform_kubernetes.rbac?).to be_truthy
           expect(platform_kubernetes.api_url).to eq(api_url)
           expect(platform_kubernetes.namespace).to eq(namespace)

spec/services/clusters/refresh_service_spec.rb

@@ -121,5 +121,11 @@ describe Clusters::RefreshService do
         end
       end
     end
+
+    context 'cluster is not managed' do
+      let!(:cluster) { create(:cluster, :project, :not_managed, projects: [project]) }
+
+      include_examples 'does not create a kubernetes namespace'
+    end
   end
 end

spec/workers/cluster_configure_worker_spec.rb

@@ -68,6 +68,16 @@ describe ClusterConfigureWorker, '#perform' do
     it_behaves_like 'configured cluster'
   end
 
+  context 'when cluster is not managed' do
+    let(:cluster) { create(:cluster, :not_managed) }
+
+    it 'does not configure the cluster' do
+      expect(Clusters::RefreshService).not_to receive(:create_or_update_namespaces_for_cluster)
+
+      described_class.new.perform(cluster.id)
+    end
+  end
+
   context 'when cluster does not exist' do
     it 'does not provision a cluster' do
       expect_any_instance_of(Clusters::Gcp::Kubernetes::CreateOrUpdateNamespaceService).not_to receive(:execute)