Merge branch '56557-disable-kubernetes-namespace-service-account-backend' into 'master'

Add gitlab-managed option to clusters form

Closes #56557

See merge request gitlab-org/gitlab-ce!26565

app/controllers/clusters/clusters_controller.rb

@@ -156,6 +156,7 @@ class Clusters::ClustersController < Clusters::BaseController
       :enabled,
       :name,
       :environment_scope,
+      :managed,
       provider_gcp_attributes: [
         :gcp_project_id,
         :zone,
@@ -174,6 +175,7 @@ class Clusters::ClustersController < Clusters::BaseController
       :enabled,
       :name,
       :environment_scope,
+      :managed,
       platform_kubernetes_attributes: [
         :namespace,
         :api_url,

app/models/clusters/cluster.rb

@@ -94,6 +94,7 @@ module Clusters
     scope :user_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:user]) }
     scope :gcp_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:gcp]) }
     scope :gcp_installed, -> { gcp_provided.includes(:provider_gcp).where(cluster_providers_gcp: { status: ::Clusters::Providers::Gcp.state_machines[:status].states[:created].value }) }
+    scope :managed, -> { where(managed: true) }
 
     scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) }
 

app/models/clusters/platforms/kubernetes.rb

@@ -92,11 +92,12 @@ module Clusters
 
           if kubernetes_namespace = cluster.kubernetes_namespaces.has_service_account_token.find_by(project: project)
             variables.concat(kubernetes_namespace.predefined_variables)
-          elsif cluster.project_type?
-            # From 11.5, every Clusters::Project should have at least one
-            # Clusters::KubernetesNamespace, so once migration has been completed,
-            # this 'else' branch will be removed. For more information, please see
-            # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22433
+          elsif cluster.project_type? || !cluster.managed?
+            # As of 11.11 a user can create a cluster that they manage themselves,
+            # which replicates the existing project-level cluster behaviour.
+            # Once we have marked all project-level clusters that make use of this
+            # behaviour as "unmanaged", we can remove the `cluster.project_type?`
+            # check here.
             variables
               .append(key: 'KUBE_URL', value: api_url)
               .append(key: 'KUBE_TOKEN', value: token, public: false, masked: true)

app/services/clusters/refresh_service.rb

@@ -22,9 +22,9 @@ module Clusters
 
     def self.clusters_with_missing_kubernetes_namespaces_for_project(project)
       if Feature.enabled?(:ci_preparing_state, default_enabled: true)
-        project.clusters.missing_kubernetes_namespace(project.kubernetes_namespaces)
+        project.clusters.managed.missing_kubernetes_namespace(project.kubernetes_namespaces)
       else
-        project.all_clusters.missing_kubernetes_namespace(project.kubernetes_namespaces)
+        project.all_clusters.managed.missing_kubernetes_namespace(project.kubernetes_namespaces)
       end
     end
 

app/views/clusters/clusters/gcp/_form.html.haml

@@ -74,6 +74,13 @@
         = link_to _('More information'), help_page_path('user/project/clusters/index.md',
           anchor: 'role-based-access-control-rbac-core-only'), target: '_blank'
 
+  .form-group
+    = field.check_box :managed, { label: s_('ClusterIntegration|GitLab-managed cluster'),
+      label_class: 'label-bold' }
+    .form-text.text-muted
+      = s_('ClusterIntegration|Allow GitLab to manage namespace and service accounts for this cluster.')
+      = link_to _('More information'), help_page_path('user/project/clusters/index.md', anchor: 'gitlab-managed-clusters'), target: '_blank'
+
   .form-group
     = field.submit s_('ClusterIntegration|Create Kubernetes cluster'),
       class: 'js-gke-cluster-creation-submit btn btn-success', disabled: true

app/views/clusters/clusters/user/_form.html.haml

@@ -44,5 +44,12 @@
         { class: 'qa-rbac-checkbox', label: s_('ClusterIntegration|RBAC-enabled cluster'),
         label_class: 'label-bold', inline: true }, 'rbac', 'abac'
 
+  .form-group
+    = field.check_box :managed, { label: s_('ClusterIntegration|GitLab-managed cluster'),
+      label_class: 'label-bold' }
+    .form-text.text-muted
+      = s_('ClusterIntegration|Allow GitLab to manage namespace and service accounts for this cluster.')
+      = link_to _('More information'), help_page_path('user/project/clusters/index.md', anchor: 'gitlab-managed-clusters'), target: '_blank'
+
   .form-group
     = field.submit s_('ClusterIntegration|Add Kubernetes cluster'), class: 'btn btn-success'

app/views/clusters/platforms/kubernetes/_form.html.haml

@@ -47,5 +47,12 @@
         = s_('ClusterIntegration|Enable this setting if using role-based access control (RBAC).')
         = s_('ClusterIntegration|This option will allow you to install applications on RBAC clusters.')
 
+    .form-group
+      = field.check_box :managed, { disabled: true, label: s_('ClusterIntegration|GitLab-managed cluster'),
+        label_class: 'label-bold' }
+      .form-text.text-muted
+        = s_('ClusterIntegration|Allow GitLab to manage namespace and service accounts for this cluster.')
+        = link_to _('More information'), help_page_path('user/project/clusters/index.md', anchor: 'gitlab-managed-clusters'), target: '_blank'
+
   .form-group
     = field.submit s_('ClusterIntegration|Save changes'), class: 'btn btn-success'

app/workers/cluster_configure_worker.rb

@@ -5,7 +5,7 @@ class ClusterConfigureWorker
   include ClusterQueue
 
   def perform(cluster_id)
-    Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
+    Clusters::Cluster.managed.find_by_id(cluster_id).try do |cluster|
       if cluster.project_type? || Feature.disabled?(:ci_preparing_state, default_enabled: true)
         Clusters::RefreshService.create_or_update_namespaces_for_cluster(cluster)
       end

changelogs/unreleased/56557-disable-kubernetes-namespace-service-account-backend.yml

+---
+title: Disables kubernetes resources creation if a cluster is not managed
+merge_request: 26565
+author:
+type: added

doc/api/project_clusters.md

@@ -161,6 +161,7 @@ Parameters:
 | `name` | String | yes | The name of the cluster |
 | `domain` | String | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster |
 | `enabled` | Boolean | no | Determines if cluster is active or not, defaults to true |
+| `managed` | Boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true |
 | `platform_kubernetes_attributes[api_url]` | String | yes | The URL to access the Kubernetes API |
 | `platform_kubernetes_attributes[token]` | String | yes | The token to authenticate against Kubernetes |
 | `platform_kubernetes_attributes[ca_cert]` | String | no | TLS certificate (needed if API is using a self-signed TLS certificate |

doc/user/group/clusters/index.md

@@ -72,6 +72,29 @@ Add another cluster similar to the first one and make sure to
 [set an environment scope](#environment-scopes-premium) that will
 differentiate the new cluster from the rest.
 
+## Gitlab-managed clusters
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22011) in GitLab 11.5.
+> Became [optional](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/26565) in GitLab 11.11.
+
+NOTE: **Note:**
+Only available when creating clusters. Existing clusters not managed by GitLab
+cannot become GitLab-managed later.
+
+You can choose to allow GitLab to manage your cluster for you. If your cluster is
+managed by GitLab, resources for your projects will be automatically created. See the
+[Access controls](../../project/clusters/index.md#access-controls) section for details on which resources will
+be created.
+
+If you choose to manage your own cluster, project-specific resources will not be created
+automatically. If you are using [Auto DevOps](../../../topics/autodevops/index.md), you will
+need to explicitly provide the `KUBE_NAMESPACE` [deployment variable](../../project/clusters/index.md#deployment-variables)
+that will be used by your deployment jobs.
+
+NOTE: **Note:**
+If you [install applications](#installing-applications) on your cluster, GitLab will create
+the resources required to run these even if you have chosen to manage your own cluster.
+
 ## Base domain
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/24580) in GitLab 11.8.

doc/user/project/clusters/index.md

@@ -70,6 +70,7 @@ new Kubernetes cluster to your project:
    - **Machine type** - The [machine type](https://cloud.google.com/compute/docs/machine-types)
      of the Virtual Machine instance that the cluster will be based on.
    - **RBAC-enabled cluster** - Leave this checked if using default GKE creation options, see the [RBAC section](#role-based-access-control-rbac) for more information.
+   - **GitLab-managed cluster** - Leave this checked if you want GitLab to manage namespaces and service accounts for this cluster. See the [Managed clusters section](#gitlab-managed-clusters) for more information.
 1. Finally, click the **Create Kubernetes cluster** button.
 
 After a couple of minutes, your cluster will be ready to go. You can now proceed
@@ -188,6 +189,9 @@ To add an existing Kubernetes cluster to your project:
       role binding. You can follow the [Google Cloud
       documentation](https://cloud.google.com/iam/docs/granting-changing-revoking-access)
       to grant access.
+
+    - **GitLab-managed cluster** - Leave this checked if you want GitLab to manage namespaces and service accounts for this cluster. See the [Managed clusters section](#gitlab-managed-clusters) for more information.
+
     - **Project namespace** (optional) - You don't have to fill it in; by leaving
       it blank, GitLab will create one for you. Also:
        - Each project should have a unique namespace.
@@ -214,6 +218,29 @@ functionalities needed to successfully build and deploy a containerized
 application. Bear in mind that the same credentials are used for all the
 applications running on the cluster.
 
+## Gitlab-managed clusters
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22011) in GitLab 11.5.
+> Became [optional](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/26565) in GitLab 11.11.
+
+NOTE: **Note:**
+Only available when creating clusters. Existing clusters not managed by GitLab
+cannot become GitLab-managed later.
+
+You can choose to allow GitLab to manage your cluster for you. If your cluster is
+managed by GitLab, resources for your projects will be automatically created. See the
+[Access controls](#access-controls) section for details on which resources will
+be created.
+
+If you choose to manage your own cluster, project-specific resources will not be created
+automatically. If you are using [Auto DevOps](../../../topics/autodevops/index.md), you will
+need to explicitly provide the `KUBE_NAMESPACE` [deployment variable](#deployment-variables)
+that will be used by your deployment jobs, otherwise a namespace will be created for you.
+
+NOTE: **Note:**
+If you [install applications](#installing-applications) on your cluster, GitLab will create
+the resources required to run these even if you have chosen to manage your own cluster.
+
 ## Base domain
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/24580) in GitLab 11.8.
@@ -278,8 +305,8 @@ The following sections summarize which resources will be created on ABAC/RBAC cl
 | `gitlab-token`    | `Secret`             | Token for `gitlab` ServiceAccount | Creating a new GKE Cluster        |
 | `tiller`          | `ServiceAccount`     | `gitlab-managed-apps` namespace   | Installing Helm Tiller            |
 | `tiller-admin`    | `ClusterRoleBinding` | `cluster-admin` roleRef           | Installing Helm Tiller            |
-| Project namespace | `ServiceAccount`     | Uses namespace of Project         | Creating/Adding a new GKE Cluster |
-| Project namespace | `Secret`             | Token for project ServiceAccount  | Creating/Adding a new GKE Cluster |
+| Project namespace | `ServiceAccount`     | Uses namespace of Project         | Deploying to a cluster |
+| Project namespace | `Secret`             | Token for project ServiceAccount  | Deploying to a cluster |
 
 ### Role-based access control (RBAC)
 
@@ -290,9 +317,12 @@ The following sections summarize which resources will be created on ABAC/RBAC cl
 | `gitlab-token`      | `Secret`             | Token for `gitlab` ServiceAccount | Creating a new GKE Cluster        |
 | `tiller`            | `ServiceAccount`     | `gitlab-managed-apps` namespace   | Installing Helm Tiller            |
 | `tiller-admin`      | `ClusterRoleBinding` | `cluster-admin` roleRef           | Installing Helm Tiller            |
-| Project namespace   | `ServiceAccount`     | Uses namespace of Project         | Creating/Adding a new GKE Cluster |
-| Project namespace   | `Secret`             | Token for project ServiceAccount  | Creating/Adding a new GKE Cluster |
-| Project namespace   | `RoleBinding`        | [`edit`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) roleRef                  | Creating/Adding a new GKE Cluster |
+| Project namespace   | `ServiceAccount`     | Uses namespace of Project         | Deploying to a cluster |
+| Project namespace   | `Secret`             | Token for project ServiceAccount  | Deploying to a cluster |
+| Project namespace   | `RoleBinding`        | [`edit`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) roleRef                  | Deploying to a cluster |
+
+NOTE: **Note:**
+Project-specific resources are only created if your cluster is [managed by GitLab](#gitlab-managed-clusters).
 
 ### Security of GitLab Runners
 

lib/api/project_clusters.rb

@@ -54,6 +54,7 @@ module API
         requires :name, type: String, desc: 'Cluster name'
         optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true'
         optional :domain, type: String, desc: 'Cluster base domain'
+        optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true'
         requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do
           requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API'
           requires :token, type: String, desc: 'Token to authenticate against Kubernetes'

lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb

@@ -7,6 +7,7 @@ module Gitlab
         class KubernetesNamespace < Base
           def unmet?
             deployment_cluster.present? &&
+              deployment_cluster.managed? &&
               !deployment_cluster.project_type? &&
               kubernetes_namespace.new_record?
           end

locale/gitlab.pot

@@ -2017,6 +2017,9 @@ msgstr ""
 msgid "ClusterIntegration|All data will be deleted and cannot be restored."
 msgstr ""
 
+msgid "ClusterIntegration|Allow GitLab to manage namespace and service accounts for this cluster."
+msgstr ""
+
 msgid "ClusterIntegration|Alternatively"
 msgstr ""
 
@@ -2140,6 +2143,9 @@ msgstr ""
 msgid "ClusterIntegration|GitLab Runner connects to the repository and executes CI/CD jobs, pushing results back and deploying applications to production."
 msgstr ""
 
+msgid "ClusterIntegration|GitLab-managed cluster"
+msgstr ""
+
 msgid "ClusterIntegration|Google Cloud Platform project"
 msgstr ""
 

spec/controllers/groups/clusters_controller_spec.rb

@@ -189,6 +189,7 @@ describe Groups::ClustersController do
       {
         cluster: {
           name: 'new-cluster',
+          managed: '1',
           provider_gcp_attributes: {
             gcp_project_id: 'gcp-project-12345',
             legacy_abac: legacy_abac_param
@@ -218,6 +219,7 @@ describe Groups::ClustersController do
           expect(cluster).to be_gcp
           expect(cluster).to be_kubernetes
           expect(cluster.provider_gcp).to be_legacy_abac
+          expect(cluster).to be_managed
         end
 
         context 'when legacy_abac param is false' do
@@ -278,6 +280,7 @@ describe Groups::ClustersController do
       {
         cluster: {
           name: 'new-cluster',
+          managed: '1',
           platform_kubernetes_attributes: {
             api_url: 'http://my-url',
             token: 'test'
@@ -303,6 +306,7 @@ describe Groups::ClustersController do
           expect(response).to redirect_to(group_cluster_path(group, cluster))
           expect(cluster).to be_user
           expect(cluster).to be_kubernetes
+          expect(cluster).to be_managed
         end
       end
 
@@ -334,6 +338,29 @@ describe Groups::ClustersController do
           expect(cluster).to be_platform_kubernetes_rbac
         end
       end
+
+      context 'when creates a user-managed cluster' do
+        let(:params) do
+          {
+            cluster: {
+              name: 'new-cluster',
+              managed: '0',
+              platform_kubernetes_attributes: {
+                api_url: 'http://my-url',
+                token: 'test',
+                authorization_type: 'rbac'
+              }
+            }
+          }
+        end
+
+        it 'creates a new user-managed cluster' do
+          go
+
+          cluster = group.clusters.first
+          expect(cluster.managed?).to be_falsy
+        end
+      end
     end
 
     describe 'security' do

spec/controllers/projects/clusters_controller_spec.rb

@@ -165,6 +165,7 @@ describe Projects::ClustersController do
       {
         cluster: {
           name: 'new-cluster',
+          managed: '1',
           provider_gcp_attributes: {
             gcp_project_id: 'gcp-project-12345',
             legacy_abac: legacy_abac_param
@@ -191,6 +192,7 @@ describe Projects::ClustersController do
           expect(project.clusters.first).to be_gcp
           expect(project.clusters.first).to be_kubernetes
           expect(project.clusters.first.provider_gcp).to be_legacy_abac
+          expect(project.clusters.first.managed?).to be_truthy
         end
 
         context 'when legacy_abac param is false' do
@@ -251,6 +253,7 @@ describe Projects::ClustersController do
       {
         cluster: {
           name: 'new-cluster',
+          managed: '1',
           platform_kubernetes_attributes: {
             api_url: 'http://my-url',
             token: 'test',
@@ -302,9 +305,35 @@ describe Projects::ClustersController do
 
           expect(response).to redirect_to(project_cluster_path(project, project.clusters.first))
 
-          expect(project.clusters.first).to be_user
-          expect(project.clusters.first).to be_kubernetes
-          expect(project.clusters.first).to be_platform_kubernetes_rbac
+          cluster = project.clusters.first
+
+          expect(cluster).to be_user
+          expect(cluster).to be_kubernetes
+          expect(cluster).to be_platform_kubernetes_rbac
+        end
+      end
+
+      context 'when creates a user-managed cluster' do
+        let(:params) do
+          {
+            cluster: {
+              name: 'new-cluster',
+              managed: '0',
+              platform_kubernetes_attributes: {
+                api_url: 'http://my-url',
+                token: 'test',
+                namespace: 'aaa',
+                authorization_type: 'rbac'
+              }
+            }
+          }
+        end
+
+        it 'creates a new user-managed cluster' do
+          go
+          cluster = project.clusters.first
+
+          expect(cluster.managed?).to be_falsy
         end
       end
     end

spec/factories/clusters/clusters.rb

@@ -65,7 +65,7 @@ FactoryBot.define do
       domain 'example.com'
     end
 
-    trait :user_managed do
+    trait :not_managed do
       managed false
     end
   end

spec/lib/gitlab/ci/build/prerequisite/kubernetes_namespace_spec.rb

@@ -28,6 +28,12 @@ describe Gitlab::Ci::Build::Prerequisite::KubernetesNamespace do
 
         it { is_expected.to be_truthy }
 
+        context 'and the cluster is not managed' do
+          let(:cluster) { create(:cluster, :not_managed, projects: [build.project]) }
+
+          it { is_expected.to be_falsey }
+        end
+
         context 'and a namespace is already created for this project' do
           let!(:kubernetes_namespace) { create(:cluster_kubernetes_namespace, cluster: cluster, project: build.project) }
 

spec/models/clusters/cluster_spec.rb

@@ -95,6 +95,24 @@ describe Clusters::Cluster do
     it { is_expected.to contain_exactly(cluster) }
   end
 
+  describe '.managed' do
+    subject do
+      described_class.managed
+    end
+
+    context 'cluster is not managed' do
+      let!(:cluster) { create(:cluster, :not_managed) }
+
+      it { is_expected.not_to include(cluster) }
+    end
+
+    context 'cluster is managed' do
+      let!(:cluster) { create(:cluster) }
+
+      it { is_expected.to include(cluster) }
+    end
+  end
+
   describe '.missing_kubernetes_namespace' do
     let!(:cluster) { create(:cluster, :provided_by_gcp, :project) }
     let(:project) { cluster.project }

spec/models/clusters/platforms/kubernetes_spec.rb

@@ -331,6 +331,18 @@ describe Clusters::Platforms::Kubernetes, :use_clean_rails_memory_store_caching
             { key: 'KUBE_TOKEN', value: kubernetes.token, public: false }
           )
         end
+
+        context 'the cluster is not managed' do
+          let!(:cluster) { create(:cluster, :group, :not_managed, platform_kubernetes: kubernetes) }
+
+          it_behaves_like 'setting variables'
+
+          it 'sets KUBE_TOKEN' do
+            expect(subject).to include(
+              { key: 'KUBE_TOKEN', value: kubernetes.token, public: false, masked: true }
+            )
+          end
+        end
       end
 
       context 'kubernetes namespace exists for the project' do

spec/requests/api/project_clusters_spec.rb

@@ -189,6 +189,7 @@ describe API::ProjectClusters do
       {
         name: 'test-cluster',
         domain: 'domain.example.com',
+        managed: false,
         platform_kubernetes_attributes: platform_kubernetes_attributes
       }
     end
@@ -220,6 +221,7 @@ describe API::ProjectClusters do
           expect(cluster_result.project).to eq(project)
           expect(cluster_result.name).to eq('test-cluster')
           expect(cluster_result.domain).to eq('domain.example.com')
+          expect(cluster_result.managed).to be_falsy
           expect(platform_kubernetes.rbac?).to be_truthy
           expect(platform_kubernetes.api_url).to eq(api_url)
           expect(platform_kubernetes.namespace).to eq(namespace)

spec/services/clusters/refresh_service_spec.rb

@@ -121,5 +121,11 @@ describe Clusters::RefreshService do
         end
       end
     end
+
+    context 'cluster is not managed' do
+      let!(:cluster) { create(:cluster, :project, :not_managed, projects: [project]) }
+
+      include_examples 'does not create a kubernetes namespace'
+    end
   end
 end

spec/workers/cluster_configure_worker_spec.rb

@@ -68,6 +68,16 @@ describe ClusterConfigureWorker, '#perform' do
     it_behaves_like 'configured cluster'
   end
 
+  context 'when cluster is not managed' do
+    let(:cluster) { create(:cluster, :not_managed) }
+
+    it 'does not configure the cluster' do
+      expect(Clusters::RefreshService).not_to receive(:create_or_update_namespaces_for_cluster)
+
+      described_class.new.perform(cluster.id)
+    end
+  end
+
   context 'when cluster does not exist' do
     it 'does not provision a cluster' do
       expect_any_instance_of(Clusters::Gcp::Kubernetes::CreateOrUpdateNamespaceService).not_to receive(:execute)