Merge branch...

Merge branch '40418-migrate-existing-data-from-kubernetesservice-to-clusters-platforms-kubernetes' into 'master'

Migrate existing data from KubernetesService to Clusters::Platforms::Kubernetes

Closes #40418

See merge request gitlab-org/gitlab-ce!15589

changelogs/unreleased/40418-migrate-existing-data-from-kubernetesservice-to-clusters-platforms-kubernetes.yml

+---
+title: Migrate existing data from KubernetesService to Clusters::Platforms::Kubernetes
+merge_request: 15589
+author:
+type: changed

db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb

+class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME = 'KubernetesService'.freeze
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :clusters, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    has_many :services, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service'
+    has_one :kubernetes_service, -> { where(category: 'deployment', type: 'KubernetesService') }, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service', inverse_of: :project, foreign_key: :project_id
+  end
+
+  class Cluster < ActiveRecord::Base
+    self.table_name = 'clusters'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :projects, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+    has_one :platform_kubernetes, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::PlatformsKubernetes'
+
+    accepts_nested_attributes_for :platform_kubernetes
+
+    enum platform_type: {
+      kubernetes: 1
+    }
+
+    enum provider_type: {
+      user: 0,
+      gcp: 1
+    }
+  end
+
+  class ClustersProject < ActiveRecord::Base
+    self.table_name = 'cluster_projects'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+  end
+
+  class PlatformsKubernetes < ActiveRecord::Base
+    self.table_name = 'cluster_platforms_kubernetes'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+
+    attr_encrypted :token,
+      mode: :per_attribute_iv,
+      key: Gitlab::Application.secrets.db_key_base,
+      algorithm: 'aes-256-cbc'
+  end
+
+  class Service < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'services'
+    self.inheritance_column = :_type_disabled # Disable STI, otherwise KubernetesModel will be looked up
+
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project', foreign_key: :project_id
+
+    scope :unmanaged_kubernetes_service, -> do
+      joins('LEFT JOIN projects ON projects.id = services.project_id')
+      .joins('LEFT JOIN cluster_projects ON cluster_projects.project_id = projects.id')
+      .joins('LEFT JOIN cluster_platforms_kubernetes ON cluster_platforms_kubernetes.cluster_id = cluster_projects.cluster_id')
+      .where(category: 'deployment', type: 'KubernetesService', template: false)
+      .where("services.properties LIKE '%api_url%'")
+      .where("(services.properties NOT LIKE CONCAT('%', cluster_platforms_kubernetes.api_url, '%')) OR cluster_platforms_kubernetes.api_url IS NULL")
+      .group(:id)
+      .order(id: :asc)
+    end
+
+    scope :kubernetes_service_without_template, -> do
+      where(category: 'deployment', type: 'KubernetesService', template: false)
+    end
+
+    def api_url
+      parsed_properties['api_url']
+    end
+
+    def ca_pem
+      parsed_properties['ca_pem']
+    end
+
+    def namespace
+      parsed_properties['namespace']
+    end
+
+    def token
+      parsed_properties['token']
+    end
+
+    private
+
+    def parsed_properties
+      @parsed_properties ||= JSON.parse(self.properties)
+    end
+  end
+
+  def find_dedicated_environement_scope(project)
+    environment_scopes = project.clusters.map(&:environment_scope)
+
+    return '*' if environment_scopes.exclude?('*') # KubernetesService should be added as a default cluster (environment_scope: '*') at first place
+    return 'migrated/*' if environment_scopes.exclude?('migrated/*') # If it's conflicted, the KubernetesService added as a migrated cluster
+
+    unique_iid = 0
+
+    # If it's still conflicted, finding an unique environment scope incrementaly
+    loop do
+      candidate = "migrated#{unique_iid}/*"
+      return candidate if environment_scopes.exclude?(candidate)
+
+      unique_iid += 1
+    end
+  end
+
+  def up
+    ActiveRecord::Base.transaction do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service
+        .unmanaged_kubernetes_service.find_each(batch_size: 1) do |kubernetes_service|
+        MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create(
+          enabled: kubernetes_service.active,
+          user_id: nil, # KubernetesService doesn't have
+          name: DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME,
+          provider_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.provider_types[:user],
+          platform_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.platform_types[:kubernetes],
+          projects: [kubernetes_service.project],
+          environment_scope: find_dedicated_environement_scope(kubernetes_service.project),
+          platform_kubernetes_attributes: {
+            api_url: kubernetes_service.api_url,
+            ca_cert: kubernetes_service.ca_pem,
+            namespace: kubernetes_service.namespace,
+            username: nil, # KubernetesService doesn't have
+            encrypted_password: nil, # KubernetesService doesn't have
+            encrypted_password_iv: nil, # KubernetesService doesn't have
+            token: kubernetes_service.token # encrypted_token and encrypted_token_iv
+          } )
+      end
+    end
+
+    MigrateKubernetesServiceToNewClustersArchitectures::Service
+      .kubernetes_service_without_template.each_batch(of: 100) do |kubernetes_service|
+      kubernetes_service.update_all(active: false)
+    end
+  end
+
+  def down
+    # noop
+  end
+end

spec/migrations/migrate_kubernetes_service_to_new_clusters_architectures_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb')
+
+describe MigrateKubernetesServiceToNewClustersArchitectures, :migration do
+  context 'when unique KubernetesService exists' do
+    shared_examples 'KubernetesService migration' do
+      let(:sample_num) { 2 }
+
+      let(:projects) do
+        (1..sample_num).each_with_object([]) do |n, array|
+          array << MigrateKubernetesServiceToNewClustersArchitectures::Project.create!
+        end
+      end
+
+      let!(:kubernetes_services) do
+        projects.map do |project|
+          MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+            project: project,
+            active: active,
+            category: 'deployment',
+            type: 'KubernetesService',
+            properties: "{\"namespace\":\"prod\",\"api_url\":\"https://kubernetes#{project.id}.com\",\"ca_pem\":\"ca_pem#{project.id}\",\"token\":\"token#{project.id}\"}")
+        end
+      end
+
+      it 'migrates the KubernetesService to Platform::Kubernetes' do
+        expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(sample_num)
+
+        projects.each do |project|
+          project.clusters.last.tap do |cluster|
+            expect(cluster.enabled).to eq(active)
+            expect(cluster.platform_kubernetes.api_url).to eq(project.kubernetes_service.api_url)
+            expect(cluster.platform_kubernetes.ca_cert).to eq(project.kubernetes_service.ca_pem)
+            expect(cluster.platform_kubernetes.token).to eq(project.kubernetes_service.token)
+            expect(project.kubernetes_service).not_to be_active
+          end
+        end
+      end
+    end
+
+    context 'when KubernetesService is active' do
+      let(:active) { true }
+
+      it_behaves_like 'KubernetesService migration'
+    end
+  end
+
+  context 'when unique KubernetesService spawned from Service Template' do
+    let(:sample_num) { 2 }
+
+    let(:projects) do
+      (1..sample_num).each_with_object([]) do |n, array|
+        array << MigrateKubernetesServiceToNewClustersArchitectures::Project.create!
+      end
+    end
+
+    let!(:kubernetes_service_template) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        template: true,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"namespace\":\"prod\",\"api_url\":\"https://sample.kubernetes.com\",\"ca_pem\":\"ca_pem-sample\",\"token\":\"token-sample\"}")
+    end
+
+    let!(:kubernetes_services) do
+      projects.map do |project|
+        MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+          project: project,
+          category: 'deployment',
+          type: 'KubernetesService',
+          properties: "{\"namespace\":\"prod\",\"api_url\":\"#{kubernetes_service_template.api_url}\",\"ca_pem\":\"#{kubernetes_service_template.ca_pem}\",\"token\":\"#{kubernetes_service_template.token}\"}")
+      end
+    end
+
+    it 'migrates the KubernetesService to Platform::Kubernetes without template' do
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(sample_num)
+
+      projects.each do |project|
+        project.clusters.last.tap do |cluster|
+          expect(cluster.platform_kubernetes.api_url).to eq(project.kubernetes_service.api_url)
+          expect(cluster.platform_kubernetes.ca_cert).to eq(project.kubernetes_service.ca_pem)
+          expect(cluster.platform_kubernetes.token).to eq(project.kubernetes_service.token)
+          expect(project.kubernetes_service).not_to be_active
+        end
+      end
+    end
+  end
+
+  context 'when managed KubernetesService exists' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    let(:cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:kubernetes_service) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: cluster.enabled,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"api_url\":\"#{cluster.platform_kubernetes.api_url}\"}")
+    end
+
+    it 'does not migrate the KubernetesService and disables the kubernetes_service' do # Because the corresponding Platform::Kubernetes already exists
+      expect { migrate! }.not_to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }
+
+      kubernetes_service.reload
+      expect(kubernetes_service).not_to be_active
+    end
+  end
+
+  context 'when production cluster has already been existed' do # i.e. There are no environment_scope conflicts
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    let(:cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        environment_scope: 'production/*',
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:kubernetes_service) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: true,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"api_url\":\"https://debug.kube.com\"}")
+    end
+
+    it 'migrates the KubernetesService to Platform::Kubernetes' do
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(1)
+
+      kubernetes_service.reload
+      project.clusters.last.tap do |cluster|
+        expect(cluster.environment_scope).to eq('*')
+        expect(cluster.platform_kubernetes.api_url).to eq(kubernetes_service.api_url)
+        expect(cluster.platform_kubernetes.ca_cert).to eq(kubernetes_service.ca_pem)
+        expect(cluster.platform_kubernetes.token).to eq(kubernetes_service.token)
+        expect(kubernetes_service).not_to be_active
+      end
+    end
+  end
+
+  context 'when default cluster has already been existed' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    let!(:cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        environment_scope: '*',
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:kubernetes_service) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: true,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"api_url\":\"https://debug.kube.com\"}")
+    end
+
+    it 'migrates the KubernetesService to Platform::Kubernetes with dedicated environment_scope' do # Because environment_scope is duplicated
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(1)
+
+      kubernetes_service.reload
+      project.clusters.last.tap do |cluster|
+        expect(cluster.environment_scope).to eq('migrated/*')
+        expect(cluster.platform_kubernetes.api_url).to eq(kubernetes_service.api_url)
+        expect(cluster.platform_kubernetes.ca_cert).to eq(kubernetes_service.ca_pem)
+        expect(cluster.platform_kubernetes.token).to eq(kubernetes_service.token)
+        expect(kubernetes_service).not_to be_active
+      end
+    end
+  end
+
+  context 'when default cluster and migrated cluster has already been existed' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    let!(:cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        environment_scope: '*',
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:migrated_cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        environment_scope: 'migrated/*',
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:kubernetes_service) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: true,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"api_url\":\"https://debug.kube.com\"}")
+    end
+
+    it 'migrates the KubernetesService to Platform::Kubernetes with dedicated environment_scope' do # Because environment_scope is duplicated
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(1)
+
+      kubernetes_service.reload
+      project.clusters.last.tap do |cluster|
+        expect(cluster.environment_scope).to eq('migrated0/*')
+        expect(cluster.platform_kubernetes.api_url).to eq(kubernetes_service.api_url)
+        expect(cluster.platform_kubernetes.ca_cert).to eq(kubernetes_service.ca_pem)
+        expect(cluster.platform_kubernetes.token).to eq(kubernetes_service.token)
+        expect(kubernetes_service).not_to be_active
+      end
+    end
+  end
+
+  context 'when KubernetesService has nullified parameters' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    before do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: false,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{}")
+    end
+
+    it 'does not migrate the KubernetesService and disables the kubernetes_service' do
+      expect { migrate! }.not_to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }
+
+      expect(project.kubernetes_service).not_to be_active
+    end
+  end
+
+  # Platforms::Kubernetes validates `token` reagdless of the activeness,
+  # whereas KubernetesService validates `token` if only it's activated
+  # However, in this migration file, there are no validations because of the re-defined model class
+  # therefore, we should safely add this raw to Platform::Kubernetes
+  context 'when KubernetesService has empty token' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    before do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: false,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"namespace\":\"prod\",\"api_url\":\"http://111.111.111.111\",\"ca_pem\":\"a\",\"token\":\"\"}")
+    end
+
+    it 'does not migrate the KubernetesService and disables the kubernetes_service' do
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(1)
+
+      project.clusters.last.tap do |cluster|
+        expect(cluster.environment_scope).to eq('*')
+        expect(cluster.platform_kubernetes.namespace).to eq('prod')
+        expect(cluster.platform_kubernetes.api_url).to eq('http://111.111.111.111')
+        expect(cluster.platform_kubernetes.ca_cert).to eq('a')
+        expect(cluster.platform_kubernetes.token).to be_empty
+        expect(project.kubernetes_service).not_to be_active
+      end
+    end
+  end
+
+  context 'when KubernetesService does not exist' do
+    let!(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    it 'does not migrate the KubernetesService' do
+      expect { migrate! }.not_to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }
+    end
+  end
+end