Merge branch 'copy-deployment-cluster-data' into 'master'

Copy deployments.cluster_id to deployment_clusters

See merge request gitlab-org/gitlab!26841

db/migrate/20200406102111_add_index_to_deployments_where_cluster_id_is_not_null.rb

+# frozen_string_literal: true
+
+class AddIndexToDeploymentsWhereClusterIdIsNotNull < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :deployments, :id, where: 'cluster_id IS NOT NULL', name: 'index_deployments_on_id_where_cluster_id_present'
+  end
+
+  def down
+    remove_concurrent_index :deployments, :id, where: 'cluster_id IS NOT NULL', name: 'index_deployments_on_id_where_cluster_id_present'
+  end
+end

db/post_migrate/20200406102120_backfill_deployment_clusters_from_deployments.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class BackfillDeploymentClustersFromDeployments < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'BackfillDeploymentClustersFromDeployments'
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 10_000
+
+  disable_ddl_transaction!
+
+  class Deployment < ActiveRecord::Base
+    include EachBatch
+
+    default_scope { where('cluster_id IS NOT NULL') }
+
+    self.table_name = 'deployments'
+  end
+
+  def up
+    say "Scheduling `#{MIGRATION}` jobs"
+
+    queue_background_migration_jobs_by_range_at_intervals(Deployment, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # NOOP
+  end
+end

db/structure.sql

@@ -9087,6 +9087,8 @@ CREATE INDEX index_deployments_on_environment_id_and_status ON public.deployment
 
 CREATE INDEX index_deployments_on_id_and_status ON public.deployments USING btree (id, status);
 
+CREATE INDEX index_deployments_on_id_where_cluster_id_present ON public.deployments USING btree (id) WHERE (cluster_id IS NOT NULL);
+
 CREATE INDEX index_deployments_on_project_id_and_id ON public.deployments USING btree (project_id, id DESC);
 
 CREATE UNIQUE INDEX index_deployments_on_project_id_and_iid ON public.deployments USING btree (project_id, iid);
@@ -13137,6 +13139,8 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200403184110
 20200403185127
 20200403185422
+20200406102111
+20200406102120
 20200406135648
 20200406165950
 20200406171857

lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfill deployment_clusters for a range of deployments
+    class BackfillDeploymentClustersFromDeployments
+      def perform(start_id, end_id)
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO deployment_clusters (deployment_id, cluster_id)
+            SELECT deployments.id, deployments.cluster_id
+            FROM deployments
+            WHERE deployments.cluster_id IS NOT NULL
+              AND deployments.id BETWEEN #{start_id} AND #{end_id}
+            ON CONFLICT DO NOTHING
+        SQL
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::BackfillDeploymentClustersFromDeployments, :migration, schema: 20200227140242 do
+  subject { described_class.new }
+
+  describe '#perform' do
+    it 'backfills deployment_cluster for all deployments in the given range with a non-null cluster_id' do
+      deployment_clusters = table(:deployment_clusters)
+
+      namespace = table(:namespaces).create(name: 'the-namespace', path: 'the-path')
+      project = table(:projects).create(name: 'the-project', namespace_id: namespace.id)
+      environment = table(:environments).create(name: 'the-environment', project_id: project.id, slug: 'slug')
+      cluster = table(:clusters).create(name: 'the-cluster')
+
+      deployment_data = { cluster_id: cluster.id, project_id: project.id, environment_id: environment.id, ref: 'abc', tag: false, sha: 'sha', status: 1 }
+      expected_deployment_1 = create_deployment(**deployment_data)
+      create_deployment(**deployment_data, cluster_id: nil) # no cluster_id
+      expected_deployment_2 = create_deployment(**deployment_data)
+      out_of_range_deployment = create_deployment(**deployment_data, cluster_id: cluster.id) # expected to be out of range
+
+      # to test "ON CONFLICT DO NOTHING"
+      existing_record_for_deployment_2 = deployment_clusters.create(
+        deployment_id: expected_deployment_2.id,
+        cluster_id: expected_deployment_2.cluster_id,
+        kubernetes_namespace: 'production'
+      )
+
+      subject.perform(expected_deployment_1.id, out_of_range_deployment.id - 1)
+
+      expect(deployment_clusters.all.pluck(:deployment_id, :cluster_id, :kubernetes_namespace)).to contain_exactly(
+        [expected_deployment_1.id, cluster.id, nil],
+        [expected_deployment_2.id, cluster.id, existing_record_for_deployment_2.kubernetes_namespace]
+      )
+    end
+
+    def create_deployment(**data)
+      @iid ||= 0
+      @iid += 1
+      table(:deployments).create(iid: @iid, **data)
+    end
+  end
+end

spec/migrations/20200406102120_backfill_deployment_clusters_from_deployments_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200406102120_backfill_deployment_clusters_from_deployments.rb')
+
+describe BackfillDeploymentClustersFromDeployments, :migration, :sidekiq, schema: 20200227140242 do
+  describe '#up' do
+    it 'schedules BackfillDeploymentClustersFromDeployments background jobs' do
+      stub_const("#{described_class}::BATCH_SIZE", 2)
+
+      namespace = table(:namespaces).create(name: 'the-namespace', path: 'the-path')
+      project = table(:projects).create(name: 'the-project', namespace_id: namespace.id)
+      environment = table(:environments).create(name: 'the-environment', project_id: project.id, slug: 'slug')
+      cluster = table(:clusters).create(name: 'the-cluster')
+
+      deployment_data = { cluster_id: cluster.id, project_id: project.id, environment_id: environment.id, ref: 'abc', tag: false, sha: 'sha', status: 1 }
+
+      # batch 1
+      batch_1_begin = create_deployment(**deployment_data)
+      batch_1_end = create_deployment(**deployment_data)
+
+      # value that should not be included due to default scope
+      create_deployment(**deployment_data, cluster_id: nil)
+
+      # batch 2
+      batch_2_begin = create_deployment(**deployment_data)
+      batch_2_end = create_deployment(**deployment_data)
+
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          migrate!
+
+          # batch 1
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, batch_1_begin.id, batch_1_end.id)
+
+          # batch 2
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, batch_2_begin.id, batch_2_end.id)
+
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+        end
+      end
+    end
+
+    def create_deployment(**data)
+      @iid ||= 0
+      @iid += 1
+      table(:deployments).create(iid: @iid, **data)
+    end
+  end
+end