Merge branch 'mc/backstage/schedule-artifact-expiration-backfill-properly' into 'master'

Schedule artifact expiry date backfill

See merge request gitlab-org/gitlab!51822

changelogs/unreleased/mc-backstage-schedule-artifact-expiration-backfill-properly.yml

+---
+title: Schedule artifact expiry date backfill background jobs.
+merge_request: 51822
+author:
+type: fixed

db/post_migrate/20210115220610_schedule_artifact_expiry_backfill.rb

+# frozen_string_literal: true
+
+class ScheduleArtifactExpiryBackfill < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'BackfillArtifactExpiryDate'.freeze
+  SWITCH_DATE = Date.new(2020, 06, 22).freeze
+  INDEX_NAME = 'expired_artifacts_temp_index'.freeze
+  OLD_INDEX_CONDITION = "expire_at IS NULL AND created_at < '#{SWITCH_DATE}'"
+  INDEX_CONDITION = "expire_at IS NULL AND date(created_at AT TIME ZONE 'UTC') < '2020-06-22'::date".freeze
+
+  disable_ddl_transaction!
+
+  class JobArtifact < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'ci_job_artifacts'
+
+    scope :without_expiry_date, -> { where(expire_at: nil) }
+    scope :before_switch, -> { where("date(created_at AT TIME ZONE 'UTC') < ?::date", SWITCH_DATE) }
+  end
+
+  def up
+    # Create temporary index for expired artifacts
+    # Needs to be removed in a later migration
+    remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME
+    add_concurrent_index(:ci_job_artifacts, %i(id created_at), where: INDEX_CONDITION, name: INDEX_NAME)
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      JobArtifact.without_expiry_date.before_switch,
+      MIGRATION,
+      2.minutes,
+      batch_size: 200_000
+    )
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME
+    add_concurrent_index(:ci_job_artifacts, %i(id created_at), where: OLD_INDEX_CONDITION, name: INDEX_NAME)
+
+    Gitlab::BackgroundMigration.steal(MIGRATION) do |job|
+      job.delete
+
+      false
+    end
+  end
+end

db/schema_migrations/20210115220610

+541a6626d3afd4fd421fd59fe5eb8ab7764952ae780c83c3805fd4a29e3f42fb
\ No newline at end of file

db/structure.sql

@@ -21047,7 +21047,7 @@ CREATE UNIQUE INDEX epic_user_mentions_on_epic_id_and_note_id_index ON epic_user
 
 CREATE UNIQUE INDEX epic_user_mentions_on_epic_id_index ON epic_user_mentions USING btree (epic_id) WHERE (note_id IS NULL);
 
-CREATE INDEX expired_artifacts_temp_index ON ci_job_artifacts USING btree (id, created_at) WHERE ((expire_at IS NULL) AND (created_at < '2020-06-22 00:00:00+00'::timestamp with time zone));
+CREATE INDEX expired_artifacts_temp_index ON ci_job_artifacts USING btree (id, created_at) WHERE ((expire_at IS NULL) AND (date(timezone('UTC'::text, created_at)) < '2020-06-22'::date));
 
 CREATE INDEX finding_links_on_vulnerability_occurrence_id ON vulnerability_finding_links USING btree (vulnerability_occurrence_id);