Merge branch 'mc/backstage/create-artifact-expiry-backfill-migration' into 'master'

Create artifact expiry backfill migration

See merge request gitlab-org/gitlab!47723

changelogs/unreleased/mc-backstage-create-artifact-expiry-backfill-migration.yml

+---
+title: Backfill artifact expiry date.
+merge_request: 47723
+author:
+type: other

db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb

+# frozen_string_literal: true
+
+class ScheduleBackfillingArtifactExpiryMigration < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  SWITCH_DATE = Time.utc(2020, 6, 22).freeze
+  INDEX_NAME = 'expired_artifacts_temp_index'.freeze
+  INDEX_CONDITION = "expire_at IS NULL AND created_at < '#{SWITCH_DATE}'"
+
+  disable_ddl_transaction!
+
+  class JobArtifact < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'ci_job_artifacts'
+
+    scope :without_expiry_date, -> { where(expire_at: nil) }
+    scope :before_switch, -> { where('created_at < ?', SWITCH_DATE) }
+  end
+
+  def up
+    # Create temporary index for expired artifacts
+    # Needs to be removed in a later migration
+    add_concurrent_index(:ci_job_artifacts, %i(id created_at), where: INDEX_CONDITION, name: INDEX_NAME)
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      JobArtifact.without_expiry_date.before_switch,
+      ::Gitlab::BackgroundMigration::BackfillArtifactExpiryDate,
+      2.minutes,
+      batch_size: 200_000
+    )
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME
+  end
+end

db/schema_migrations/20201208175117

+68971e7f9a722e98d9e611f614b5465de83ff3d4dc8c7a8078ed1db8f21e6590
\ No newline at end of file

db/structure.sql

@@ -20807,6 +20807,8 @@ CREATE UNIQUE INDEX epic_user_mentions_on_epic_id_and_note_id_index ON epic_user
 
 CREATE UNIQUE INDEX epic_user_mentions_on_epic_id_index ON epic_user_mentions USING btree (epic_id) WHERE (note_id IS NULL);
 
+CREATE INDEX expired_artifacts_temp_index ON ci_job_artifacts USING btree (id, created_at) WHERE ((expire_at IS NULL) AND (created_at < '2020-06-22 00:00:00+00'::timestamp with time zone));
+
 CREATE INDEX finding_links_on_vulnerability_occurrence_id ON vulnerability_finding_links USING btree (vulnerability_occurrence_id);
 
 CREATE INDEX idx_audit_events_on_entity_id_desc_author_id_created_at ON audit_events_archived USING btree (entity_id, entity_type, id DESC, author_id, created_at);

lib/gitlab/background_migration/backfill_artifact_expiry_date.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfill expire_at for a range of Ci::JobArtifact
+    class BackfillArtifactExpiryDate
+      include Gitlab::Utils::StrongMemoize
+
+      BATCH_SIZE = 1_000
+      DEFAULT_EXPIRATION_SWITCH_DATE = Date.new(2020, 6, 22).freeze
+      OLD_ARTIFACT_AGE = 15.months
+      OLD_ARTIFACT_EXPIRY_OFFSET = 3.months
+      RECENT_ARTIFACT_EXPIRY_OFFSET = 1.year
+
+      # Ci::JobArtifact model
+      class Ci::JobArtifact < ActiveRecord::Base
+        include ::EachBatch
+
+        self.table_name = 'ci_job_artifacts'
+
+        scope :between, -> (start_id, end_id) { where(id: start_id..end_id) }
+        scope :before_default_expiration_switch, -> { where('created_at < ?', DEFAULT_EXPIRATION_SWITCH_DATE) }
+        scope :without_expiry_date, -> { where(expire_at: nil) }
+        scope :old, -> { where(self.arel_table[:created_at].lt(OLD_ARTIFACT_AGE.ago)) }
+        scope :recent, -> { where(self.arel_table[:created_at].gt(OLD_ARTIFACT_AGE.ago)) }
+      end
+
+      def perform(start_id, end_id)
+        Ci::JobArtifact.between(start_id, end_id)
+          .without_expiry_date.before_default_expiration_switch
+          .each_batch(of: BATCH_SIZE) do |batch|
+          batch.old.update_all(expire_at: old_artifact_expiry_date)
+          batch.recent.update_all(expire_at: recent_artifact_expiry_date)
+        end
+      end
+
+      private
+
+      def offset_date
+        strong_memoize(:offset_date) do
+          current_date = Time.current
+          target_date = Time.zone.local(current_date.year, current_date.month, 22, 0, 0, 0)
+
+          current_date.day < 22 ? target_date : target_date.next_month
+        end
+      end
+
+      def old_artifact_expiry_date
+        offset_date + OLD_ARTIFACT_EXPIRY_OFFSET
+      end
+
+      def recent_artifact_expiry_date
+        offset_date + RECENT_ARTIFACT_EXPIRY_OFFSET
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_artifact_expiry_date_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillArtifactExpiryDate, :migration, schema: 20201111152859 do
+  subject(:perform) { migration.perform(1, 99) }
+
+  let(:migration) { described_class.new }
+  let(:artifact_outside_id_range) { create_artifact!(id: 100, created_at: 1.year.ago, expire_at: nil) }
+  let(:artifact_outside_date_range) { create_artifact!(id: 40, created_at: Time.current, expire_at: nil) }
+  let(:old_artifact) { create_artifact!(id: 10, created_at: 16.months.ago, expire_at: nil) }
+  let(:recent_artifact) { create_artifact!(id: 20, created_at: 1.year.ago, expire_at: nil) }
+  let(:artifact_with_expiry) { create_artifact!(id: 30, created_at: 1.year.ago, expire_at: Time.current + 1.day) }
+
+  before do
+    table(:namespaces).create!(id: 1, name: 'the-namespace', path: 'the-path')
+    table(:projects).create!(id: 1, name: 'the-project', namespace_id: 1)
+    table(:ci_builds).create!(id: 1, allow_failure: false)
+  end
+
+  context 'when current date is before the 22nd' do
+    before do
+      travel_to(Time.zone.local(2020, 1, 1, 0, 0, 0))
+    end
+
+    it 'backfills the expiry date for old artifacts' do
+      expect(old_artifact.reload.expire_at).to eq(nil)
+
+      perform
+
+      expect(old_artifact.reload.expire_at).to be_within(1.minute).of(Time.zone.local(2020, 4, 22, 0, 0, 0))
+    end
+
+    it 'backfills the expiry date for recent artifacts' do
+      expect(recent_artifact.reload.expire_at).to eq(nil)
+
+      perform
+
+      expect(recent_artifact.reload.expire_at).to be_within(1.minute).of(Time.zone.local(2021, 1, 22, 0, 0, 0))
+    end
+  end
+
+  context 'when current date is after the 22nd' do
+    before do
+      travel_to(Time.zone.local(2020, 1, 23, 0, 0, 0))
+    end
+
+    it 'backfills the expiry date for old artifacts' do
+      expect(old_artifact.reload.expire_at).to eq(nil)
+
+      perform
+
+      expect(old_artifact.reload.expire_at).to be_within(1.minute).of(Time.zone.local(2020, 5, 22, 0, 0, 0))
+    end
+
+    it 'backfills the expiry date for recent artifacts' do
+      expect(recent_artifact.reload.expire_at).to eq(nil)
+
+      perform
+
+      expect(recent_artifact.reload.expire_at).to be_within(1.minute).of(Time.zone.local(2021, 2, 22, 0, 0, 0))
+    end
+  end
+
+  it 'does not touch artifacts with expiry date' do
+    expect { perform }.not_to change { artifact_with_expiry.reload.expire_at }
+  end
+
+  it 'does not touch artifacts outside id range' do
+    expect { perform }.not_to change { artifact_outside_id_range.reload.expire_at }
+  end
+
+  it 'does not touch artifacts outside date range' do
+    expect { perform }.not_to change { artifact_outside_date_range.reload.expire_at }
+  end
+
+  private
+
+  def create_artifact!(**args)
+    table(:ci_job_artifacts).create!(**args, project_id: 1, job_id: 1, file_type: 1)
+  end
+end