Merge branch '322817-clean-up-old-expired-job-artifacts-2' into 'master'

Add locked column to Ci::JobArtifact

See merge request gitlab-org/gitlab!68893

app/models/ci/job_artifact.rb

@@ -235,6 +235,17 @@ module Ci
       hashed_path: 2
     }
 
+    # `locked` will be populated from the source of truth on Ci::Pipeline
+    # in order to clean up expired job artifacts in a performant way.
+    # The values should be the same as `Ci::Pipeline.lockeds` with the
+    # additional value of `unknown` to indicate rows that have not
+    # yet been populated from the parent Ci::Pipeline
+    enum locked: {
+      unlocked: 0,
+      artifacts_locked: 1,
+      unknown: 2
+    }, _prefix: :artifact
+
     def validate_file_format!
       unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym
         errors.add(:base, _('Invalid file format with specified file type'))

db/migrate/20210826120834_add_locked_to_ci_job_artifacts.rb

+# frozen_string_literal: true
+
+class AddLockedToCiJobArtifacts < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  TABLE_NAME = 'ci_job_artifacts'
+  COLUMN_NAME = 'locked'
+
+  def up
+    with_lock_retries do
+      add_column TABLE_NAME, COLUMN_NAME, :smallint, default: 2
+    end
+  end
+
+  def down
+    with_lock_retries do
+      remove_column TABLE_NAME, COLUMN_NAME
+    end
+  end
+end

db/post_migrate/20210826110839_prepare_indexes_for_ci_job_artifacts_expire_at_unlocked.rb

+# frozen_string_literal: true
+
+class PrepareIndexesForCiJobArtifactsExpireAtUnlocked < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  TABLE_NAME = 'ci_job_artifacts'
+  INDEX_NAME = 'ci_job_artifacts_expire_at_unlocked_idx'
+
+  def up
+    prepare_async_index TABLE_NAME, [:expire_at], where: 'locked = 0', name: INDEX_NAME
+  end
+
+  def down
+    unprepare_async_index_by_name TABLE_NAME, INDEX_NAME
+  end
+end

db/schema_migrations/20210826110839

+72b64ddbaf86eb296fe49fd38bea759d5247414142fe1cd11aee7e1d6171e142
\ No newline at end of file

db/schema_migrations/20210826120834

+a8cd5165815a2f1e6b825ea3ee2a9bde88c1790f6ebe92296bee6a9a892b83f2
\ No newline at end of file

db/structure.sql

@@ -11567,6 +11567,7 @@ CREATE TABLE ci_job_artifacts (
     file_location smallint,
     id bigint NOT NULL,
     job_id bigint NOT NULL,
+    locked smallint DEFAULT 2,
     CONSTRAINT check_27f0f6dbab CHECK ((file_store IS NOT NULL))
 );