Merge branch '223793-api-bulk-delete-project-artifacts' into 'master'

Add API to bulk delete project artifacts

See merge request gitlab-org/gitlab!75488

app/models/ci/job_artifact.rb

@@ -181,9 +181,7 @@ module Ci
     end
 
     scope :erasable, -> do
-      types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values
-
-      where(file_type: types)
+      where(file_type: self.erasable_file_types)
     end
 
     scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) }
@@ -263,6 +261,10 @@ module Ci
       [file_type]
     end
 
+    def self.erasable_file_types
+      self.file_types.keys - NON_ERASABLE_FILE_TYPES
+    end
+
     def self.total_size
       self.sum(:size)
     end

app/services/ci/job_artifacts/delete_project_artifacts_service.rb

+# frozen_string_literal: true
+
+module Ci
+  module JobArtifacts
+    class DeleteProjectArtifactsService < BaseProjectService
+      def execute
+        ExpireProjectBuildArtifactsWorker.perform_async(project.id)
+      end
+    end
+  end
+end

app/services/ci/job_artifacts/expire_project_build_artifacts_service.rb

+# frozen_string_literal: true
+
+module Ci
+  module JobArtifacts
+    class ExpireProjectBuildArtifactsService
+      BATCH_SIZE = 1000
+
+      def initialize(project_id, expiry_time)
+        @project_id = project_id
+        @expiry_time = expiry_time
+      end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def execute
+        scope = Ci::JobArtifact.for_project(project_id).order(:id)
+        file_type_values = Ci::JobArtifact.erasable_file_types.map { |file_type| [Ci::JobArtifact.file_types[file_type]] }
+        from_sql = Arel::Nodes::Grouping.new(Arel::Nodes::ValuesList.new(file_type_values)).as('file_types (file_type)').to_sql
+        array_scope = Ci::JobArtifact.from(from_sql).select(:file_type)
+        array_mapping_scope = -> (file_type_expression) { Ci::JobArtifact.where(Ci::JobArtifact.arel_table[:file_type].eq(file_type_expression)) }
+
+        Gitlab::Pagination::Keyset::Iterator
+          .new(scope: scope, in_operator_optimization_options: { array_scope: array_scope, array_mapping_scope: array_mapping_scope })
+          .each_batch(of: BATCH_SIZE) do |batch|
+          ids = batch.reselect!(:id).to_a.map(&:id)
+          Ci::JobArtifact.unlocked.where(id: ids).update_all(locked: Ci::JobArtifact.lockeds[:unlocked], expire_at: expiry_time)
+        end
+      end
+      # rubocop:enable CodeReuse/ActiveRecord
+
+      private
+
+      attr_reader :project_id, :expiry_time
+    end
+  end
+end

app/workers/all_queues.yml

@@ -1987,6 +1987,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: ci_job_artifacts_expire_project_build_artifacts
+  :worker_name: Ci::JobArtifacts::ExpireProjectBuildArtifactsWorker
+  :feature_category: :build_artifacts
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: create_commit_signature
   :worker_name: CreateCommitSignatureWorker
   :feature_category: :source_code_management

app/workers/ci/job_artifacts/expire_project_build_artifacts_worker.rb

+# frozen_string_literal: true
+
+module Ci
+  module JobArtifacts
+    class ExpireProjectBuildArtifactsWorker
+      include ApplicationWorker
+
+      data_consistency :always
+
+      feature_category :build_artifacts
+      idempotent!
+
+      def perform(project_id)
+        return unless Project.id_in(project_id).exists?
+
+        ExpireProjectBuildArtifactsService.new(project_id, Time.current).execute
+      end
+    end
+  end
+end

config/feature_flags/development/bulk_expire_project_artifacts.yml

+---
+name: bulk_expire_project_artifacts
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75488
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/347405
+milestone: '14.6'
+type: development
+group: group::testing
+default_enabled: false

config/sidekiq_queues.yml

@@ -73,6 +73,8 @@
   - 1
 - - ci_delete_objects
   - 1
+- - ci_job_artifacts_expire_project_build_artifacts
+  - 1
 - - ci_upstream_projects_subscriptions_cleanup
   - 1
 - - container_repository

db/post_migrate/20211207081708_add_index_ci_job_artifacts_project_id_file_type.rb

+# frozen_string_literal: true
+
+class AddIndexCiJobArtifactsProjectIdFileType < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_ci_job_artifacts_on_id_project_id_and_file_type'
+
+  def up
+    add_concurrent_index :ci_job_artifacts, [:project_id, :file_type, :id], name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME
+  end
+end

db/schema_migrations/20211207081708

+e26065e63eca51e4138b6e9f07e9ec1ee45838afa82c5832849e360375beeae2
\ No newline at end of file

db/structure.sql

@@ -25465,6 +25465,8 @@ CREATE INDEX index_ci_job_artifacts_on_file_store ON ci_job_artifacts USING btre
 
 CREATE INDEX index_ci_job_artifacts_on_file_type_for_devops_adoption ON ci_job_artifacts USING btree (file_type, project_id, created_at) WHERE (file_type = ANY (ARRAY[5, 6, 8, 23]));
 
+CREATE INDEX index_ci_job_artifacts_on_id_project_id_and_file_type ON ci_job_artifacts USING btree (project_id, file_type, id);
+
 CREATE UNIQUE INDEX index_ci_job_artifacts_on_job_id_and_file_type ON ci_job_artifacts USING btree (job_id, file_type);
 
 CREATE INDEX index_ci_job_artifacts_on_project_id ON ci_job_artifacts USING btree (project_id);

doc/api/job_artifacts.md

@@ -259,7 +259,7 @@ Example response:
 }
 ```
 
-## Delete artifacts
+## Delete job artifacts
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25522) in GitLab 11.9.
 
@@ -284,3 +284,34 @@ NOTE:
 At least Maintainer role is required to delete artifacts.
 
 If the artifacts were deleted successfully, a response with status `204 No Content` is returned.
+
+## Delete project artifacts
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223793) in GitLab 14.7 [with a flag](../administration/feature_flags.md) named `bulk_expire_project_artifacts`. Disabled by default.
+
+FLAG:
+On self-managed GitLab, by default this feature is not available. To make it
+available, ask an administrator to [enable the `bulk_expire_project_artifacts` flag](../administration/feature_flags.md).
+On GitLab.com, this feature is not available.
+
+[Expire artifacts of a project that can be deleted](https://gitlab.com/gitlab-org/gitlab/-/issues/223793) but that don't have an expiry time.
+
+```plaintext
+DELETE /projects/:id/artifacts
+```
+
+| Attribute | Type           | Required | Description                                                                 |
+|-----------|----------------|----------|-----------------------------------------------------------------------------|
+| `id`      | integer/string | yes      | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) |
+
+Example request:
+
+```shell
+curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/artifacts"
+```
+
+NOTE:
+At least Maintainer role is required to delete artifacts.
+
+Schedules a worker to update to the current time the expiry of all artifacts that can be deleted.
+A response with status `202 Accepted` is returned.

lib/api/ci/job_artifacts.rb

@@ -137,6 +137,17 @@ module API
 
           status :no_content
         end
+
+        desc 'Expire the artifacts files from a project'
+        delete ':id/artifacts' do
+          not_found! unless Feature.enabled?(:bulk_expire_project_artifacts, default_enabled: :yaml)
+
+          authorize_destroy_artifacts!
+
+          ::Ci::JobArtifacts::DeleteProjectArtifactsService.new(project: user_project).execute
+
+          accepted!
+        end
       end
     end
   end

spec/factories/ci/job_artifacts.rb

@@ -10,6 +10,10 @@ FactoryBot.define do
       expire_at { Date.yesterday }
     end
 
+    trait :locked do
+      locked { Ci::JobArtifact.lockeds[:artifacts_locked] }
+    end
+
     trait :remote_store do
       file_store { JobArtifactUploader::Store::REMOTE}
     end

spec/factories/ci/pipelines.rb

@@ -87,6 +87,10 @@ FactoryBot.define do
         locked { Ci::Pipeline.lockeds[:unlocked] }
       end
 
+      trait :artifacts_locked do
+        locked { Ci::Pipeline.lockeds[:artifacts_locked] }
+      end
+
       trait :protected do
         add_attribute(:protected) { true }
       end

spec/models/ci/job_artifact_spec.rb

@@ -143,6 +143,17 @@ RSpec.describe Ci::JobArtifact do
     end
   end
 
+  describe '.erasable_file_types' do
+    subject { described_class.erasable_file_types }
+
+    it 'returns a list of erasable file types' do
+      all_types = described_class.file_types.keys
+      erasable_types = all_types - described_class::NON_ERASABLE_FILE_TYPES
+
+      expect(subject).to contain_exactly(*erasable_types)
+    end
+  end
+
   describe '.erasable' do
     subject { described_class.erasable }
 

spec/requests/api/ci/job_artifacts_spec.rb

@@ -81,6 +81,71 @@ RSpec.describe API::Ci::JobArtifacts do
     end
   end
 
+  describe 'DELETE /projects/:id/artifacts' do
+    context 'when feature flag is disabled' do
+      before do
+        stub_feature_flags(bulk_expire_project_artifacts: false)
+      end
+
+      it 'returns 404' do
+        delete api("/projects/#{project.id}/artifacts", api_user)
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when user is anonymous' do
+      let(:api_user) { nil }
+
+      it 'does not execute Ci::JobArtifacts::DeleteProjectArtifactsService' do
+        expect(Ci::JobArtifacts::DeleteProjectArtifactsService)
+          .not_to receive(:new)
+
+        delete api("/projects/#{project.id}/artifacts", api_user)
+      end
+
+      it 'returns status 401 (unauthorized)' do
+        delete api("/projects/#{project.id}/artifacts", api_user)
+
+        expect(response).to have_gitlab_http_status(:unauthorized)
+      end
+    end
+
+    context 'with developer' do
+      it 'does not execute Ci::JobArtifacts::DeleteProjectArtifactsService' do
+        expect(Ci::JobArtifacts::DeleteProjectArtifactsService)
+          .not_to receive(:new)
+
+        delete api("/projects/#{project.id}/artifacts", api_user)
+      end
+
+      it 'returns status 403 (forbidden)' do
+        delete api("/projects/#{project.id}/artifacts", api_user)
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+
+    context 'with authorized user' do
+      let(:maintainer) { create(:project_member, :maintainer, project: project).user }
+      let!(:api_user) { maintainer }
+
+      it 'executes Ci::JobArtifacts::DeleteProjectArtifactsService' do
+        expect_next_instance_of(Ci::JobArtifacts::DeleteProjectArtifactsService, project: project) do |service|
+          expect(service).to receive(:execute).and_call_original
+        end
+
+        delete api("/projects/#{project.id}/artifacts", api_user)
+      end
+
+      it 'returns status 202 (accepted)' do
+        delete api("/projects/#{project.id}/artifacts", api_user)
+
+        expect(response).to have_gitlab_http_status(:accepted)
+      end
+    end
+  end
+
   describe 'GET /projects/:id/jobs/:job_id/artifacts/:artifact_path' do
     context 'when job has artifacts' do
       let(:job) { create(:ci_build, :artifacts, pipeline: pipeline) }

spec/services/ci/job_artifacts/delete_project_artifacts_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::JobArtifacts::DeleteProjectArtifactsService do
+  let_it_be(:project) { create(:project) }
+
+  subject { described_class.new(project: project) }
+
+  describe '#execute' do
+    it 'enqueues a Ci::ExpireProjectBuildArtifactsWorker' do
+      expect(Ci::JobArtifacts::ExpireProjectBuildArtifactsWorker).to receive(:perform_async).with(project.id).and_call_original
+
+      subject.execute
+    end
+  end
+end

spec/services/ci/job_artifacts/expire_project_build_artifacts_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::JobArtifacts::ExpireProjectBuildArtifactsService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:pipeline, reload: true) { create(:ci_pipeline, :unlocked, project: project) }
+
+  let(:expiry_time) { Time.current }
+
+  RSpec::Matchers.define :have_locked_status do |expected_status|
+    match do |job_artifacts|
+      predicate = "#{expected_status}?".to_sym
+      job_artifacts.all? { |artifact| artifact.__send__(predicate) }
+    end
+  end
+
+  RSpec::Matchers.define :expire_at do |expected_expiry|
+    match do |job_artifacts|
+      job_artifacts.all? { |artifact| artifact.expire_at.to_i == expected_expiry.to_i }
+    end
+  end
+
+  RSpec::Matchers.define :have_no_expiry do
+    match do |job_artifacts|
+      job_artifacts.all? { |artifact| artifact.expire_at.nil? }
+    end
+  end
+
+  describe '#execute' do
+    subject(:execute) { described_class.new(project.id, expiry_time).execute }
+
+    context 'with job containing erasable artifacts' do
+      let_it_be(:job, reload: true) { create(:ci_build, :erasable, pipeline: pipeline) }
+
+      it 'unlocks erasable job artifacts' do
+        execute
+
+        expect(job.job_artifacts).to have_locked_status(:artifact_unlocked)
+      end
+
+      it 'expires erasable job artifacts' do
+        execute
+
+        expect(job.job_artifacts).to expire_at(expiry_time)
+      end
+    end
+
+    context 'with job containing trace artifacts' do
+      let_it_be(:job, reload: true) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+      it 'does not unlock trace artifacts' do
+        execute
+
+        expect(job.job_artifacts).to have_locked_status(:artifact_unknown)
+      end
+
+      it 'does not expire trace artifacts' do
+        execute
+
+        expect(job.job_artifacts).to have_no_expiry
+      end
+    end
+
+    context 'with job from artifact locked pipeline' do
+      let_it_be(:job, reload: true) { create(:ci_build, pipeline: pipeline) }
+      let_it_be(:locked_artifact, reload: true) { create(:ci_job_artifact, :locked, job: job) }
+
+      before do
+        pipeline.artifacts_locked!
+      end
+
+      it 'does not unlock locked artifacts' do
+        execute
+
+        expect(job.job_artifacts).to have_locked_status(:artifact_artifacts_locked)
+      end
+
+      it 'does not expire locked artifacts' do
+        execute
+
+        expect(job.job_artifacts).to have_no_expiry
+      end
+    end
+
+    context 'with job containing both erasable and trace artifacts' do
+      let_it_be(:job, reload: true) { create(:ci_build, pipeline: pipeline) }
+      let_it_be(:erasable_artifact, reload: true) { create(:ci_job_artifact, :archive, job: job) }
+      let_it_be(:trace_artifact, reload: true) { create(:ci_job_artifact, :trace, job: job) }
+
+      it 'unlocks erasable artifacts' do
+        execute
+
+        expect(erasable_artifact.artifact_unlocked?).to be_truthy
+      end
+
+      it 'expires erasable artifacts' do
+        execute
+
+        expect(erasable_artifact.expire_at.to_i).to eq(expiry_time.to_i)
+      end
+
+      it 'does not unlock trace artifacts' do
+        execute
+
+        expect(trace_artifact.artifact_unlocked?).to be_falsey
+      end
+
+      it 'does not expire trace artifacts' do
+        execute
+
+        expect(trace_artifact.expire_at).to be_nil
+      end
+    end
+
+    context 'with multiple pipelines' do
+      let_it_be(:job, reload: true) { create(:ci_build, :erasable, pipeline: pipeline) }
+
+      let_it_be(:pipeline2, reload: true) { create(:ci_pipeline, :unlocked, project: project) }
+      let_it_be(:job2, reload: true) { create(:ci_build, :erasable, pipeline: pipeline) }
+
+      it 'unlocks artifacts across pipelines' do
+        execute
+
+        expect(job.job_artifacts).to have_locked_status(:artifact_unlocked)
+        expect(job2.job_artifacts).to have_locked_status(:artifact_unlocked)
+      end
+
+      it 'expires artifacts across pipelines' do
+        execute
+
+        expect(job.job_artifacts).to expire_at(expiry_time)
+        expect(job2.job_artifacts).to expire_at(expiry_time)
+      end
+    end
+
+    context 'with artifacts belonging to another project' do
+      let_it_be(:job, reload: true) { create(:ci_build, :erasable, pipeline: pipeline) }
+
+      let_it_be(:another_project, reload: true) { create(:project) }
+      let_it_be(:another_pipeline, reload: true) { create(:ci_pipeline, project: another_project) }
+      let_it_be(:another_job, reload: true) { create(:ci_build, :erasable, pipeline: another_pipeline) }
+
+      it 'does not unlock erasable artifacts in other projects' do
+        execute
+
+        expect(another_job.job_artifacts).to have_locked_status(:artifact_unknown)
+      end
+
+      it 'does not expire erasable artifacts in other projects' do
+        execute
+
+        expect(another_job.job_artifacts).to have_no_expiry
+      end
+    end
+  end
+end

spec/workers/ci/job_artifacts/expire_project_build_artifacts_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::JobArtifacts::ExpireProjectBuildArtifactsWorker do
+  let(:worker) { described_class.new }
+  let(:current_time) { Time.current }
+
+  let_it_be(:project) { create(:project) }
+
+  around do |example|
+    freeze_time { example.run }
+  end
+
+  describe '#perform' do
+    it 'executes ExpireProjectArtifactsService service with the project' do
+      expect_next_instance_of(Ci::JobArtifacts::ExpireProjectBuildArtifactsService, project.id, current_time) do |instance|
+        expect(instance).to receive(:execute).and_call_original
+      end
+
+      worker.perform(project.id)
+    end
+
+    context 'when project does not exist' do
+      it 'does nothing' do
+        expect(Ci::JobArtifacts::ExpireProjectBuildArtifactsService).not_to receive(:new)
+
+        worker.perform(non_existing_record_id)
+      end
+    end
+  end
+end