Merge branch 'mb-ci-throttle-pipeline-creation' into 'master'

Throttle pipelines creation rate

See merge request gitlab-org/gitlab!83969

app/services/ci/create_pipeline_service.rb

@@ -14,6 +14,7 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Build::Associations,
                 Gitlab::Ci::Pipeline::Chain::Validate::Abilities,
                 Gitlab::Ci::Pipeline::Chain::Validate::Repository,
+                Gitlab::Ci::Pipeline::Chain::Limit::RateLimit,
                 Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy,
                 Gitlab::Ci::Pipeline::Chain::Skip,
                 Gitlab::Ci::Pipeline::Chain::Config::Content,

config/feature_flags/development/ci_throttle_pipelines_creation.yml

+---
+name: ci_throttle_pipelines_creation
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83969
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/357520
+milestone: '14.10'
+type: development
+group: group::pipeline execution
+default_enabled: false

config/feature_flags/development/ci_throttle_pipelines_creation_dry_run.yml

+---
+name: ci_throttle_pipelines_creation_dry_run
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83969
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/357524
+milestone: '14.10'
+type: development
+group: group::pipeline execution
+default_enabled: true

lib/gitlab/application_rate_limiter.rb

@@ -41,7 +41,8 @@ module Gitlab
           auto_rollback_deployment:           { threshold: 1, interval: 3.minutes },
           search_rate_limit:                  { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute },
           search_rate_limit_unauthenticated:  { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute },
-          gitlab_shell_operation:       { threshold: 600, interval: 1.minute }
+          gitlab_shell_operation:       { threshold: 600, interval: 1.minute },
+          pipelines_create:             { threshold: 4, interval: 1.minute }
         }.freeze
       end
 

lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Limit
+          class RateLimit < Chain::Base
+            include Chain::Helpers
+
+            def perform!
+              return unless throttle_enabled?
+
+              # We exclude child-pipelines from the rate limit because they represent
+              # sub-pipelines that would otherwise hit the rate limit due to having the
+              # same scope (project, user, sha).
+              #
+              return if pipeline.parent_pipeline?
+
+              if rate_limit_throttled?
+                create_log_entry
+                error(throttle_message) unless dry_run?
+              end
+            end
+
+            def break?
+              @pipeline.errors.any?
+            end
+
+            private
+
+            def rate_limit_throttled?
+              ::Gitlab::ApplicationRateLimiter.throttled?(
+                :pipelines_create, scope: [project, current_user, command.sha]
+              )
+            end
+
+            def create_log_entry
+              Gitlab::AppJsonLogger.info(
+                class: self.class.name,
+                namespace_id: project.namespace_id,
+                project_id: project.id,
+                commit_sha: command.sha,
+                current_user_id: current_user.id,
+                subscription_plan: project.actual_plan_name,
+                message: 'Activated pipeline creation rate limit'
+              )
+            end
+
+            def throttle_message
+              'Too many pipelines created in the last minute. Try again later.'
+            end
+
+            def throttle_enabled?
+              ::Feature.enabled?(
+                :ci_throttle_pipelines_creation,
+                project,
+                default_enabled: :yaml)
+            end
+
+            def dry_run?
+              ::Feature.enabled?(
+                :ci_throttle_pipelines_creation_dry_run,
+                project,
+                default_enabled: :yaml)
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/pipeline/chain/limit/rate_limit_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ::Gitlab::Ci::Pipeline::Chain::Limit::RateLimit, :freeze_time, :clean_gitlab_redis_rate_limiting do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:namespace) { create(:namespace) }
+  let_it_be(:project, reload: true) { create(:project, namespace: namespace) }
+
+  let(:save_incompleted) { false }
+  let(:throttle_message) do
+    'Too many pipelines created in the last minute. Try again later.'
+  end
+
+  let(:command) do
+    Gitlab::Ci::Pipeline::Chain::Command.new(
+      project: project,
+      current_user: user,
+      save_incompleted: save_incompleted
+    )
+  end
+
+  let(:pipeline) { build(:ci_pipeline, project: project, source: source) }
+  let(:source) { 'push' }
+  let(:step) { described_class.new(pipeline, command) }
+
+  def perform(count: 2)
+    count.times { step.perform! }
+  end
+
+  context 'when the limit is exceeded' do
+    before do
+      allow(Gitlab::ApplicationRateLimiter).to receive(:rate_limits)
+        .and_return(pipelines_create: { threshold: 1, interval: 1.minute })
+
+      stub_feature_flags(ci_throttle_pipelines_creation_dry_run: false)
+    end
+
+    it 'does not persist the pipeline' do
+      perform
+
+      expect(pipeline).not_to be_persisted
+      expect(pipeline.errors.added?(:base, throttle_message)).to be_truthy
+    end
+
+    it 'breaks the chain' do
+      perform
+
+      expect(step.break?).to be_truthy
+    end
+
+    it 'creates a log entry' do
+      expect(Gitlab::AppJsonLogger).to receive(:info).with(
+        a_hash_including(
+          class: described_class.name,
+          project_id: project.id,
+          subscription_plan: project.actual_plan_name,
+          commit_sha: command.sha
+        )
+      )
+
+      perform
+    end
+
+    context 'with child pipelines' do
+      let(:source) { 'parent_pipeline' }
+
+      it 'does not break the chain' do
+        perform
+
+        expect(step.break?).to be_falsey
+      end
+
+      it 'does not invalidate the pipeline' do
+        perform
+
+        expect(pipeline.errors).to be_empty
+      end
+
+      it 'does not log anything' do
+        expect(Gitlab::AppJsonLogger).not_to receive(:info)
+
+        perform
+      end
+    end
+
+    context 'when saving incompleted pipelines' do
+      let(:save_incompleted) { true }
+
+      it 'does not persist the pipeline' do
+        perform
+
+        expect(pipeline).not_to be_persisted
+        expect(pipeline.errors.added?(:base, throttle_message)).to be_truthy
+      end
+
+      it 'breaks the chain' do
+        perform
+
+        expect(step.break?).to be_truthy
+      end
+    end
+
+    context 'when ci_throttle_pipelines_creation is disabled' do
+      before do
+        stub_feature_flags(ci_throttle_pipelines_creation: false)
+      end
+
+      it 'does not break the chain' do
+        perform
+
+        expect(step.break?).to be_falsey
+      end
+
+      it 'does not invalidate the pipeline' do
+        perform
+
+        expect(pipeline.errors).to be_empty
+      end
+
+      it 'does not log anything' do
+        expect(Gitlab::AppJsonLogger).not_to receive(:info)
+
+        perform
+      end
+    end
+
+    context 'when ci_throttle_pipelines_creation_dry_run is enabled' do
+      before do
+        stub_feature_flags(ci_throttle_pipelines_creation_dry_run: true)
+      end
+
+      it 'does not break the chain' do
+        perform
+
+        expect(step.break?).to be_falsey
+      end
+
+      it 'does not invalidate the pipeline' do
+        perform
+
+        expect(pipeline.errors).to be_empty
+      end
+
+      it 'creates a log entry' do
+        expect(Gitlab::AppJsonLogger).to receive(:info).with(
+          a_hash_including(
+            class: described_class.name,
+            project_id: project.id,
+            subscription_plan: project.actual_plan_name,
+            commit_sha: command.sha
+          )
+        )
+
+        perform
+      end
+    end
+  end
+
+  context 'when the limit is not exceeded' do
+    it 'does not break the chain' do
+      perform
+
+      expect(step.break?).to be_falsey
+    end
+
+    it 'does not invalidate the pipeline' do
+      perform
+
+      expect(pipeline.errors).to be_empty
+    end
+
+    it 'does not log anything' do
+      expect(Gitlab::AppJsonLogger).not_to receive(:info)
+
+      perform
+    end
+  end
+end

spec/services/ci/create_pipeline_service/rate_limit_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Ci::CreatePipelineService, :freeze_time, :clean_gitlab_redis_rate_limiting do
+  describe 'rate limiting' do
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:user)    { project.first_owner }
+
+    let(:ref) { 'refs/heads/master' }
+
+    before do
+      stub_ci_pipeline_yaml_file(gitlab_ci_yaml)
+      stub_feature_flags(ci_throttle_pipelines_creation_dry_run: false)
+
+      allow(Gitlab::ApplicationRateLimiter).to receive(:rate_limits)
+        .and_return(pipelines_create: { threshold: 1, interval: 1.minute })
+    end
+
+    context 'when user is under the limit' do
+      let(:pipeline) { create_pipelines(count: 1) }
+
+      it 'allows pipeline creation' do
+        expect(pipeline).to be_created_successfully
+        expect(pipeline.statuses).not_to be_empty
+      end
+    end
+
+    context 'when user is over the limit' do
+      let(:pipeline) { create_pipelines }
+
+      it 'blocks pipeline creation' do
+        throttle_message = 'Too many pipelines created in the last minute. Try again later.'
+
+        expect(pipeline).not_to be_persisted
+        expect(pipeline.statuses).to be_empty
+        expect(pipeline.errors.added?(:base, throttle_message)).to be_truthy
+      end
+    end
+
+    context 'with different users' do
+      let(:other_user) { create(:user) }
+
+      before do
+        project.add_maintainer(other_user)
+      end
+
+      it 'allows other members to create pipelines' do
+        blocked_pipeline = create_pipelines(user: user)
+        allowed_pipeline = create_pipelines(count: 1, user: other_user)
+
+        expect(blocked_pipeline).not_to be_persisted
+        expect(allowed_pipeline).to be_created_successfully
+      end
+    end
+
+    context 'with different commits' do
+      it 'allows user to create pipeline' do
+        blocked_pipeline = create_pipelines(ref: ref)
+        allowed_pipeline = create_pipelines(count: 1, ref: 'refs/heads/feature')
+
+        expect(blocked_pipeline).not_to be_persisted
+        expect(allowed_pipeline).to be_created_successfully
+      end
+    end
+
+    context 'with different projects' do
+      let_it_be(:other_project) { create(:project, :repository) }
+
+      before do
+        other_project.add_maintainer(user)
+      end
+
+      it 'allows user to create pipeline' do
+        blocked_pipeline = create_pipelines(project: project)
+        allowed_pipeline = create_pipelines(count: 1, project: other_project)
+
+        expect(blocked_pipeline).not_to be_persisted
+        expect(allowed_pipeline).to be_created_successfully
+      end
+    end
+  end
+
+  def create_pipelines(attrs = {})
+    attrs.reverse_merge!(user: user, ref: ref, project: project, count: 2)
+
+    service = described_class.new(attrs[:project], attrs[:user], { ref: attrs[:ref] })
+
+    attrs[:count].pred.times { service.execute(:push) }
+    service.execute(:push).payload
+  end
+end

spec/services/ci/create_pipeline_service_spec.rb

@@ -12,6 +12,10 @@ RSpec.describe Ci::CreatePipelineService do
 
   before do
     stub_ci_pipeline_to_return_yaml_file
+
+    # Disable rate limiting for pipeline creation
+    allow(Gitlab::ApplicationRateLimiter).to receive(:rate_limits)
+      .and_return(pipelines_create: { threshold: 0, interval: 1.minute })
   end
 
   describe '#execute' do