Introduce JobActivity limit for alive jobs

To protect from accidentally launching too many CI pipelines
we introduce a new JobActivity limit which compare the count
of jobs in alive pipelines created in the past hour.

app/models/ci/pipeline.rb

@@ -203,6 +203,7 @@ module Ci
     scope :for_sha, -> (sha) { where(sha: sha) }
     scope :for_source_sha, -> (source_sha) { where(source_sha: source_sha) }
     scope :for_sha_or_source_sha, -> (sha) { for_sha(sha).or(for_source_sha(sha)) }
+    scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) }
 
     scope :triggered_by_merge_request, -> (merge_request) do
       where(source: :merge_request_event, merge_request: merge_request)

app/services/ci/create_pipeline_service.rb

@@ -15,7 +15,8 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Limit::Size,
                 Gitlab::Ci::Pipeline::Chain::Populate,
                 Gitlab::Ci::Pipeline::Chain::Create,
-                Gitlab::Ci::Pipeline::Chain::Limit::Activity].freeze
+                Gitlab::Ci::Pipeline::Chain::Limit::Activity,
+                Gitlab::Ci::Pipeline::Chain::Limit::JobActivity].freeze
 
     def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, merge_request: nil, **options, &block)
       @pipeline = Ci::Pipeline.new

db/migrate/20190816151221_add_active_jobs_limit_to_plans.rb

+# frozen_string_literal: true
+
+class AddActiveJobsLimitToPlans < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :plans, :active_jobs_limit, :integer, default: 0
+  end
+
+  def down
+    remove_column :plans, :active_jobs_limit
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_15_093949) do
+ActiveRecord::Schema.define(version: 2019_08_16_151221) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -2500,6 +2500,7 @@ ActiveRecord::Schema.define(version: 2019_08_15_093949) do
     t.string "title"
     t.integer "active_pipelines_limit"
     t.integer "pipeline_size_limit"
+    t.integer "active_jobs_limit", default: 0
     t.index ["name"], name: "index_plans_on_name"
   end
 

ee/app/models/ee/ci/pipeline_enums.rb

@@ -10,7 +10,11 @@ module EE
 
         override :failure_reasons
         def failure_reasons
-          super.merge(activity_limit_exceeded: 20, size_limit_exceeded: 21)
+          super.merge(
+            activity_limit_exceeded: 20,
+            size_limit_exceeded: 21,
+            job_activity_limit_exceeded: 22
+          )
         end
 
         override :sources

ee/app/models/ee/namespace.rb

@@ -223,6 +223,10 @@ module EE
       actual_plan&.pipeline_size_limit.to_i
     end
 
+    def max_active_jobs
+      actual_plan&.active_jobs_limit.to_i
+    end
+
     def memoized_plans=(plans)
       @plans = plans # rubocop: disable Gitlab/ModuleWithInstanceVariables
     end

ee/changelogs/unreleased/security-add-job-activity-limit.yml

+---
+title: Limit number of jobs in running pipelines for the past hour on per plan basis
+merge_request: 1182
+author:
+type: security

ee/lib/ee/gitlab/ci/pipeline/chain/limit/job_activity.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module Ci
+      module Pipeline
+        module Chain
+          module Limit
+            module JobActivity
+              extend ::Gitlab::Utils::Override
+              include ::Gitlab::Ci::Pipeline::Chain::Helpers
+              include ::Gitlab::OptimisticLocking
+
+              attr_reader :limit
+              private :limit
+
+              def initialize(*)
+                super
+
+                @limit = Pipeline::Quota::JobActivity
+                  .new(project.namespace, pipeline.project)
+              end
+
+              override :perform!
+              def perform!
+                return unless limit.exceeded?
+
+                retry_optimistic_lock(pipeline) do
+                  pipeline.drop!(:job_activity_limit_exceeded)
+                end
+              end
+
+              override :break?
+              def break?
+                limit.exceeded?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/ee/gitlab/ci/pipeline/quota/job_activity.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module Ci
+      module Pipeline
+        module Quota
+          class JobActivity < Ci::Limit
+            include ::Gitlab::Utils::StrongMemoize
+            include ActionView::Helpers::TextHelper
+
+            def initialize(namespace, project)
+              @namespace = namespace
+              @project = project
+            end
+
+            def enabled?
+              strong_memoize(:enabled) do
+                @namespace.max_active_jobs > 0
+              end
+            end
+
+            def exceeded?
+              return false unless enabled?
+
+              excessive_jobs_count > 0
+            end
+
+            def message
+              return unless exceeded?
+
+              'Active jobs limit exceeded by ' \
+                "#{pluralize(excessive_jobs_count, 'job')} in the past 24 hours!"
+            end
+
+            private
+
+            def excessive_jobs_count
+              @excessive ||= jobs_in_alive_pipelines_count - max_active_jobs_count
+            end
+
+            # rubocop: disable CodeReuse/ActiveRecord
+            def jobs_in_alive_pipelines_count
+              @project.all_pipelines.created_after(24.hours.ago).alive.joins(:builds).count
+            end
+            # rubocop: enable CodeReuse/ActiveRecord
+
+            def max_active_jobs_count
+              @namespace.max_active_jobs
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/ci/pipeline/quota/job_activity_spec.rb

+require 'spec_helper'
+
+describe EE::Gitlab::Ci::Pipeline::Quota::JobActivity do
+  set(:namespace) { create(:namespace) }
+  set(:project) { create(:project, namespace: namespace) }
+
+  let(:active_jobs_limit) { 0 }
+  let(:gold_plan) { create(:gold_plan, active_jobs_limit: active_jobs_limit) }
+
+  let(:limit) { described_class.new(namespace, project) }
+
+  before do
+    create(:gitlab_subscription, namespace: namespace, hosted_plan: gold_plan)
+  end
+
+  describe '#enabled?' do
+    context 'when limit is enabled in plan' do
+      let(:active_jobs_limit) { 10 }
+
+      it 'is enabled' do
+        expect(limit).to be_enabled
+      end
+    end
+
+    context 'when limit is not enabled' do
+      it 'is not enabled' do
+        expect(limit).not_to be_enabled
+      end
+    end
+  end
+
+  describe '#exceeded?' do
+    let(:active_jobs_limit) { 2 }
+
+    context 'when pipelines created recently' do
+      context 'and pipelines are running' do
+        let(:pipeline1) { create(:ci_pipeline, project: project, status: 'created', created_at: Time.now) }
+        let(:pipeline2) { create(:ci_pipeline, project: project, status: 'created', created_at: Time.now) }
+
+        before do
+          create(:ci_build, pipeline: pipeline1)
+          create(:ci_build, pipeline: pipeline2)
+        end
+
+        context 'when count of jobs in alive pipelines is below the limit' do
+          it 'is not exceeded' do
+            expect(limit).not_to be_exceeded
+          end
+        end
+
+        context 'when count of jobs in alive pipelines is above the limit' do
+          before do
+            create(:ci_build, pipeline: pipeline2)
+          end
+
+          it 'is exceeded' do
+            expect(limit).to be_exceeded
+          end
+        end
+      end
+
+      context 'and pipelines are completed' do
+        before do
+          create(:ci_pipeline, project: project, status: 'success', created_at: Time.now).tap do |pipeline|
+            create(:ci_build, pipeline: pipeline)
+            create(:ci_build, pipeline: pipeline)
+            create(:ci_build, pipeline: pipeline)
+          end
+        end
+
+        it 'is not exceeded' do
+          expect(limit).not_to be_exceeded
+        end
+      end
+    end
+
+    context 'when pipelines are older than 24 hours' do
+      before do
+        create(:ci_pipeline, project: project, status: 'created', created_at: 25.hours.ago).tap do |pipeline|
+          create(:ci_build, pipeline: pipeline)
+          create(:ci_build, pipeline: pipeline)
+          create(:ci_build, pipeline: pipeline)
+        end
+      end
+
+      it 'is not exceeded' do
+        expect(limit).not_to be_exceeded
+      end
+    end
+  end
+
+  describe '#message' do
+    context 'when limit is exceeded' do
+      let(:active_jobs_limit) { 1 }
+
+      before do
+        create(:ci_pipeline, project: project, status: 'created', created_at: Time.now).tap do |pipeline|
+          create(:ci_build, pipeline: pipeline)
+          create(:ci_build, pipeline: pipeline)
+          create(:ci_build, pipeline: pipeline)
+        end
+      end
+
+      it 'returns info about pipeline activity limit exceeded' do
+        expect(limit.message)
+          .to eq "Active jobs limit exceeded by 2 jobs in the past 24 hours!"
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/pipeline/chain/limit/activity_spec.rb

@@ -43,7 +43,7 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Activity do
     end
   end
 
-  context 'when pipeline size limit is not exceeded' do
+  context 'when pipeline activity limit is not exceeded' do
     before do
       step.perform!
     end

ee/spec/lib/gitlab/ci/pipeline/chain/limit/job_activity_spec.rb

+require 'spec_helper'
+
+describe ::Gitlab::Ci::Pipeline::Chain::Limit::JobActivity do
+  set(:namespace) { create(:namespace) }
+  set(:project) { create(:project, namespace: namespace) }
+  set(:user) { create(:user) }
+
+  let(:command) do
+    double('command', project: project, current_user: user)
+  end
+
+  let(:pipeline) do
+    create(:ci_pipeline, project: project)
+  end
+
+  let(:step) { described_class.new(pipeline, command) }
+
+  context 'when active jobs limit is exceeded' do
+    before do
+      gold_plan = create(:gold_plan, active_jobs_limit: 2)
+      create(:gitlab_subscription, namespace: namespace, hosted_plan: gold_plan)
+
+      pipeline = create(:ci_pipeline, project: project, status: 'running', created_at: Time.now)
+      create(:ci_build, pipeline: pipeline)
+      create(:ci_build, pipeline: pipeline)
+      create(:ci_build, pipeline: pipeline)
+
+      step.perform!
+    end
+
+    it 'drops the pipeline' do
+      expect(pipeline.reload).to be_failed
+    end
+
+    it 'persists the pipeline' do
+      expect(pipeline).to be_persisted
+    end
+
+    it 'breaks the chain' do
+      expect(step.break?).to be true
+    end
+
+    it 'sets a valid failure reason' do
+      expect(pipeline.job_activity_limit_exceeded?).to be true
+    end
+  end
+
+  context 'when job activity limit is not exceeded' do
+    before do
+      step.perform!
+    end
+
+    it 'does not break the chain' do
+      expect(step.break?).to be false
+    end
+
+    it 'does not invalidate the pipeline' do
+      expect(pipeline.errors).to be_empty
+    end
+  end
+end

ee/spec/models/namespace_spec.rb

@@ -345,6 +345,45 @@ describe Namespace do
     end
   end
 
+  describe '#max_active_jobs' do
+    context 'when there is no limit defined' do
+      it 'returns zero' do
+        expect(namespace.max_active_jobs).to be_zero
+      end
+    end
+
+    context 'when free plan has limit defined' do
+      before do
+        free_plan.update_column(:active_jobs_limit, 100)
+      end
+
+      it 'returns a free plan limits' do
+        expect(namespace.max_active_jobs).to be 100
+      end
+    end
+
+    context 'when associated plan has no limit defined' do
+      before do
+        create(:gitlab_subscription, namespace: namespace, hosted_plan: gold_plan)
+      end
+
+      it 'returns zero' do
+        expect(namespace.max_active_jobs).to be_zero
+      end
+    end
+
+    context 'when limit is defined' do
+      before do
+        gold_plan.update_column(:active_jobs_limit, 10)
+        create(:gitlab_subscription, namespace: namespace, hosted_plan: gold_plan)
+      end
+
+      it 'returns a number of maximum active jobs' do
+        expect(namespace.max_active_jobs).to eq 10
+      end
+    end
+  end
+
   describe '#shared_runners_enabled?' do
     subject { namespace.shared_runners_enabled? }
 

lib/gitlab/ci/pipeline/chain/limit/job_activity.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Limit
+          class JobActivity < Chain::Base
+            def perform!
+              # to be overridden in EE
+            end
+
+            def break?
+              false # to be overridden in EE
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::JobActivity')