Commit ed6b8745 authored by Shubham Kumar's avatar Shubham Kumar

Add log count of active jobs

Counts the active jobs when pipeline is created

Changelog: added
parent d26b12d6
...@@ -393,6 +393,10 @@ module Ci ...@@ -393,6 +393,10 @@ module Ci
newest_first(ref: ref).failed.take newest_first(ref: ref).failed.take
end end
def self.jobs_count_in_alive_pipelines
created_after(24.hours.ago).alive.joins(:builds).count
end
# Returns a Hash containing the latest pipeline for every given # Returns a Hash containing the latest pipeline for every given
# commit. # commit.
# #
......
...@@ -45,6 +45,7 @@ The following metrics are available: ...@@ -45,6 +45,7 @@ The following metrics are available:
| `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` | | `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` |
| `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | `worker` | | `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | `worker` |
| `job_waiter_timeouts_total` | Counter | 12.9 | Number of batches of jobs that timed out where a web request is waiting for the jobs to complete | `worker` | | `job_waiter_timeouts_total` | Counter | 12.9 | Number of batches of jobs that timed out where a web request is waiting for the jobs to complete | `worker` |
| `gitlab_ci_active_jobs` | Histogram | 14.2 | Count of active jobs when pipeline is created | |
| `gitlab_database_transaction_seconds` | Histogram | 12.1 | Time spent in database transactions, in seconds | | | `gitlab_database_transaction_seconds` | Histogram | 12.1 | Time spent in database transactions, in seconds | |
| `gitlab_method_call_duration_seconds` | Histogram | 10.2 | Method calls real duration | `controller`, `action`, `module`, `method` | | `gitlab_method_call_duration_seconds` | Histogram | 10.2 | Method calls real duration | `controller`, `action`, `module`, `method` |
| `gitlab_page_out_of_bounds` | Counter | 12.8 | Counter for the PageLimiter pagination limit being hit | `controller`, `action`, `bot` | | `gitlab_page_out_of_bounds` | Counter | 12.8 | Counter for the PageLimiter pagination limit being hit | `controller`, `action`, `bot` |
......
...@@ -38,13 +38,11 @@ module EE ...@@ -38,13 +38,11 @@ module EE
@excessive ||= jobs_in_alive_pipelines_count - ci_active_jobs_limit @excessive ||= jobs_in_alive_pipelines_count - ci_active_jobs_limit
end end
# rubocop: disable CodeReuse/ActiveRecord
def jobs_in_alive_pipelines_count def jobs_in_alive_pipelines_count
strong_memoize(:jobs_in_alive_pipelines_count) do strong_memoize(:jobs_in_alive_pipelines_count) do
@project.all_pipelines.created_after(24.hours.ago).alive.joins(:builds).count @project.all_pipelines.jobs_count_in_alive_pipelines
end end
end end
# rubocop: enable CodeReuse/ActiveRecord
def ci_active_jobs_limit def ci_active_jobs_limit
strong_memoize(:ci_active_jobs_limit) do strong_memoize(:ci_active_jobs_limit) do
......
...@@ -97,6 +97,11 @@ module Gitlab ...@@ -97,6 +97,11 @@ module Gitlab
.observe({ source: pipeline.source.to_s }, pipeline.total_size) .observe({ source: pipeline.source.to_s }, pipeline.total_size)
end end
def observe_jobs_count_in_alive_pipelines
metrics.active_jobs_histogram
.observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines)
end
def increment_pipeline_failure_reason_counter(reason) def increment_pipeline_failure_reason_counter(reason)
metrics.pipeline_failure_reason_counter metrics.pipeline_failure_reason_counter
.increment(reason: (reason || :unknown_failure).to_s) .increment(reason: (reason || :unknown_failure).to_s)
......
...@@ -22,6 +22,7 @@ module Gitlab ...@@ -22,6 +22,7 @@ module Gitlab
@command.observe_creation_duration(Time.now - @start) @command.observe_creation_duration(Time.now - @start)
@command.observe_pipeline_size(@pipeline) @command.observe_pipeline_size(@pipeline)
@command.observe_jobs_count_in_alive_pipelines
@pipeline @pipeline
end end
......
...@@ -29,6 +29,15 @@ module Gitlab ...@@ -29,6 +29,15 @@ module Gitlab
::Gitlab::Metrics.histogram(name, comment, labels, buckets) ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
end end
def self.active_jobs_histogram
name = :gitlab_ci_active_jobs
comment = 'Total amount of active jobs'
labels = { plan: nil }
buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000]
::Gitlab::Metrics.histogram(name, comment, labels, buckets)
end
def self.pipeline_processing_events_counter def self.pipeline_processing_events_counter
name = :gitlab_ci_pipeline_processing_events_total name = :gitlab_ci_pipeline_processing_events_total
comment = 'Total amount of pipeline processing events' comment = 'Total amount of pipeline processing events'
......
...@@ -7,7 +7,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do ...@@ -7,7 +7,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do
let_it_be(:user) { create(:user) } let_it_be(:user) { create(:user) }
let(:pipeline) { build_stubbed(:ci_pipeline) } let(:pipeline) { build_stubbed(:ci_pipeline) }
let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new } let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new(project: project) }
let(:first_step) { spy('first step') } let(:first_step) { spy('first step') }
let(:second_step) { spy('second step') } let(:second_step) { spy('second step') }
let(:sequence) { [first_step, second_step] } let(:sequence) { [first_step, second_step] }
...@@ -71,5 +71,20 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do ...@@ -71,5 +71,20 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do
expect(histogram).to have_received(:observe) expect(histogram).to have_received(:observe)
.with({ source: 'push' }, 0) .with({ source: 'push' }, 0)
end end
it 'records active jobs by pipeline plan in a histogram' do
allow(command.metrics)
.to receive(:active_jobs_histogram)
.and_return(histogram)
pipeline = create(:ci_pipeline, project: project, status: :running)
create(:ci_build, :finished, project: project, pipeline: pipeline)
create(:ci_build, :failed, project: project, pipeline: pipeline)
create(:ci_build, :running, project: project, pipeline: pipeline)
subject.build!
expect(histogram).to have_received(:observe)
.with(hash_including(plan: project.actual_plan_name), 3)
end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment