Commit c2a52670 authored by Thong Kuah's avatar Thong Kuah Committed by Yannis Roussos

Reduce query count for extremely frequent worker

Consolide 4x queries into one using eager_load, which saves us 3N query
count for a worker which runs 1 million times an hour at peak.
parent b4bf4e7a
...@@ -309,6 +309,7 @@ module Ci ...@@ -309,6 +309,7 @@ module Ci
scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) }
scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) } scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) }
scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) }
scope :eager_load_project, -> { eager_load(project: [:route, { namespace: :route }]) }
scope :outside_pipeline_family, ->(pipeline) do scope :outside_pipeline_family, ->(pipeline) do
where.not(id: pipeline.same_family_pipeline_ids) where.not(id: pipeline.same_family_pipeline_ids)
......
...@@ -12,7 +12,7 @@ class ExpirePipelineCacheWorker ...@@ -12,7 +12,7 @@ class ExpirePipelineCacheWorker
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def perform(pipeline_id) def perform(pipeline_id)
pipeline = Ci::Pipeline.find_by(id: pipeline_id) pipeline = Ci::Pipeline.eager_load_project.find_by(id: pipeline_id)
return unless pipeline return unless pipeline
Ci::ExpirePipelineCacheService.new.execute(pipeline) Ci::ExpirePipelineCacheService.new.execute(pipeline)
......
---
title: Reduce query count for ExpirePipelineCacheWorker
merge_request: 57304
author:
type: performance
...@@ -18,6 +18,23 @@ RSpec.describe ExpirePipelineCacheWorker do ...@@ -18,6 +18,23 @@ RSpec.describe ExpirePipelineCacheWorker do
subject.perform(pipeline.id) subject.perform(pipeline.id)
end end
it 'does not perform extra queries', :aggregate_failures do
recorder = ActiveRecord::QueryRecorder.new { subject.perform(pipeline.id) }
project_queries = recorder.data.values.flat_map {|v| v[:occurrences]}.select {|s| s.include?('FROM "projects"')}
namespace_queries = recorder.data.values.flat_map {|v| v[:occurrences]}.select {|s| s.include?('FROM "namespaces"')}
route_queries = recorder.data.values.flat_map {|v| v[:occurrences]}.select {|s| s.include?('FROM "routes"')}
# This worker is run 1 million times an hour, so we need to save as much
# queries as possible.
expect(recorder.count).to be <= 6
# These arises from #update_etag_cache
expect(project_queries.size).to eq(1)
expect(namespace_queries.size).to eq(1)
expect(route_queries.size).to eq(1)
end
it "doesn't do anything if the pipeline not exist" do it "doesn't do anything if the pipeline not exist" do
expect_any_instance_of(Ci::ExpirePipelineCacheService).not_to receive(:execute) expect_any_instance_of(Ci::ExpirePipelineCacheService).not_to receive(:execute)
expect_any_instance_of(Gitlab::EtagCaching::Store).not_to receive(:touch) expect_any_instance_of(Gitlab::EtagCaching::Store).not_to receive(:touch)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment