Commit 11ff9fc6 authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch '28359-skip-process-commit-worker-unless-issues-referenced' into 'master'

Use regex to skip unnecessary reference processing in ProcessCommitWorker

Closes #28359

See merge request !10867
parents 1d0aa480 29519edb
...@@ -78,6 +78,8 @@ module Mentionable ...@@ -78,6 +78,8 @@ module Mentionable
# Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference. # Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference.
def referenced_mentionables(current_user = self.author) def referenced_mentionables(current_user = self.author)
return [] unless matches_cross_reference_regex?
refs = all_references(current_user) refs = all_references(current_user)
refs = (refs.issues + refs.merge_requests + refs.commits) refs = (refs.issues + refs.merge_requests + refs.commits)
...@@ -87,6 +89,20 @@ module Mentionable ...@@ -87,6 +89,20 @@ module Mentionable
refs.reject { |ref| ref == local_reference } refs.reject { |ref| ref == local_reference }
end end
# Uses regex to quickly determine if mentionables might be referenced
# Allows heavy processing to be skipped
def matches_cross_reference_regex?
reference_pattern = if project.default_issues_tracker?
ReferenceRegexes::DEFAULT_PATTERN
else
ReferenceRegexes::EXTERNAL_PATTERN
end
self.class.mentionable_attrs.any? do |attr, _|
__send__(attr) =~ reference_pattern
end
end
# Create a cross-reference Note for each GFM reference to another Mentionable found in the +mentionable_attrs+. # Create a cross-reference Note for each GFM reference to another Mentionable found in the +mentionable_attrs+.
def create_cross_references!(author = self.author, without = []) def create_cross_references!(author = self.author, without = [])
refs = referenced_mentionables(author) refs = referenced_mentionables(author)
......
module Mentionable
module ReferenceRegexes
def self.reference_pattern(link_patterns, issue_pattern)
Regexp.union(link_patterns,
issue_pattern,
Commit.reference_pattern,
MergeRequest.reference_pattern)
end
DEFAULT_PATTERN = begin
issue_pattern = Issue.reference_pattern
link_patterns = Regexp.union([Issue, Commit, MergeRequest].map(&:link_reference_pattern))
reference_pattern(link_patterns, issue_pattern)
end
EXTERNAL_PATTERN = begin
issue_pattern = ExternalIssue.reference_pattern
link_patterns = URI.regexp(%w(http https))
reference_pattern(link_patterns, issue_pattern)
end
end
end
...@@ -85,10 +85,12 @@ class GitPushService < BaseService ...@@ -85,10 +85,12 @@ class GitPushService < BaseService
default = is_default_branch? default = is_default_branch?
push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit| push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit|
if commit.matches_cross_reference_regex?
ProcessCommitWorker. ProcessCommitWorker.
perform_async(project.id, current_user.id, commit.to_hash, default) perform_async(project.id, current_user.id, commit.to_hash, default)
end end
end end
end
protected protected
......
...@@ -23,6 +23,9 @@ class ProcessCommitWorker ...@@ -23,6 +23,9 @@ class ProcessCommitWorker
return unless user return unless user
commit = build_commit(project, commit_hash) commit = build_commit(project, commit_hash)
return unless commit.matches_cross_reference_regex?
author = commit.author || user author = commit.author || user
process_commit_message(project, commit, user, author, default) process_commit_message(project, commit, user, author, default)
......
...@@ -6,7 +6,7 @@ feature 'Cycle Analytics', feature: true, js: true do ...@@ -6,7 +6,7 @@ feature 'Cycle Analytics', feature: true, js: true do
let(:project) { create(:project, :repository) } let(:project) { create(:project, :repository) }
let(:issue) { create(:issue, project: project, created_at: 2.days.ago) } let(:issue) { create(:issue, project: project, created_at: 2.days.ago) }
let(:milestone) { create(:milestone, project: project) } let(:milestone) { create(:milestone, project: project) }
let(:mr) { create_merge_request_closing_issue(issue) } let(:mr) { create_merge_request_closing_issue(issue, commit_message: "References #{issue.to_reference}") }
let(:pipeline) { create(:ci_empty_pipeline, status: 'created', project: project, ref: mr.source_branch, sha: mr.source_branch_sha) } let(:pipeline) { create(:ci_empty_pipeline, status: 'created', project: project, ref: mr.source_branch, sha: mr.source_branch_sha) }
context 'as an allowed user' do context 'as an allowed user' do
...@@ -32,7 +32,6 @@ feature 'Cycle Analytics', feature: true, js: true do ...@@ -32,7 +32,6 @@ feature 'Cycle Analytics', feature: true, js: true do
before do before do
project.team << [user, :master] project.team << [user, :master]
allow_any_instance_of(Gitlab::ReferenceExtractor).to receive(:issues).and_return([issue])
create_cycle create_cycle
deploy_master deploy_master
......
...@@ -11,8 +11,6 @@ describe 'cycle analytics events' do ...@@ -11,8 +11,6 @@ describe 'cycle analytics events' do
end end
before do before do
allow_any_instance_of(Gitlab::ReferenceExtractor).to receive(:issues).and_return([context])
setup(context) setup(context)
end end
...@@ -332,7 +330,7 @@ describe 'cycle analytics events' do ...@@ -332,7 +330,7 @@ describe 'cycle analytics events' do
def setup(context) def setup(context)
milestone = create(:milestone, project: project) milestone = create(:milestone, project: project)
context.update(milestone: milestone) context.update(milestone: milestone)
mr = create_merge_request_closing_issue(context) mr = create_merge_request_closing_issue(context, commit_message: "References #{context.to_reference}")
ProcessCommitWorker.new.perform(project.id, user.id, mr.commits.last.to_hash) ProcessCommitWorker.new.perform(project.id, user.id, mr.commits.last.to_hash)
end end
......
...@@ -163,3 +163,52 @@ describe Issue, "Mentionable" do ...@@ -163,3 +163,52 @@ describe Issue, "Mentionable" do
end end
end end
end end
describe Commit, 'Mentionable' do
let(:project) { create(:project, :public, :repository) }
let(:commit) { project.commit }
describe '#matches_cross_reference_regex?' do
it "is false when message doesn't reference anything" do
allow(commit.raw).to receive(:message).and_return "WIP: Do something"
expect(commit.matches_cross_reference_regex?).to be false
end
it 'is true if issue #number mentioned in title' do
allow(commit.raw).to receive(:message).and_return "#1"
expect(commit.matches_cross_reference_regex?).to be true
end
it 'is true if references an MR' do
allow(commit.raw).to receive(:message).and_return "See merge request !12"
expect(commit.matches_cross_reference_regex?).to be true
end
it 'is true if references a commit' do
allow(commit.raw).to receive(:message).and_return "a1b2c3d4"
expect(commit.matches_cross_reference_regex?).to be true
end
it 'is true if issue referenced by url' do
issue = create(:issue, project: project)
allow(commit.raw).to receive(:message).and_return Gitlab::UrlBuilder.build(issue)
expect(commit.matches_cross_reference_regex?).to be true
end
context 'with external issue tracker' do
let(:project) { create(:jira_project) }
it 'is true if external issues referenced' do
allow(commit.raw).to receive(:message).and_return 'JIRA-123'
expect(commit.matches_cross_reference_regex?).to be true
end
end
end
end
...@@ -9,8 +9,6 @@ describe 'cycle analytics events', api: true do ...@@ -9,8 +9,6 @@ describe 'cycle analytics events', api: true do
before do before do
project.team << [user, :developer] project.team << [user, :developer]
allow_any_instance_of(Gitlab::ReferenceExtractor).to receive(:issues).and_return([issue])
3.times do |count| 3.times do |count|
Timecop.freeze(Time.now + count.days) do Timecop.freeze(Time.now + count.days) do
create_cycle create_cycle
...@@ -121,7 +119,7 @@ describe 'cycle analytics events', api: true do ...@@ -121,7 +119,7 @@ describe 'cycle analytics events', api: true do
def create_cycle def create_cycle
milestone = create(:milestone, project: project) milestone = create(:milestone, project: project)
issue.update(milestone: milestone) issue.update(milestone: milestone)
mr = create_merge_request_closing_issue(issue) mr = create_merge_request_closing_issue(issue, commit_message: "References #{issue.to_reference}")
pipeline = create(:ci_empty_pipeline, status: 'created', project: project, ref: mr.source_branch, sha: mr.source_branch_sha) pipeline = create(:ci_empty_pipeline, status: 'created', project: project, ref: mr.source_branch, sha: mr.source_branch_sha)
pipeline.run pipeline.run
......
...@@ -622,12 +622,21 @@ describe GitPushService, services: true do ...@@ -622,12 +622,21 @@ describe GitPushService, services: true do
it 'only schedules a limited number of commits' do it 'only schedules a limited number of commits' do
allow(service).to receive(:push_commits). allow(service).to receive(:push_commits).
and_return(Array.new(1000, double(:commit, to_hash: {}))) and_return(Array.new(1000, double(:commit, to_hash: {}, matches_cross_reference_regex?: true)))
expect(ProcessCommitWorker).to receive(:perform_async).exactly(100).times expect(ProcessCommitWorker).to receive(:perform_async).exactly(100).times
service.process_commit_messages service.process_commit_messages
end end
it "skips commits which don't include cross-references" do
allow(service).to receive(:push_commits).
and_return([double(:commit, to_hash: {}, matches_cross_reference_regex?: false)])
expect(ProcessCommitWorker).not_to receive(:perform_async)
service.process_commit_messages
end
end end
def execute_service(project, user, oldrev, newrev, ref) def execute_service(project, user, oldrev, newrev, ref)
......
...@@ -20,7 +20,7 @@ module CycleAnalyticsHelpers ...@@ -20,7 +20,7 @@ module CycleAnalyticsHelpers
ref: 'refs/heads/master').execute ref: 'refs/heads/master').execute
end end
def create_merge_request_closing_issue(issue, message: nil, source_branch: nil) def create_merge_request_closing_issue(issue, message: nil, source_branch: nil, commit_message: 'commit message')
if !source_branch || project.repository.commit(source_branch).blank? if !source_branch || project.repository.commit(source_branch).blank?
source_branch = generate(:branch) source_branch = generate(:branch)
project.repository.add_branch(user, source_branch, 'master') project.repository.add_branch(user, source_branch, 'master')
...@@ -30,7 +30,7 @@ module CycleAnalyticsHelpers ...@@ -30,7 +30,7 @@ module CycleAnalyticsHelpers
user, user,
generate(:branch), generate(:branch),
'content', 'content',
message: 'commit message', message: commit_message,
branch_name: source_branch) branch_name: source_branch)
project.repository.commit(sha) project.repository.commit(sha)
......
...@@ -20,6 +20,14 @@ describe ProcessCommitWorker do ...@@ -20,6 +20,14 @@ describe ProcessCommitWorker do
worker.perform(project.id, -1, commit.to_hash) worker.perform(project.id, -1, commit.to_hash)
end end
it 'does not process the commit when no issues are referenced' do
allow(worker).to receive(:build_commit).and_return(double(matches_cross_reference_regex?: false))
expect(worker).not_to receive(:process_commit_message)
worker.perform(project.id, user.id, commit.to_hash)
end
it 'processes the commit message' do it 'processes the commit message' do
expect(worker).to receive(:process_commit_message).and_call_original expect(worker).to receive(:process_commit_message).and_call_original
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment