Commit 9a81550f authored by Francisco Javier López's avatar Francisco Javier López Committed by Nick Thomas

Create GPG commit signature in bulk

parent 5b6553a0
...@@ -548,6 +548,10 @@ class Project < ActiveRecord::Base ...@@ -548,6 +548,10 @@ class Project < ActiveRecord::Base
repository.commit_by(oid: oid) repository.commit_by(oid: oid)
end end
def commits_by(oids:)
repository.commits_by(oids: oids)
end
# ref can't be HEAD, can only be branch/tag name or SHA # ref can't be HEAD, can only be branch/tag name or SHA
def latest_successful_builds_for(ref = default_branch) def latest_successful_builds_for(ref = default_branch)
latest_pipeline = pipelines.latest_successful_for(ref) latest_pipeline = pipelines.latest_successful_for(ref)
......
...@@ -76,7 +76,7 @@ class GitPushService < BaseService ...@@ -76,7 +76,7 @@ class GitPushService < BaseService
else else
paths = Set.new paths = Set.new
@push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit| last_pushed_commits.each do |commit|
commit.raw_deltas.each do |diff| commit.raw_deltas.each do |diff|
paths << diff.new_path paths << diff.new_path
end end
...@@ -92,7 +92,7 @@ class GitPushService < BaseService ...@@ -92,7 +92,7 @@ class GitPushService < BaseService
end end
def update_signatures def update_signatures
commit_shas = @push_commits.last(PROCESS_COMMIT_LIMIT).map(&:sha) commit_shas = last_pushed_commits.map(&:sha)
return if commit_shas.empty? return if commit_shas.empty?
...@@ -103,16 +103,14 @@ class GitPushService < BaseService ...@@ -103,16 +103,14 @@ class GitPushService < BaseService
commit_shas = Gitlab::Git::Commit.shas_with_signatures(project.repository, commit_shas) commit_shas = Gitlab::Git::Commit.shas_with_signatures(project.repository, commit_shas)
commit_shas.each do |sha| CreateGpgSignatureWorker.perform_async(commit_shas, project.id)
CreateGpgSignatureWorker.perform_async(sha, project.id)
end
end end
# Schedules processing of commit messages. # Schedules processing of commit messages.
def process_commit_messages def process_commit_messages
default = default_branch? default = default_branch?
@push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit| last_pushed_commits.each do |commit|
if commit.matches_cross_reference_regex? if commit.matches_cross_reference_regex?
ProcessCommitWorker ProcessCommitWorker
.perform_async(project.id, current_user.id, commit.to_hash, default) .perform_async(project.id, current_user.id, commit.to_hash, default)
...@@ -206,4 +204,8 @@ class GitPushService < BaseService ...@@ -206,4 +204,8 @@ class GitPushService < BaseService
def branch_name def branch_name
@branch_name ||= Gitlab::Git.ref_name(params[:ref]) @branch_name ||= Gitlab::Git.ref_name(params[:ref])
end end
def last_pushed_commits
@last_pushed_commits ||= @push_commits.last(PROCESS_COMMIT_LIMIT)
end
end end
...@@ -3,15 +3,23 @@ ...@@ -3,15 +3,23 @@
class CreateGpgSignatureWorker class CreateGpgSignatureWorker
include ApplicationWorker include ApplicationWorker
def perform(commit_sha, project_id) def perform(commit_shas, project_id)
return if commit_shas.empty?
project = Project.find_by(id: project_id) project = Project.find_by(id: project_id)
return unless project return unless project
commit = project.commit(commit_sha) commits = project.commits_by(oids: commit_shas)
return unless commit return if commits.empty?
# This calculates and caches the signature in the database # This calculates and caches the signature in the database
Gitlab::Gpg::Commit.new(commit).signature commits.each do |commit|
begin
Gitlab::Gpg::Commit.new(commit).signature
rescue => e
Rails.logger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}")
end
end
end end
end end
---
title: Performing Commit GPG signature calculation in bulk
merge_request: 20870
author:
type: performance
...@@ -3877,6 +3877,16 @@ describe Project do ...@@ -3877,6 +3877,16 @@ describe Project do
end end
end end
context '#commits_by' do
let(:project) { create(:project, :repository) }
let(:commits) { project.repository.commits('HEAD', limit: 3).commits }
let(:commit_shas) { commits.map(&:id) }
it 'retrieves several commits from the repository by oid' do
expect(project.commits_by(oids: commit_shas)).to eq commits
end
end
def rugged_config def rugged_config
Gitlab::GitalyClient::StorageSettings.allow_disk_access do Gitlab::GitalyClient::StorageSettings.allow_disk_access do
project.repository.rugged.config project.repository.rugged.config
......
...@@ -761,7 +761,7 @@ describe GitPushService, services: true do ...@@ -761,7 +761,7 @@ describe GitPushService, services: true do
end end
it 'does not queue a CreateGpgSignatureWorker' do it 'does not queue a CreateGpgSignatureWorker' do
expect(CreateGpgSignatureWorker).not_to receive(:perform_async).with(sample_commit.id, project.id) expect(CreateGpgSignatureWorker).not_to receive(:perform_async)
execute_service(project, user, oldrev, newrev, ref) execute_service(project, user, oldrev, newrev, ref)
end end
...@@ -769,7 +769,15 @@ describe GitPushService, services: true do ...@@ -769,7 +769,15 @@ describe GitPushService, services: true do
context 'when the signature is not yet cached' do context 'when the signature is not yet cached' do
it 'queues a CreateGpgSignatureWorker' do it 'queues a CreateGpgSignatureWorker' do
expect(CreateGpgSignatureWorker).to receive(:perform_async).with(sample_commit.id, project.id) expect(CreateGpgSignatureWorker).to receive(:perform_async).with([sample_commit.id], project.id)
execute_service(project, user, oldrev, newrev, ref)
end
it 'can queue several commits to create the gpg signature' do
allow(Gitlab::Git::Commit).to receive(:shas_with_signatures).and_return([sample_commit.id, another_sample_commit.id])
expect(CreateGpgSignatureWorker).to receive(:perform_async).with([sample_commit.id, another_sample_commit.id], project.id)
execute_service(project, user, oldrev, newrev, ref) execute_service(project, user, oldrev, newrev, ref)
end end
......
...@@ -2,21 +2,37 @@ require 'spec_helper' ...@@ -2,21 +2,37 @@ require 'spec_helper'
describe CreateGpgSignatureWorker do describe CreateGpgSignatureWorker do
let(:project) { create(:project, :repository) } let(:project) { create(:project, :repository) }
let(:commits) { project.repository.commits('HEAD', limit: 3).commits }
let(:commit_shas) { commits.map(&:id) }
context 'when GpgKey is found' do context 'when GpgKey is found' do
let(:commit_sha) { '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33' } let(:gpg_commit) { instance_double(Gitlab::Gpg::Commit) }
before do
allow(Project).to receive(:find_by).with(id: project.id).and_return(project)
allow(project).to receive(:commits_by).with(oids: commit_shas).and_return(commits)
end
subject { described_class.new.perform(commit_shas, project.id) }
it 'calls Gitlab::Gpg::Commit#signature' do it 'calls Gitlab::Gpg::Commit#signature' do
commit = instance_double(Commit) commits.each do |commit|
gpg_commit = instance_double(Gitlab::Gpg::Commit) expect(Gitlab::Gpg::Commit).to receive(:new).with(commit).and_return(gpg_commit).once
end
allow(Project).to receive(:find_by).with(id: project.id).and_return(project) expect(gpg_commit).to receive(:signature).exactly(commits.size).times
allow(project).to receive(:commit).with(commit_sha).and_return(commit)
subject
end
it 'can recover form exception and continue the siganture frocess' do
allow(gpg_commit).to receive(:signature)
allow(Gitlab::Gpg::Commit).to receive(:new).and_return(gpg_commit)
allow(Gitlab::Gpg::Commit).to receive(:new).with(commits.first).and_raise(StandardError)
expect(Gitlab::Gpg::Commit).to receive(:new).with(commit).and_return(gpg_commit) expect(gpg_commit).to receive(:signature).exactly(2).times
expect(gpg_commit).to receive(:signature)
described_class.new.perform(commit_sha, project.id) subject
end end
end end
...@@ -24,7 +40,7 @@ describe CreateGpgSignatureWorker do ...@@ -24,7 +40,7 @@ describe CreateGpgSignatureWorker do
let(:nonexisting_commit_sha) { '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a34' } let(:nonexisting_commit_sha) { '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a34' }
it 'does not raise errors' do it 'does not raise errors' do
expect { described_class.new.perform(nonexisting_commit_sha, project.id) }.not_to raise_error expect { described_class.new.perform([nonexisting_commit_sha], project.id) }.not_to raise_error
end end
end end
...@@ -32,13 +48,13 @@ describe CreateGpgSignatureWorker do ...@@ -32,13 +48,13 @@ describe CreateGpgSignatureWorker do
let(:nonexisting_project_id) { -1 } let(:nonexisting_project_id) { -1 }
it 'does not raise errors' do it 'does not raise errors' do
expect { described_class.new.perform(anything, nonexisting_project_id) }.not_to raise_error expect { described_class.new.perform(commit_shas, nonexisting_project_id) }.not_to raise_error
end end
it 'does not call Gitlab::Gpg::Commit#signature' do it 'does not call Gitlab::Gpg::Commit#signature' do
expect_any_instance_of(Gitlab::Gpg::Commit).not_to receive(:signature) expect_any_instance_of(Gitlab::Gpg::Commit).not_to receive(:signature)
described_class.new.perform(anything, nonexisting_project_id) described_class.new.perform(commit_shas, nonexisting_project_id)
end end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment