Commit d25239ee authored by Jan Provaznik's avatar Jan Provaznik Committed by Stan Hu

Use git_garbage_collect_worker to run pack_refs

PackRefs is not an expensive gitaly call - we want to
call it more often (than as part of full `gc`) because
it helps to keep number of refs files small - too many
refs file may be a problem for deployments with
slow storage.
parent 8163e233
...@@ -11,6 +11,7 @@ module Projects ...@@ -11,6 +11,7 @@ module Projects
class HousekeepingService < BaseService class HousekeepingService < BaseService
# Timeout set to 24h # Timeout set to 24h
LEASE_TIMEOUT = 86400 LEASE_TIMEOUT = 86400
PACK_REFS_PERIOD = 6
class LeaseTaken < StandardError class LeaseTaken < StandardError
def to_s def to_s
...@@ -76,13 +77,15 @@ module Projects ...@@ -76,13 +77,15 @@ module Projects
:gc :gc
elsif pushes_since_gc % full_repack_period == 0 elsif pushes_since_gc % full_repack_period == 0
:full_repack :full_repack
else elsif pushes_since_gc % repack_period == 0
:incremental_repack :incremental_repack
else
:pack_refs
end end
end end
def period_match? def period_match?
[gc_period, full_repack_period, repack_period].any? { |period| pushes_since_gc % period == 0 } [gc_period, full_repack_period, repack_period, PACK_REFS_PERIOD].any? { |period| pushes_since_gc % period == 0 }
end end
def housekeeping_enabled? def housekeeping_enabled?
......
...@@ -29,7 +29,7 @@ class GitGarbageCollectWorker ...@@ -29,7 +29,7 @@ class GitGarbageCollectWorker
# Refresh the branch cache in case garbage collection caused a ref lookup to fail # Refresh the branch cache in case garbage collection caused a ref lookup to fail
flush_ref_caches(project) if task == :gc flush_ref_caches(project) if task == :gc
project.repository.expire_statistics_caches project.repository.expire_statistics_caches if task != :pack_refs
# In case pack files are deleted, release libgit2 cache and open file # In case pack files are deleted, release libgit2 cache and open file
# descriptors ASAP instead of waiting for Ruby garbage collection # descriptors ASAP instead of waiting for Ruby garbage collection
...@@ -58,7 +58,12 @@ class GitGarbageCollectWorker ...@@ -58,7 +58,12 @@ class GitGarbageCollectWorker
## `repository` has to be a Gitlab::Git::Repository ## `repository` has to be a Gitlab::Git::Repository
def gitaly_call(task, repository) def gitaly_call(task, repository)
client = Gitlab::GitalyClient::RepositoryService.new(repository) client = if task == :pack_refs
Gitlab::GitalyClient::RefService.new(repository)
else
Gitlab::GitalyClient::RepositoryService.new(repository)
end
case task case task
when :gc when :gc
client.garbage_collect(bitmaps_enabled?) client.garbage_collect(bitmaps_enabled?)
...@@ -66,6 +71,8 @@ class GitGarbageCollectWorker ...@@ -66,6 +71,8 @@ class GitGarbageCollectWorker
client.repack_full(bitmaps_enabled?) client.repack_full(bitmaps_enabled?)
when :incremental_repack when :incremental_repack
client.repack_incremental client.repack_incremental
when :pack_refs
client.pack_refs
end end
rescue GRPC::NotFound => e rescue GRPC::NotFound => e
Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found") Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found")
......
...@@ -239,6 +239,12 @@ module Gitlab ...@@ -239,6 +239,12 @@ module Gitlab
messages messages
end end
def pack_refs
request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo)
GitalyClient.call(@storage, :ref_service, :pack_refs, request)
end
private private
def consume_refs_response(response) def consume_refs_response(response)
......
...@@ -165,4 +165,15 @@ describe Gitlab::GitalyClient::RefService do ...@@ -165,4 +165,15 @@ describe Gitlab::GitalyClient::RefService do
client.delete_refs(except_with_prefixes: prefixes) client.delete_refs(except_with_prefixes: prefixes)
end end
end end
describe '#pack_refs' do
it 'sends a pack_refs message' do
expect_any_instance_of(Gitaly::RefService::Stub)
.to receive(:pack_refs)
.with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash))
.and_return(double(:pack_refs_response))
client.pack_refs
end
end
end end
...@@ -81,6 +81,9 @@ describe Projects::HousekeepingService do ...@@ -81,6 +81,9 @@ describe Projects::HousekeepingService do
# At push 10, 20, ... (except those above) # At push 10, 20, ... (except those above)
expect(GitGarbageCollectWorker).to receive(:perform_async).with(project.id, :incremental_repack, :the_lease_key, :the_uuid) expect(GitGarbageCollectWorker).to receive(:perform_async).with(project.id, :incremental_repack, :the_lease_key, :the_uuid)
.exactly(16).times .exactly(16).times
# At push 6, 12, 18, ... (except those above)
expect(GitGarbageCollectWorker).to receive(:perform_async).with(project.id, :pack_refs, :the_lease_key, :the_uuid)
.exactly(27).times
201.times do 201.times do
subject.increment! subject.increment!
......
...@@ -115,6 +115,19 @@ describe GitGarbageCollectWorker do ...@@ -115,6 +115,19 @@ describe GitGarbageCollectWorker do
end end
end end
context "pack_refs" do
before do
expect(subject).to receive(:get_lease_uuid).and_return(lease_uuid)
end
it "calls Gitaly" do
expect_any_instance_of(Gitlab::GitalyClient::RefService).to receive(:pack_refs)
.and_return(nil)
subject.perform(project.id, :pack_refs, lease_key, lease_uuid)
end
end
context "repack_incremental" do context "repack_incremental" do
before do before do
expect(subject).to receive(:get_lease_uuid).and_return(lease_uuid) expect(subject).to receive(:get_lease_uuid).and_return(lease_uuid)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment