Commit 29fa930b authored by Ahmad Sherif's avatar Ahmad Sherif

Migrate .batch_lfs_pointers to Gitaly

Closes gitaly#921
parent a403011e
...@@ -34,7 +34,7 @@ module Gitlab ...@@ -34,7 +34,7 @@ module Gitlab
def raw(repository, sha) def raw(repository, sha)
Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled|
if is_enabled if is_enabled
Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
else else
rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE) rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE)
end end
...@@ -70,12 +70,20 @@ module Gitlab ...@@ -70,12 +70,20 @@ module Gitlab
# Returns array of Gitlab::Git::Blob # Returns array of Gitlab::Git::Blob
# Does not guarantee blob data will be set # Does not guarantee blob data will be set
def batch_lfs_pointers(repository, blob_ids) def batch_lfs_pointers(repository, blob_ids)
return [] if blob_ids.empty?
repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled|
if is_enabled
repository.gitaly_blob_client.batch_lfs_pointers(blob_ids)
else
blob_ids.lazy blob_ids.lazy
.select { |sha| possible_lfs_blob?(repository, sha) } .select { |sha| possible_lfs_blob?(repository, sha) }
.map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) }
.select(&:lfs_pointer?) .select(&:lfs_pointer?)
.force .force
end end
end
end
def binary?(data) def binary?(data)
EncodingHelper.detect_libgit2_binary?(data) EncodingHelper.detect_libgit2_binary?(data)
...@@ -258,7 +266,7 @@ module Gitlab ...@@ -258,7 +266,7 @@ module Gitlab
Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled|
@data = begin @data = begin
if is_enabled if is_enabled
Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data
else else
repository.lookup(id).content repository.lookup(id).content
end end
......
...@@ -1331,6 +1331,10 @@ module Gitlab ...@@ -1331,6 +1331,10 @@ module Gitlab
@gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self) @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self)
end end
def gitaly_blob_client
@gitaly_blob_client ||= Gitlab::GitalyClient::BlobService.new(self)
end
def gitaly_conflicts_client(our_commit_oid, their_commit_oid) def gitaly_conflicts_client(our_commit_oid, their_commit_oid)
Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid)
end end
......
...@@ -32,6 +32,26 @@ module Gitlab ...@@ -32,6 +32,26 @@ module Gitlab
binary: Gitlab::Git::Blob.binary?(data) binary: Gitlab::Git::Blob.binary?(data)
) )
end end
def batch_lfs_pointers(blob_ids)
request = Gitaly::GetLFSPointersRequest.new(
repository: @gitaly_repo,
blob_ids: blob_ids
)
response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request)
response.flat_map do |message|
message.lfs_pointers.map do |lfs_pointer|
Gitlab::Git::Blob.new(
id: lfs_pointer.oid,
size: lfs_pointer.size,
data: lfs_pointer.data,
binary: Gitlab::Git::Blob.binary?(lfs_pointer.data)
)
end
end
end
end end
end end
end end
...@@ -260,6 +260,7 @@ describe Gitlab::Git::Blob, seed_helper: true do ...@@ -260,6 +260,7 @@ describe Gitlab::Git::Blob, seed_helper: true do
) )
end end
shared_examples 'fetching batch of LFS pointers' do
it 'returns a list of Gitlab::Git::Blob' do it 'returns a list of Gitlab::Git::Blob' do
blobs = described_class.batch_lfs_pointers(repository, [lfs_blob.id]) blobs = described_class.batch_lfs_pointers(repository, [lfs_blob.id])
...@@ -280,9 +281,21 @@ describe Gitlab::Git::Blob, seed_helper: true do ...@@ -280,9 +281,21 @@ describe Gitlab::Git::Blob, seed_helper: true do
end end
it 'avoids loading large blobs into memory' do it 'avoids loading large blobs into memory' do
# This line could call `lookup` on `repository`, so do here before mocking.
non_lfs_blob_id = non_lfs_blob.id
expect(repository).not_to receive(:lookup) expect(repository).not_to receive(:lookup)
described_class.batch_lfs_pointers(repository, [non_lfs_blob.id]) described_class.batch_lfs_pointers(repository, [non_lfs_blob_id])
end
end
context 'when Gitaly batch_lfs_pointers is enabled' do
it_behaves_like 'fetching batch of LFS pointers'
end
context 'when Gitaly batch_lfs_pointers is disabled', :disable_gitaly do
it_behaves_like 'fetching batch of LFS pointers'
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment