Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
29fa930b
Commit
29fa930b
authored
Jan 17, 2018
by
Ahmad Sherif
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Migrate .batch_lfs_pointers to Gitaly
Closes gitaly#921
parent
a403011e
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
67 additions
and
22 deletions
+67
-22
lib/gitlab/git/blob.rb
lib/gitlab/git/blob.rb
+15
-7
lib/gitlab/git/repository.rb
lib/gitlab/git/repository.rb
+4
-0
lib/gitlab/gitaly_client/blob_service.rb
lib/gitlab/gitaly_client/blob_service.rb
+20
-0
spec/lib/gitlab/git/blob_spec.rb
spec/lib/gitlab/git/blob_spec.rb
+28
-15
No files found.
lib/gitlab/git/blob.rb
View file @
29fa930b
...
@@ -34,7 +34,7 @@ module Gitlab
...
@@ -34,7 +34,7 @@ module Gitlab
def
raw
(
repository
,
sha
)
def
raw
(
repository
,
sha
)
Gitlab
::
GitalyClient
.
migrate
(
:git_blob_raw
)
do
|
is_enabled
|
Gitlab
::
GitalyClient
.
migrate
(
:git_blob_raw
)
do
|
is_enabled
|
if
is_enabled
if
is_enabled
Gitlab
::
GitalyClient
::
BlobService
.
new
(
repository
)
.
get_blob
(
oid:
sha
,
limit:
MAX_DATA_DISPLAY_SIZE
)
repository
.
gitaly_blob_client
.
get_blob
(
oid:
sha
,
limit:
MAX_DATA_DISPLAY_SIZE
)
else
else
rugged_raw
(
repository
,
sha
,
limit:
MAX_DATA_DISPLAY_SIZE
)
rugged_raw
(
repository
,
sha
,
limit:
MAX_DATA_DISPLAY_SIZE
)
end
end
...
@@ -70,12 +70,20 @@ module Gitlab
...
@@ -70,12 +70,20 @@ module Gitlab
# Returns array of Gitlab::Git::Blob
# Returns array of Gitlab::Git::Blob
# Does not guarantee blob data will be set
# Does not guarantee blob data will be set
def
batch_lfs_pointers
(
repository
,
blob_ids
)
def
batch_lfs_pointers
(
repository
,
blob_ids
)
return
[]
if
blob_ids
.
empty?
repository
.
gitaly_migrate
(
:batch_lfs_pointers
)
do
|
is_enabled
|
if
is_enabled
repository
.
gitaly_blob_client
.
batch_lfs_pointers
(
blob_ids
)
else
blob_ids
.
lazy
blob_ids
.
lazy
.
select
{
|
sha
|
possible_lfs_blob?
(
repository
,
sha
)
}
.
select
{
|
sha
|
possible_lfs_blob?
(
repository
,
sha
)
}
.
map
{
|
sha
|
rugged_raw
(
repository
,
sha
,
limit:
LFS_POINTER_MAX_SIZE
)
}
.
map
{
|
sha
|
rugged_raw
(
repository
,
sha
,
limit:
LFS_POINTER_MAX_SIZE
)
}
.
select
(
&
:lfs_pointer?
)
.
select
(
&
:lfs_pointer?
)
.
force
.
force
end
end
end
end
def
binary?
(
data
)
def
binary?
(
data
)
EncodingHelper
.
detect_libgit2_binary?
(
data
)
EncodingHelper
.
detect_libgit2_binary?
(
data
)
...
@@ -258,7 +266,7 @@ module Gitlab
...
@@ -258,7 +266,7 @@ module Gitlab
Gitlab
::
GitalyClient
.
migrate
(
:git_blob_load_all_data
)
do
|
is_enabled
|
Gitlab
::
GitalyClient
.
migrate
(
:git_blob_load_all_data
)
do
|
is_enabled
|
@data
=
begin
@data
=
begin
if
is_enabled
if
is_enabled
Gitlab
::
GitalyClient
::
BlobService
.
new
(
repository
)
.
get_blob
(
oid:
id
,
limit:
-
1
).
data
repository
.
gitaly_blob_client
.
get_blob
(
oid:
id
,
limit:
-
1
).
data
else
else
repository
.
lookup
(
id
).
content
repository
.
lookup
(
id
).
content
end
end
...
...
lib/gitlab/git/repository.rb
View file @
29fa930b
...
@@ -1331,6 +1331,10 @@ module Gitlab
...
@@ -1331,6 +1331,10 @@ module Gitlab
@gitaly_remote_client
||=
Gitlab
::
GitalyClient
::
RemoteService
.
new
(
self
)
@gitaly_remote_client
||=
Gitlab
::
GitalyClient
::
RemoteService
.
new
(
self
)
end
end
def
gitaly_blob_client
@gitaly_blob_client
||=
Gitlab
::
GitalyClient
::
BlobService
.
new
(
self
)
end
def
gitaly_conflicts_client
(
our_commit_oid
,
their_commit_oid
)
def
gitaly_conflicts_client
(
our_commit_oid
,
their_commit_oid
)
Gitlab
::
GitalyClient
::
ConflictsService
.
new
(
self
,
our_commit_oid
,
their_commit_oid
)
Gitlab
::
GitalyClient
::
ConflictsService
.
new
(
self
,
our_commit_oid
,
their_commit_oid
)
end
end
...
...
lib/gitlab/gitaly_client/blob_service.rb
View file @
29fa930b
...
@@ -32,6 +32,26 @@ module Gitlab
...
@@ -32,6 +32,26 @@ module Gitlab
binary:
Gitlab
::
Git
::
Blob
.
binary?
(
data
)
binary:
Gitlab
::
Git
::
Blob
.
binary?
(
data
)
)
)
end
end
def
batch_lfs_pointers
(
blob_ids
)
request
=
Gitaly
::
GetLFSPointersRequest
.
new
(
repository:
@gitaly_repo
,
blob_ids:
blob_ids
)
response
=
GitalyClient
.
call
(
@gitaly_repo
.
storage_name
,
:blob_service
,
:get_lfs_pointers
,
request
)
response
.
flat_map
do
|
message
|
message
.
lfs_pointers
.
map
do
|
lfs_pointer
|
Gitlab
::
Git
::
Blob
.
new
(
id:
lfs_pointer
.
oid
,
size:
lfs_pointer
.
size
,
data:
lfs_pointer
.
data
,
binary:
Gitlab
::
Git
::
Blob
.
binary?
(
lfs_pointer
.
data
)
)
end
end
end
end
end
end
end
end
end
spec/lib/gitlab/git/blob_spec.rb
View file @
29fa930b
...
@@ -260,6 +260,7 @@ describe Gitlab::Git::Blob, seed_helper: true do
...
@@ -260,6 +260,7 @@ describe Gitlab::Git::Blob, seed_helper: true do
)
)
end
end
shared_examples
'fetching batch of LFS pointers'
do
it
'returns a list of Gitlab::Git::Blob'
do
it
'returns a list of Gitlab::Git::Blob'
do
blobs
=
described_class
.
batch_lfs_pointers
(
repository
,
[
lfs_blob
.
id
])
blobs
=
described_class
.
batch_lfs_pointers
(
repository
,
[
lfs_blob
.
id
])
...
@@ -280,9 +281,21 @@ describe Gitlab::Git::Blob, seed_helper: true do
...
@@ -280,9 +281,21 @@ describe Gitlab::Git::Blob, seed_helper: true do
end
end
it
'avoids loading large blobs into memory'
do
it
'avoids loading large blobs into memory'
do
# This line could call `lookup` on `repository`, so do here before mocking.
non_lfs_blob_id
=
non_lfs_blob
.
id
expect
(
repository
).
not_to
receive
(
:lookup
)
expect
(
repository
).
not_to
receive
(
:lookup
)
described_class
.
batch_lfs_pointers
(
repository
,
[
non_lfs_blob
.
id
])
described_class
.
batch_lfs_pointers
(
repository
,
[
non_lfs_blob_id
])
end
end
context
'when Gitaly batch_lfs_pointers is enabled'
do
it_behaves_like
'fetching batch of LFS pointers'
end
context
'when Gitaly batch_lfs_pointers is disabled'
,
:disable_gitaly
do
it_behaves_like
'fetching batch of LFS pointers'
end
end
end
end
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment