Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
74951d38
Commit
74951d38
authored
Oct 07, 2021
by
David Fernandez
Committed by
Bob Van Landuyt
Oct 07, 2021
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Improve readability of the cleanup tags service
parent
c1d94239
Changes
6
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
107 additions
and
83 deletions
+107
-83
app/services/container_expiration_policies/cleanup_service.rb
...services/container_expiration_policies/cleanup_service.rb
+2
-2
app/services/projects/container_repository/cleanup_tags_service.rb
...ces/projects/container_repository/cleanup_tags_service.rb
+96
-72
app/workers/cleanup_container_repository_worker.rb
app/workers/cleanup_container_repository_worker.rb
+2
-2
spec/services/container_expiration_policies/cleanup_service_spec.rb
...ces/container_expiration_policies/cleanup_service_spec.rb
+2
-2
spec/services/projects/container_repository/cleanup_tags_service_spec.rb
...rojects/container_repository/cleanup_tags_service_spec.rb
+2
-2
spec/workers/cleanup_container_repository_worker_spec.rb
spec/workers/cleanup_container_repository_worker_spec.rb
+3
-3
No files found.
app/services/container_expiration_policies/cleanup_service.rb
View file @
74951d38
...
@@ -24,8 +24,8 @@ module ContainerExpirationPolicies
...
@@ -24,8 +24,8 @@ module ContainerExpirationPolicies
begin
begin
service_result
=
Projects
::
ContainerRepository
::
CleanupTagsService
service_result
=
Projects
::
ContainerRepository
::
CleanupTagsService
.
new
(
project
,
nil
,
policy_params
.
merge
(
'container_expiration_policy'
=>
true
))
.
new
(
repository
,
nil
,
policy_params
.
merge
(
'container_expiration_policy'
=>
true
))
.
execute
(
repository
)
.
execute
rescue
StandardError
rescue
StandardError
repository
.
cleanup_unfinished!
repository
.
cleanup_unfinished!
...
...
app/services/projects/container_repository/cleanup_tags_service.rb
View file @
74951d38
...
@@ -2,148 +2,152 @@
...
@@ -2,148 +2,152 @@
module
Projects
module
Projects
module
ContainerRepository
module
ContainerRepository
class
CleanupTagsService
<
BaseService
class
CleanupTagsService
include
BaseServiceUtility
include
::
Gitlab
::
Utils
::
StrongMemoize
include
::
Gitlab
::
Utils
::
StrongMemoize
def
execute
(
container_repository
)
def
initialize
(
container_repository
,
user
=
nil
,
params
=
{})
return
error
(
'access denied'
)
unless
can_destroy?
@container_repository
=
container_repository
return
error
(
'invalid regex'
)
unless
valid_regex?
@current_user
=
user
@params
=
params
.
dup
tags
=
container_repository
.
tags
@project
=
container_repository
.
project
original_size
=
tags
.
size
@tags
=
container_repository
.
tags
tags_size
=
@tags
.
size
@counts
=
{
original_size:
tags_size
,
cached_tags_count:
0
}
end
tags
=
without_latest
(
tags
)
def
execute
tags
=
filter_by_name
(
tags
)
return
error
(
'access denied'
)
unless
can_destroy?
return
error
(
'invalid regex'
)
unless
valid_regex?
before_truncate_size
=
tags
.
size
filter_out_latest
tags
=
truncate
(
tags
)
filter_by_name
after_truncate_size
=
tags
.
size
cached_tags_count
=
populate_tags_from_cache
(
container_repository
,
tags
)
||
0
truncate
populate_from_cache
tags
=
filter_keep_n
(
container_repository
,
tags
)
filter_keep_n
tags
=
filter_by_older_than
(
container_repository
,
tags
)
filter_by_older_than
delete_tags
(
container_repository
,
tags
).
tap
do
|
result
|
delete_tags
.
merge
(
@counts
).
tap
do
|
result
|
result
[
:original_size
]
=
original_size
result
[
:before_delete_size
]
=
@tags
.
size
result
[
:before_truncate_size
]
=
before_truncate_size
result
[
:after_truncate_size
]
=
after_truncate_size
result
[
:cached_tags_count
]
=
cached_tags_count
result
[
:before_delete_size
]
=
tags
.
size
result
[
:deleted_size
]
=
result
[
:deleted
]
&
.
size
result
[
:deleted_size
]
=
result
[
:deleted
]
&
.
size
result
[
:status
]
=
:error
if
before_truncate_size
!=
after_truncate_size
result
[
:status
]
=
:error
if
@counts
[
:before_truncate_size
]
!=
@counts
[
:after_truncate_size
]
end
end
end
end
private
private
def
delete_tags
(
container_repository
,
tags
)
def
delete_tags
return
success
(
deleted:
[])
unless
tags
.
any?
return
success
(
deleted:
[])
unless
@tags
.
any?
tag_names
=
tags
.
map
(
&
:name
)
service
=
Projects
::
ContainerRepository
::
DeleteTagsService
.
new
(
service
=
Projects
::
ContainerRepository
::
DeleteTagsService
.
new
(
container_repository
.
project
,
@
project
,
current_user
,
@
current_user
,
tags:
tag_names
,
tags:
@tags
.
map
(
&
:name
)
,
container_expiration_policy:
params
[
'container_expiration_policy'
]
container_expiration_policy:
container_expiration_policy
)
)
service
.
execute
(
container_repository
)
service
.
execute
(
@
container_repository
)
end
end
def
without_latest
(
tags
)
def
filter_out_latest
tags
.
reject
(
&
:latest?
)
@tags
.
reject!
(
&
:latest?
)
end
end
def
order_by_date
(
tags
)
def
order_by_date
now
=
DateTime
.
current
now
=
DateTime
.
current
tags
.
sort_by
{
|
tag
|
tag
.
created_at
||
now
}.
reverse
@tags
.
sort_by!
{
|
tag
|
tag
.
created_at
||
now
}
.
reverse!
end
end
def
filter_by_name
(
tags
)
def
filter_by_name
regex_delete
=
::
Gitlab
::
UntrustedRegexp
.
new
(
"
\\
A
#{
params
[
'name_regex_delete'
]
||
params
[
'name_regex'
]
}
\\
z"
)
regex_delete
=
::
Gitlab
::
UntrustedRegexp
.
new
(
"
\\
A
#{
name_regex_delete
||
name_regex
}
\\
z"
)
regex_retain
=
::
Gitlab
::
UntrustedRegexp
.
new
(
"
\\
A
#{
params
[
'name_regex_keep'
]
}
\\
z"
)
regex_retain
=
::
Gitlab
::
UntrustedRegexp
.
new
(
"
\\
A
#{
name_regex_keep
}
\\
z"
)
tags
.
select
do
|
tag
|
@tags
.
select!
do
|
tag
|
# regex_retain will override any overlapping matches by regex_delete
# regex_retain will override any overlapping matches by regex_delete
regex_delete
.
match?
(
tag
.
name
)
&&
!
regex_retain
.
match?
(
tag
.
name
)
regex_delete
.
match?
(
tag
.
name
)
&&
!
regex_retain
.
match?
(
tag
.
name
)
end
end
end
end
def
filter_keep_n
(
container_repository
,
tags
)
def
filter_keep_n
return
tags
unless
params
[
'keep_n'
]
return
unless
keep_n
tags
=
order_by_date
(
tags
)
order_by_date
cache_tags
(
container_repository
,
tags
.
first
(
keep_n
))
cache_tags
(
@tags
.
first
(
keep_n_as_integer
))
tags
.
drop
(
keep_n
)
@tags
=
@tags
.
drop
(
keep_n_as_integer
)
end
end
def
filter_by_older_than
(
container_repository
,
tags
)
def
filter_by_older_than
return
tags
unless
older_than
return
unless
older_than
older_than_timestamp
=
older_than_in_seconds
.
ago
older_than_timestamp
=
older_than_in_seconds
.
ago
tags
,
tags_to_keep
=
tags
.
partition
do
|
tag
|
@tags
,
tags_to_keep
=
@
tags
.
partition
do
|
tag
|
tag
.
created_at
&&
tag
.
created_at
<
older_than_timestamp
tag
.
created_at
&&
tag
.
created_at
<
older_than_timestamp
end
end
cache_tags
(
container_repository
,
tags_to_keep
)
cache_tags
(
tags_to_keep
)
tags
end
end
def
can_destroy?
def
can_destroy?
return
true
if
params
[
'container_expiration_policy'
]
return
true
if
container_expiration_policy
can?
(
current_user
,
:destroy_container_image
,
project
)
can?
(
@current_user
,
:destroy_container_image
,
@
project
)
end
end
def
valid_regex?
def
valid_regex?
%w(name_regex_delete name_regex name_regex_keep)
.
each
do
|
param_name
|
%w(name_regex_delete name_regex name_regex_keep)
.
each
do
|
param_name
|
regex
=
params
[
param_name
]
regex
=
@
params
[
param_name
]
::
Gitlab
::
UntrustedRegexp
.
new
(
regex
)
unless
regex
.
blank?
::
Gitlab
::
UntrustedRegexp
.
new
(
regex
)
unless
regex
.
blank?
end
end
true
true
rescue
RegexpError
=>
e
rescue
RegexpError
=>
e
::
Gitlab
::
ErrorTracking
.
log_exception
(
e
,
project_id:
project
.
id
)
::
Gitlab
::
ErrorTracking
.
log_exception
(
e
,
project_id:
@
project
.
id
)
false
false
end
end
def
truncate
(
tags
)
def
truncate
return
tags
unless
throttling_enabled?
@counts
[
:before_truncate_size
]
=
@tags
.
size
return
tags
if
max_list_size
==
0
@counts
[
:after_truncate_size
]
=
@tags
.
size
return
unless
throttling_enabled?
return
if
max_list_size
==
0
# truncate the list to make sure that after the #filter_keep_n
# truncate the list to make sure that after the #filter_keep_n
# execution, the resulting list will be max_list_size
# execution, the resulting list will be max_list_size
truncated_size
=
max_list_size
+
keep_n
truncated_size
=
max_list_size
+
keep_n
_as_integer
return
tags
if
tags
.
size
<=
truncated_size
return
if
@
tags
.
size
<=
truncated_size
tags
.
sample
(
truncated_size
)
@tags
=
@tags
.
sample
(
truncated_size
)
@counts
[
:after_truncate_size
]
=
@tags
.
size
end
end
def
populate_
tags_from_cache
(
container_repository
,
tags
)
def
populate_
from_cache
cache
(
container_repository
).
populate
(
tags
)
if
caching_enabled?
(
container_repository
)
@counts
[
:cached_tags_count
]
=
cache
.
populate
(
@tags
)
if
caching_enabled?
end
end
def
cache_tags
(
container_repository
,
tags
)
def
cache_tags
(
tags
)
cache
(
container_repository
).
insert
(
tags
,
older_than_in_seconds
)
if
caching_enabled?
(
container_repository
)
cache
.
insert
(
tags
,
older_than_in_seconds
)
if
caching_enabled?
end
end
def
cache
(
container_repository
)
def
cache
# TODO Implement https://gitlab.com/gitlab-org/gitlab/-/issues/340277 to avoid passing
# the container repository parameter which is bad for a memoized function
strong_memoize
(
:cache
)
do
strong_memoize
(
:cache
)
do
::
Projects
::
ContainerRepository
::
CacheTagsCreatedAtService
.
new
(
container_repository
)
::
Projects
::
ContainerRepository
::
CacheTagsCreatedAtService
.
new
(
@
container_repository
)
end
end
end
end
def
caching_enabled?
(
container_repository
)
def
caching_enabled?
params
[
'container_expiration_policy'
]
&&
container_expiration_policy
&&
older_than
.
present?
&&
older_than
.
present?
&&
Feature
.
enabled?
(
:container_registry_expiration_policies_caching
,
container_repository
.
project
)
Feature
.
enabled?
(
:container_registry_expiration_policies_caching
,
@
project
)
end
end
def
throttling_enabled?
def
throttling_enabled?
...
@@ -155,7 +159,11 @@ module Projects
...
@@ -155,7 +159,11 @@ module Projects
end
end
def
keep_n
def
keep_n
params
[
'keep_n'
].
to_i
@params
[
'keep_n'
]
end
def
keep_n_as_integer
keep_n
.
to_i
end
end
def
older_than_in_seconds
def
older_than_in_seconds
...
@@ -165,7 +173,23 @@ module Projects
...
@@ -165,7 +173,23 @@ module Projects
end
end
def
older_than
def
older_than
params
[
'older_than'
]
@params
[
'older_than'
]
end
def
name_regex_delete
@params
[
'name_regex_delete'
]
end
def
name_regex
@params
[
'name_regex'
]
end
def
name_regex_keep
@params
[
'name_regex_keep'
]
end
def
container_expiration_policy
@params
[
'container_expiration_policy'
]
end
end
end
end
end
end
...
...
app/workers/cleanup_container_repository_worker.rb
View file @
74951d38
...
@@ -28,8 +28,8 @@ class CleanupContainerRepositoryWorker
...
@@ -28,8 +28,8 @@ class CleanupContainerRepositoryWorker
end
end
result
=
Projects
::
ContainerRepository
::
CleanupTagsService
result
=
Projects
::
ContainerRepository
::
CleanupTagsService
.
new
(
project
,
current_user
,
params
)
.
new
(
container_repository
,
current_user
,
params
)
.
execute
(
container_repository
)
.
execute
if
run_by_container_expiration_policy?
&&
result
[
:status
]
==
:success
if
run_by_container_expiration_policy?
&&
result
[
:status
]
==
:success
container_repository
.
reset_expiration_policy_started_at!
container_repository
.
reset_expiration_policy_started_at!
...
...
spec/services/container_expiration_policies/cleanup_service_spec.rb
View file @
74951d38
...
@@ -24,8 +24,8 @@ RSpec.describe ContainerExpirationPolicies::CleanupService do
...
@@ -24,8 +24,8 @@ RSpec.describe ContainerExpirationPolicies::CleanupService do
it
'completely clean up the repository'
do
it
'completely clean up the repository'
do
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
)
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
)
.
to
receive
(
:new
).
with
(
project
,
nil
,
cleanup_tags_service_params
).
and_return
(
cleanup_tags_service
)
.
to
receive
(
:new
).
with
(
repository
,
nil
,
cleanup_tags_service_params
).
and_return
(
cleanup_tags_service
)
expect
(
cleanup_tags_service
).
to
receive
(
:execute
).
with
(
repository
).
and_return
(
status: :success
)
expect
(
cleanup_tags_service
).
to
receive
(
:execute
).
and_return
(
status: :success
)
response
=
subject
response
=
subject
...
...
spec/services/projects/container_repository/cleanup_tags_service_spec.rb
View file @
74951d38
...
@@ -9,7 +9,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_
...
@@ -9,7 +9,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_
let_it_be
(
:project
,
reload:
true
)
{
create
(
:project
,
:private
)
}
let_it_be
(
:project
,
reload:
true
)
{
create
(
:project
,
:private
)
}
let
(
:repository
)
{
create
(
:container_repository
,
:root
,
project:
project
)
}
let
(
:repository
)
{
create
(
:container_repository
,
:root
,
project:
project
)
}
let
(
:service
)
{
described_class
.
new
(
project
,
user
,
params
)
}
let
(
:service
)
{
described_class
.
new
(
repository
,
user
,
params
)
}
let
(
:tags
)
{
%w[latest A Ba Bb C D E]
}
let
(
:tags
)
{
%w[latest A Ba Bb C D E]
}
before
do
before
do
...
@@ -39,7 +39,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_
...
@@ -39,7 +39,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_
end
end
describe
'#execute'
do
describe
'#execute'
do
subject
{
service
.
execute
(
repository
)
}
subject
{
service
.
execute
}
shared_examples
'reading and removing tags'
do
|
caching_enabled:
true
|
shared_examples
'reading and removing tags'
do
|
caching_enabled:
true
|
context
'when no params are specified'
do
context
'when no params are specified'
do
...
...
spec/workers/cleanup_container_repository_worker_spec.rb
View file @
74951d38
...
@@ -17,7 +17,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
...
@@ -17,7 +17,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
it
'executes the destroy service'
do
it
'executes the destroy service'
do
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
).
to
receive
(
:new
)
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
).
to
receive
(
:new
)
.
with
(
project
,
user
,
params
.
merge
(
'container_expiration_policy'
=>
false
))
.
with
(
repository
,
user
,
params
.
merge
(
'container_expiration_policy'
=>
false
))
.
and_return
(
service
)
.
and_return
(
service
)
expect
(
service
).
to
receive
(
:execute
)
expect
(
service
).
to
receive
(
:execute
)
...
@@ -49,7 +49,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
...
@@ -49,7 +49,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
expect
(
repository
).
to
receive
(
:start_expiration_policy!
).
and_call_original
expect
(
repository
).
to
receive
(
:start_expiration_policy!
).
and_call_original
expect
(
repository
).
to
receive
(
:reset_expiration_policy_started_at!
).
and_call_original
expect
(
repository
).
to
receive
(
:reset_expiration_policy_started_at!
).
and_call_original
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
).
to
receive
(
:new
)
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
).
to
receive
(
:new
)
.
with
(
project
,
nil
,
params
.
merge
(
'container_expiration_policy'
=>
true
))
.
with
(
repository
,
nil
,
params
.
merge
(
'container_expiration_policy'
=>
true
))
.
and_return
(
service
)
.
and_return
(
service
)
expect
(
service
).
to
receive
(
:execute
).
and_return
(
status: :success
)
expect
(
service
).
to
receive
(
:execute
).
and_return
(
status: :success
)
...
@@ -62,7 +62,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
...
@@ -62,7 +62,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
expect
(
repository
).
to
receive
(
:start_expiration_policy!
).
and_call_original
expect
(
repository
).
to
receive
(
:start_expiration_policy!
).
and_call_original
expect
(
repository
).
not_to
receive
(
:reset_expiration_policy_started_at!
)
expect
(
repository
).
not_to
receive
(
:reset_expiration_policy_started_at!
)
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
).
to
receive
(
:new
)
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
).
to
receive
(
:new
)
.
with
(
project
,
nil
,
params
.
merge
(
'container_expiration_policy'
=>
true
))
.
with
(
repository
,
nil
,
params
.
merge
(
'container_expiration_policy'
=>
true
))
.
and_return
(
service
)
.
and_return
(
service
)
expect
(
service
).
to
receive
(
:execute
).
and_return
(
status: :error
,
message:
'timeout while deleting tags'
)
expect
(
service
).
to
receive
(
:execute
).
and_return
(
status: :error
,
message:
'timeout while deleting tags'
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment