Commit 51d303df authored by GitLab Bot's avatar GitLab Bot

Automatic merge of gitlab-org/gitlab-ce master

parents 5bf06528 dc609187
......@@ -27,7 +27,7 @@ class Dashboard::MilestonesController < Dashboard::ApplicationController
def group_milestones
groups = GroupsFinder.new(current_user, all_available: false).execute
DashboardGroupMilestone.build_collection(groups)
DashboardGroupMilestone.build_collection(groups, params)
end
# See [#39545](https://gitlab.com/gitlab-org/gitlab-ce/issues/39545) for info about the deprecation of dynamic milestones
......
......@@ -115,7 +115,7 @@ class Groups::MilestonesController < Groups::ApplicationController
end
def search_params
params.permit(:state).merge(group_ids: group.id)
params.permit(:state, :search_title).merge(group_ids: group.id)
end
end
......
......@@ -147,6 +147,6 @@ class Projects::MilestonesController < Projects::ApplicationController
groups = project_group.self_and_ancestors.select(:id)
end
params.permit(:state).merge(project_ids: @project.id, group_ids: groups)
params.permit(:state, :search_title).merge(project_ids: @project.id, group_ids: groups)
end
end
......@@ -22,6 +22,7 @@ class MilestonesFinder
items = Milestone.all
items = by_groups_and_projects(items)
items = by_title(items)
items = by_search_title(items)
items = by_state(items)
order(items)
......@@ -43,6 +44,14 @@ class MilestonesFinder
end
# rubocop: enable CodeReuse/ActiveRecord
def by_search_title(items)
if params[:search_title].present?
items.search_title(params[:search_title])
else
items
end
end
def by_state(items)
Milestone.filter_by_state(items, params[:state])
end
......
# frozen_string_literal: true
class ContainerRepository < ActiveRecord::Base
include Gitlab::Utils::StrongMemoize
belongs_to :project
validates :name, length: { minimum: 0, allow_nil: false }
......@@ -8,6 +10,8 @@ class ContainerRepository < ActiveRecord::Base
delegate :client, to: :registry
scope :ordered, -> { order(:name) }
# rubocop: disable CodeReuse/ServiceClass
def registry
@registry ||= begin
......@@ -39,11 +43,12 @@ class ContainerRepository < ActiveRecord::Base
end
def tags
return @tags if defined?(@tags)
return [] unless manifest && manifest['tags']
@tags = manifest['tags'].map do |tag|
ContainerRegistry::Tag.new(self, tag)
strong_memoize(:tags) do
manifest['tags'].sort.map do |tag|
ContainerRegistry::Tag.new(self, tag)
end
end
end
......
......@@ -11,11 +11,12 @@ class DashboardGroupMilestone < GlobalMilestone
@group_name = milestone.group.full_name
end
def self.build_collection(groups)
Milestone.of_groups(groups.select(:id))
def self.build_collection(groups, params)
milestones = Milestone.of_groups(groups.select(:id))
.reorder_by_due_date_asc
.order_by_name_asc
.active
.map { |m| new(m) }
milestones = milestones.search_title(params[:search_title]) if params[:search_title].present?
milestones.map { |m| new(m) }
end
end
......@@ -28,6 +28,7 @@ class GlobalMilestone
items = Milestone.of_projects(projects)
.reorder_by_due_date_asc
.order_by_name_asc
items = items.search_title(params[:search_title]) if params[:search_title].present?
Milestone.filter_by_state(items, params[:state]).map { |m| new(m) }
end
......
......@@ -6,9 +6,10 @@ class GroupMilestone < GlobalMilestone
def self.build_collection(group, projects, params)
params =
{ state: params[:state] }
{ state: params[:state], search_title: params[:search_title] }
project_milestones = Milestone.of_projects(projects)
project_milestones = project_milestones.search_title(params[:search_title]) if params[:search_title].present?
child_milestones = Milestone.filter_by_state(project_milestones, params[:state])
grouped_milestones = child_milestones.group_by(&:title)
......
......@@ -79,7 +79,7 @@ class Milestone < ActiveRecord::Base
alias_attribute :name, :title
class << self
# Searches for milestones matching the given query.
# Searches for milestones with a matching title or description.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
......@@ -90,6 +90,17 @@ class Milestone < ActiveRecord::Base
fuzzy_search(query, [:title, :description])
end
# Searches for milestones with a matching title.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String
#
# Returns an ActiveRecord::Relation.
def search_title(query)
fuzzy_search(query, [:title])
end
def filter_by_state(milestones, state)
case state
when 'closed' then milestones.closed
......
# frozen_string_literal: true
class ContainerRepositoryPolicy < BasePolicy
delegate { @subject.project }
end
......@@ -3,7 +3,7 @@
class ContainerRepositoryEntity < Grape::Entity
include RequestAwareEntity
expose :id, :path, :location
expose :id, :name, :path, :location, :created_at
expose :tags_path do |repository|
project_registry_repository_tags_path(project, repository, format: :json)
......
......@@ -3,7 +3,7 @@
class ContainerTagEntity < Grape::Entity
include RequestAwareEntity
expose :name, :location, :revision, :short_revision, :total_size, :created_at
expose :name, :path, :location, :digest, :revision, :short_revision, :total_size, :created_at
expose :destroy_path, if: -> (*) { can_destroy? } do |tag|
project_registry_repository_tag_path(project, tag.repository, tag.name)
......
......@@ -6,9 +6,14 @@
#
# `#try_obtain_lease` takes a block which will be run if it was able to
# obtain the lease. Implement `#lease_timeout` to configure the timeout
# for the exclusive lease. Optionally override `#lease_key` to set the
# for the exclusive lease.
#
# Optionally override `#lease_key` to set the
# lease key, it defaults to the class name with underscores.
#
# Optionally override `#lease_release?` to prevent the job to
# be re-executed more often than LEASE_TIMEOUT.
#
module ExclusiveLeaseGuard
extend ActiveSupport::Concern
......@@ -23,7 +28,7 @@ module ExclusiveLeaseGuard
begin
yield lease
ensure
release_lease(lease)
release_lease(lease) if lease_release?
end
end
......@@ -40,6 +45,10 @@ module ExclusiveLeaseGuard
"#{self.class.name} does not implement #{__method__}"
end
def lease_release?
true
end
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
end
......
# frozen_string_literal: true
module Projects
module ContainerRepository
class CleanupTagsService < BaseService
def execute(container_repository)
return error('feature disabled') unless can_use?
return error('access denied') unless can_admin?
tags = container_repository.tags
tags_by_digest = group_by_digest(tags)
tags = without_latest(tags)
tags = filter_by_name(tags)
tags = with_manifest(tags)
tags = order_by_date(tags)
tags = filter_keep_n(tags)
tags = filter_by_older_than(tags)
deleted_tags = delete_tags(tags, tags_by_digest)
success(deleted: deleted_tags.map(&:name))
end
private
def delete_tags(tags_to_delete, tags_by_digest)
deleted_digests = group_by_digest(tags_to_delete).select do |digest, tags|
delete_tag_digest(digest, tags, tags_by_digest[digest])
end
deleted_digests.values.flatten
end
def delete_tag_digest(digest, tags, other_tags)
# Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
# we have to remove all tags due
# to Docker Distribution bug unable
# to delete single tag
return unless tags.count == other_tags.count
# delete all tags
tags.map(&:delete)
end
def group_by_digest(tags)
tags.group_by(&:digest)
end
def without_latest(tags)
tags.reject(&:latest?)
end
def with_manifest(tags)
tags.select(&:valid?)
end
def order_by_date(tags)
now = DateTime.now
tags.sort_by { |tag| tag.created_at || now }.reverse
end
def filter_by_name(tags)
regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z")
tags.select do |tag|
regex.scan(tag.name).any?
end
end
def filter_keep_n(tags)
tags.drop(params['keep_n'].to_i)
end
def filter_by_older_than(tags)
return tags unless params['older_than']
older_than = ChronicDuration.parse(params['older_than']).seconds.ago
tags.select do |tag|
tag.created_at && tag.created_at < older_than
end
end
def can_admin?
can?(current_user, :admin_container_image, project)
end
def can_use?
Feature.enabled?(:container_registry_cleanup, project, default_enabled: true)
end
end
end
end
......@@ -13,6 +13,8 @@
.top-area
= render 'shared/milestones_filter', counts: @milestone_states
.nav-controls
= render 'shared/milestones/search_form'
.milestones
%ul.content-list
......
......@@ -4,6 +4,7 @@
= render 'shared/milestones_filter', counts: @milestone_states
.nav-controls
= render 'shared/milestones/search_form'
= render 'shared/milestones_sort_dropdown'
- if can?(current_user, :admin_milestone, @group)
= link_to "New milestone", new_group_milestone_path(@group), class: "btn btn-success"
......
......@@ -6,6 +6,7 @@
= render 'shared/milestones_filter', counts: milestone_counts(@project.milestones)
.nav-controls
= render 'shared/milestones/search_form'
= render 'shared/milestones_sort_dropdown'
- if can?(current_user, :admin_milestone, @project)
= link_to new_project_milestone_path(@project), class: "btn btn-success qa-new-project-milestone", title: 'New milestone' do
......
= form_tag request.path, method: :get do |f|
= search_field_tag :search_title, params[:search_title],
placeholder: _('Filter by milestone name'),
class: 'form-control input-short',
spellcheck: false
= hidden_field_tag :state, params[:state]
= hidden_field_tag :sort, params[:sort]
......@@ -90,13 +90,15 @@
- object_pool:object_pool_join
- object_pool:object_pool_destroy
- container_repository:delete_container_repository
- container_repository:cleanup_container_repository
- default
- mailers # ActionMailer::DeliveryJob.queue_name
- authorized_projects
- background_migration
- create_gpg_signature
- delete_container_repository
- delete_merged_branches
- delete_user
- email_receiver
......
# frozen_string_literal: true
class CleanupContainerRepositoryWorker
include ApplicationWorker
include ExclusiveLeaseGuard
queue_namespace :container_repository
LEASE_TIMEOUT = 1.hour
attr_reader :container_repository, :current_user
def perform(current_user_id, container_repository_id, params)
@current_user = User.find_by_id(current_user_id)
@container_repository = ContainerRepository.find_by_id(container_repository_id)
return unless valid?
try_obtain_lease do
Projects::ContainerRepository::CleanupTagsService
.new(project, current_user, params)
.execute(container_repository)
end
end
private
def valid?
current_user && container_repository && project
end
def project
container_repository&.project
end
# For ExclusiveLeaseGuard concern
def lease_key
@lease_key ||= "container_repository:cleanup_tags:#{container_repository.id}"
end
# For ExclusiveLeaseGuard concern
def lease_timeout
LEASE_TIMEOUT
end
# For ExclusiveLeaseGuard concern
def lease_release?
# we don't allow to execute this worker
# more often than LEASE_TIMEOUT
# for given container repository
false
end
end
......@@ -4,6 +4,8 @@ class DeleteContainerRepositoryWorker
include ApplicationWorker
include ExclusiveLeaseGuard
queue_namespace :container_repository
LEASE_TIMEOUT = 1.hour
attr_reader :container_repository
......
---
title: Adds milestone search
merge_request: 24265
author: Jacopo Beschi @jacopo-beschi
type: added
---
title: Add Container Registry API with cleanup function
merge_request: 24303
author:
type: added
......@@ -47,7 +47,6 @@
- [project_service, 1]
- [delete_user, 1]
- [todos_destroyer, 1]
- [delete_container_repository, 1]
- [delete_merged_branches, 1]
- [authorized_projects, 1]
- [expire_build_instance_artifacts, 1]
......@@ -81,6 +80,7 @@
- [delete_diff_files, 1]
- [detect_repository_languages, 1]
- [auto_devops, 2]
- [container_repository, 1]
- [object_pool, 1]
- [repository_cleanup, 1]
- [delete_stored_files, 1]
......
# frozen_string_literal: true
class MigrateDeleteContainerRepositoryWorker < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
sidekiq_queue_migrate('delete_container_repository', to: 'container_repository:delete_container_repository')
end
def down
sidekiq_queue_migrate('container_repository:delete_container_repository', to: 'delete_container_repository')
end
end
......@@ -16,6 +16,7 @@ The following API resources are available:
- [Broadcast messages](broadcast_messages.md)
- [Code snippets](snippets.md)
- [Commits](commits.md)
- [Container Registry](container_registry.md)
- [Custom attributes](custom_attributes.md)
- [Deploy keys](deploy_keys.md), and [deploy keys for multiple projects](deploy_key_multiple_projects.md)
- [Deployments](deployments.md)
......
# Container Registry API
This is the API docs of the [GitLab Container Registry](../user/project/container_registry.md).
## List registry repositories
Get a list of registry repositories in a project.
```
GET /projects/:id/registry/repositories
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories"
```
Example response:
```json
[
{
"id": 1,
"name": "",
"path": "group/project",
"location": "gitlab.example.com:5000/group/project",
"created_at": "2019-01-10T13:38:57.391Z"
},
{
"id": 2,
"name": "releases",
"path": "group/project/releases",
"location": "gitlab.example.com:5000/group/project/releases",
"created_at": "2019-01-10T13:39:08.229Z"
}
]
```
## Delete registry repository
Get a list of repository commits in a project.
This operation is executed asynchronously and might take some time to get executed.
```
DELETE /projects/:id/registry/repositories/:repository_id
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
```bash
curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2"
```
## List repository tags
Get a list of tags for given registry repository.
```
GET /projects/:id/registry/repositories/:repository_id/tags
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
Example response:
```json
[
{
"name": "A",
"path": "group/project:A",
"location": "gitlab.example.com:5000/group/project:A"
},
{
"name": "latest",
"path": "group/project:latest",
"location": "gitlab.example.com:5000/group/project:latest"
}
]
```
## Get details of a repository tag
Get details of a registry repository tag.
```
GET /projects/:id/registry/repositories/:repository_id/tags/:tag_name
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
| `tag_name` | string | yes | The name of tag. |
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
```
Example response:
```json
{
"name": "v10.0.0",
"path": "group/project:latest",
"location": "gitlab.example.com:5000/group/project:latest",
"revision": "e9ed9d87c881d8c2fd3a31b41904d01ba0b836e7fd15240d774d811a1c248181",
"short_revision": "e9ed9d87c",
"digest": "sha256:c3490dcf10ffb6530c1303522a1405dfaf7daecd8f38d3e6a1ba19ea1f8a1751",
"created_at": "2019-01-06T16:49:51.272+00:00",
"total_size": 350224384
}
```
## Delete a repository tag
Delete a registry repository tag.
```
DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
| `tag_name` | string | yes | The name of tag. |
```bash
curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
```
## Delete repository tags in bulk
Delete repository tags in bulk based on given criteria.
```
DELETE /projects/:id/registry/repositories/:repository_id/tags
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
| `name_regex` | string | yes | The regex of the name to delete. To delete all tags specify `.*`. |
| `keep_n` | integer | no | The amount of latest tags of given name to keep. |
| `older_than` | string | no | Tags to delete that are older than the given time, written in human readable form `1h`, `1d`, `1month`. |
This API call performs the following operations:
1. It orders all tags by creation date. The creation date is the time of the
manifest creation, not the time of tag push.
1. It removes only the tags matching the given `name_regex`.
1. It never removes the tag named `latest`.
1. It keeps N latest matching tags (if `keep_n` is specified).
1. It only removes tags that are older than X amount of time (if `older_than` is specified).
1. It schedules the asynchronous job to be executed in the background.
These operations are executed asynchronously and it might
take time to get executed. You can run this at most
once an hour for a given container repository.
NOTE: **Note:**
Due to a [Docker Distribution deficiency](https://gitlab.com/gitlab-org/gitlab-ce/issues/21405),
it doesn't remove tags whose manifest is shared by multiple tags.
Examples:
1. Remove tag names that are matching the regex (Git SHA), keep always at least 5,
and remove ones that are older than 2 days:
```bash
curl --request DELETE --data 'name_regex=[0-9a-z]{40}' --data 'keep_n=5' --data 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
2. Remove all tags, but keep always the latest 5:
```bash
curl --request DELETE --data 'name_regex=.*' --data 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
3. Remove all tags that are older than 1 month:
```bash
curl --request DELETE --data 'name_regex=.*' --data 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
......@@ -107,6 +107,7 @@ module API
mount ::API::CircuitBreakers
mount ::API::Commits
mount ::API::CommitStatuses
mount ::API::ContainerRegistry
mount ::API::DeployKeys
mount ::API::Deployments
mount ::API::Environments
......
# frozen_string_literal: true
module API
class ContainerRegistry < Grape::API
include PaginationParams
REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(
tag_name: API::NO_SLASH_URL_PART_REGEX)
before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) }
before { authorize_read_container_images! }
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get a project container repositories' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::Repository
end
params do
use :pagination
end
get ':id/registry/repositories' do
repositories = user_project.container_repositories.ordered
present paginate(repositories), with: Entities::ContainerRegistry::Repository
end
desc 'Delete repository' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
end
delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_admin_container_image!
DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id)
status :accepted
end
desc 'Get a list of repositories tags' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::Tag
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
use :pagination
end
get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_read_container_image!
tags = Kaminari.paginate_array(repository.tags)
present paginate(tags), with: Entities::ContainerRegistry::Tag
end
desc 'Delete repository tags (in bulk)' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all'
optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name'
optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month'
end
delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_admin_container_image!
CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id,
declared_params.except(:repository_id)) # rubocop: disable CodeReuse/ActiveRecord
status :accepted
end
desc 'Get a details about repository tag' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::TagDetails
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :tag_name, type: String, desc: 'The name of the tag'
end
get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_read_container_image!
validate_tag!
present tag, with: Entities::ContainerRegistry::TagDetails
end
desc 'Delete repository tag' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :tag_name, type: String, desc: 'The name of the tag'
end
delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_destroy_container_image!
validate_tag!
tag.delete
status :ok
end
end
helpers do
def authorize_read_container_images!
authorize! :read_container_image, user_project
end
def authorize_read_container_image!
authorize! :read_container_image, repository
end
def authorize_update_container_image!
authorize! :update_container_image, repository
end
def authorize_destroy_container_image!
authorize! :admin_container_image, repository
end
def authorize_admin_container_image!
authorize! :admin_container_image, repository
end
def repository
@repository ||= user_project.container_repositories.find(params[:repository_id])
end
def tag
@tag ||= repository.tag(params[:tag_name])
end
def validate_tag!
not_found!('Tag') unless tag.valid?
end
end
end
end
# frozen_string_literal: true
module API
module Entities
module ContainerRegistry
class Repository < Grape::Entity
expose :id
expose :name
expose :path
expose :location
expose :created_at
end
class Tag < Grape::Entity
expose :name
expose :path
expose :location
end
class TagDetails < Tag
expose :revision
expose :short_revision
expose :digest
expose :created_at
expose :total_size
end
end
end
end
......@@ -2,6 +2,8 @@
module ContainerRegistry
class Tag
include Gitlab::Utils::StrongMemoize
attr_reader :repository, :name
delegate :registry, :client, to: :repository
......@@ -15,6 +17,10 @@ module ContainerRegistry
manifest.present?
end
def latest?
name == "latest"
end
def v1?
manifest && manifest['schemaVersion'] == 1
end
......@@ -24,7 +30,9 @@ module ContainerRegistry
end
def manifest
@manifest ||= client.repository_manifest(repository.path, name)
strong_memoize(:manifest) do
client.repository_manifest(repository.path, name)
end
end
def path
......@@ -42,36 +50,44 @@ module ContainerRegistry
end
def digest
@digest ||= client.repository_tag_digest(repository.path, name)
strong_memoize(:digest) do
client.repository_tag_digest(repository.path, name)
end
end
def config_blob
return @config_blob if defined?(@config_blob)
return unless manifest && manifest['config']
@config_blob = repository.blob(manifest['config'])
strong_memoize(:config_blob) do
repository.blob(manifest['config'])
end
end
def config
return unless config_blob
return unless config_blob&.data
@config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data
strong_memoize(:config) do
ContainerRegistry::Config.new(self, config_blob)
end
end
def created_at
return unless config
@created_at ||= DateTime.rfc3339(config['created'])
strong_memoize(:created_at) do
DateTime.rfc3339(config['created'])
end
end
def layers
return @layers if defined?(@layers)
return unless manifest
layers = manifest['layers'] || manifest['fsLayers']
strong_memoize(:layers) do
layers = manifest['layers'] || manifest['fsLayers']
@layers = layers.map do |layer|
repository.blob(layer)
layers.map do |layer|
repository.blob(layer)
end
end
end
......
......@@ -9,7 +9,7 @@ module Gitlab
#
# Example usage:
#
# union = Gitlab::SQL::Union.new(user.personal_projects, user.projects)
# union = Gitlab::SQL::Union.new([user.personal_projects, user.projects])
# sql = union.to_sql
#
# Project.where("id IN (#{sql})")
......
......@@ -3899,6 +3899,9 @@ msgstr ""
msgid "Filter by commit message"
msgstr ""
msgid "Filter by milestone name"
msgstr ""
msgid "Filter by two-factor authentication"
msgstr ""
......
......@@ -56,6 +56,24 @@ describe Dashboard::MilestonesController do
expect(json_response.map { |i| i["group_name"] }.compact).to match_array(group.name)
end
it 'searches legacy project milestones by title when search_title is given' do
project_milestone = create(:milestone, title: 'Project milestone title', project: project)
get :index, params: { search_title: 'Project mil' }
expect(response.body).to include(project_milestone.title)
expect(response.body).not_to include(group_milestone.title)
end
it 'searches group milestones by title when search_title is given' do
group_milestone = create(:milestone, title: 'Group milestone title', group: group)
get :index, params: { search_title: 'Group mil' }
expect(response.body).to include(group_milestone.title)
expect(response.body).not_to include(project_milestone.title)
end
it 'should contain group and project milestones to which the user belongs to' do
get :index
......
......@@ -32,10 +32,35 @@ describe Groups::MilestonesController do
end
describe '#index' do
it 'shows group milestones page' do
get :index, params: { group_id: group.to_param }
describe 'as HTML' do
render_views
expect(response).to have_gitlab_http_status(200)
it 'shows group milestones page' do
milestone
get :index, params: { group_id: group.to_param }
expect(response).to have_gitlab_http_status(200)
expect(response.body).to include(milestone.title)
end
it 'searches legacy milestones by title when search_title is given' do
project_milestone = create(:milestone, project: project, title: 'Project milestone title')
get :index, params: { group_id: group.to_param, search_title: 'Project mil' }
expect(response.body).to include(project_milestone.title)
expect(response.body).not_to include(milestone.title)
end
it 'searches group milestones by title when search_title is given' do
group_milestone = create(:milestone, title: 'Group milestone title', group: group)
get :index, params: { group_id: group.to_param, search_title: 'Group mil' }
expect(response.body).to include(group_milestone.title)
expect(response.body).not_to include(milestone.title)
end
end
context 'as JSON' do
......
......@@ -42,10 +42,11 @@ describe Projects::MilestonesController do
describe "#index" do
context "as html" do
def render_index(project:, page:)
def render_index(project:, page:, search_title: '')
get :index, params: {
namespace_id: project.namespace.id,
project_id: project.id,
search_title: search_title,
page: page
}
end
......@@ -59,6 +60,15 @@ describe Projects::MilestonesController do
expect(milestones.where(project_id: nil)).to be_empty
end
it 'searches milestones by title when search_title is given' do
milestone1 = create(:milestone, title: 'Project milestone title', project: project)
render_index project: project, page: 1, search_title: 'Project mile'
milestones = assigns(:milestones)
expect(milestones).to eq([milestone1])
end
it 'renders paginated milestones without missing or duplicates' do
allow(Milestone).to receive(:default_per_page).and_return(2)
create_list(:milestone, 5, project: project)
......
......@@ -19,7 +19,7 @@ describe Projects::Registry::TagsController do
end
before do
stub_container_registry_tags(repository: /image/, tags: tags)
stub_container_registry_tags(repository: /image/, tags: tags, with_manifest: true)
end
context 'when user can control the registry' do
......
FactoryBot.define do
factory :container_repository do
name 'test_container_image'
name 'test_image'
project
transient do
......
......@@ -25,7 +25,7 @@ describe "Container Registry", :js do
context 'when there are image repositories' do
before do
stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest])
stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest], with_manifest: true)
project.container_repositories << container_repository
end
......
......@@ -69,6 +69,12 @@ describe MilestonesFinder do
expect(result.to_a).to contain_exactly(milestone_1)
end
it 'filters by search_title' do
result = described_class.new(params.merge(search_title: 'one t')).execute
expect(result.to_a).to contain_exactly(milestone_1)
end
end
describe '#find_by' do
......
......@@ -2,20 +2,27 @@
"type": "object",
"required" : [
"id",
"name",
"path",
"location",
"tags_path"
"created_at"
],
"properties" : {
"id": {
"type": "integer"
},
"name": {
"type": "string"
},
"path": {
"type": "string"
},
"location": {
"type": "string"
},
"created_at": {
"type": "date-time"
},
"tags_path": {
"type": "string"
},
......
......@@ -2,15 +2,22 @@
"type": "object",
"required" : [
"name",
"path",
"location"
],
"properties" : {
"name": {
"type": "string"
},
"path": {
"type": "string"
},
"location": {
"type": "string"
},
"digest": {
"type": "string"
},
"revision": {
"type": "string"
},
......
......@@ -91,6 +91,12 @@ describe GlobalMilestone do
it 'sorts collection by due date' do
expect(global_milestones.map(&:due_date)).to eq [milestone1_due_date, milestone1_due_date, milestone1_due_date, nil, nil, nil]
end
it 'filters milestones by search_title when params[:search_title] is present' do
global_milestones = described_class.build_collection(projects, { search_title: 'v1.2' })
expect(global_milestones.map(&:title)).to match_array(['Milestone v1.2', 'Milestone v1.2', 'Milestone v1.2'])
end
end
context 'when adding new milestones' do
......
......@@ -242,6 +242,29 @@ describe Milestone do
end
end
describe '#search_title' do
let(:milestone) { create(:milestone, title: 'foo', description: 'bar') }
it 'returns milestones with a matching title' do
expect(described_class.search_title(milestone.title)) .to eq([milestone])
end
it 'returns milestones with a partially matching title' do
expect(described_class.search_title(milestone.title[0..2])).to eq([milestone])
end
it 'returns milestones with a matching title regardless of the casing' do
expect(described_class.search_title(milestone.title.upcase))
.to eq([milestone])
end
it 'searches only on the title and ignores milestones with a matching description' do
create(:milestone, title: 'bar', description: 'foo')
expect(described_class.search_title(milestone.title)) .to eq([milestone])
end
end
describe '#for_projects_and_groups' do
let(:project) { create(:project) }
let(:project_other) { create(:project) }
......
require 'spec_helper'
describe API::ContainerRegistry do
set(:project) { create(:project, :private) }
set(:maintainer) { create(:user) }
set(:developer) { create(:user) }
set(:reporter) { create(:user) }
set(:guest) { create(:user) }
let(:root_repository) { create(:container_repository, :root, project: project) }
let(:test_repository) { create(:container_repository, project: project) }
let(:api_user) { maintainer }
before do
project.add_maintainer(maintainer)
project.add_developer(developer)
project.add_reporter(reporter)
project.add_guest(guest)
stub_feature_flags(container_registry_api: true)
stub_container_registry_config(enabled: true)
root_repository
test_repository
end
shared_examples 'being disallowed' do |param|
context "for #{param}" do
let(:api_user) { public_send(param) }
it 'returns access denied' do
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
end
context "for anonymous" do
let(:api_user) { nil }
it 'returns not found' do
subject
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
describe 'GET /projects/:id/registry/repositories' do
subject { get api("/projects/#{project.id}/registry/repositories", api_user) }
it_behaves_like 'being disallowed', :guest
context 'for reporter' do
let(:api_user) { reporter }
it 'returns a list of repositories' do
subject
expect(json_response.length).to eq(2)
expect(json_response.map { |repository| repository['id'] }).to contain_exactly(
root_repository.id, test_repository.id)
end
it 'returns a matching schema' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response).to match_response_schema('registry/repositories')
end
end
end
describe 'DELETE /projects/:id/registry/repositories/:repository_id' do
subject { delete api("/projects/#{project.id}/registry/repositories/#{root_repository.id}", api_user) }
it_behaves_like 'being disallowed', :developer
context 'for maintainer' do
let(:api_user) { maintainer }
it 'schedules removal of repository' do
expect(DeleteContainerRepositoryWorker).to receive(:perform_async)
.with(maintainer.id, root_repository.id)
subject
expect(response).to have_gitlab_http_status(:accepted)
end
end
end
describe 'GET /projects/:id/registry/repositories/:repository_id/tags' do
subject { get api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags", api_user) }
it_behaves_like 'being disallowed', :guest
context 'for reporter' do
let(:api_user) { reporter }
before do
stub_container_registry_tags(repository: root_repository.path, tags: %w(rootA latest))
end
it 'returns a list of tags' do
subject
expect(json_response.length).to eq(2)
expect(json_response.map { |repository| repository['name'] }).to eq %w(latest rootA)
end
it 'returns a matching schema' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response).to match_response_schema('registry/tags')
end
end
end
describe 'DELETE /projects/:id/registry/repositories/:repository_id/tags' do
subject { delete api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags", api_user), params: params }
it_behaves_like 'being disallowed', :developer do
let(:params) do
{ name_regex: 'v10.*' }
end
end
context 'for maintainer' do
let(:api_user) { maintainer }
context 'without required parameters' do
let(:params) { }
it 'returns bad request' do
subject
expect(response).to have_gitlab_http_status(:bad_request)
end
end
context 'passes all declared parameters' do
let(:params) do
{ name_regex: 'v10.*',
keep_n: 100,
older_than: '1 day',
other: 'some value' }
end
let(:worker_params) do
{ name_regex: 'v10.*',
keep_n: 100,
older_than: '1 day' }
end
it 'schedules cleanup of tags repository' do
expect(CleanupContainerRepositoryWorker).to receive(:perform_async)
.with(maintainer.id, root_repository.id, worker_params)
subject
expect(response).to have_gitlab_http_status(:accepted)
end
end
end
end
describe 'GET /projects/:id/registry/repositories/:repository_id/tags/:tag_name' do
subject { get api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags/rootA", api_user) }
it_behaves_like 'being disallowed', :guest
context 'for reporter' do
let(:api_user) { reporter }
before do
stub_container_registry_tags(repository: root_repository.path, tags: %w(rootA), with_manifest: true)
end
it 'returns a details of tag' do
subject
expect(json_response).to include(
'name' => 'rootA',
'digest' => 'sha256:4c8e63ca4cb663ce6c688cb06f1c372b088dac5b6d7ad7d49cd620d85cf72a15',
'revision' => 'd7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac',
'total_size' => 2319870)
end
it 'returns a matching schema' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response).to match_response_schema('registry/tag')
end
end
end
describe 'DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name' do
subject { delete api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags/rootA", api_user) }
it_behaves_like 'being disallowed', :developer
context 'for maintainer' do
let(:api_user) { maintainer }
before do
stub_container_registry_tags(repository: root_repository.path, tags: %w(rootA), with_manifest: true)
end
it 'properly removes tag' do
expect_any_instance_of(ContainerRegistry::Client)
.to receive(:delete_repository_tag).with(root_repository.path,
'sha256:4c8e63ca4cb663ce6c688cb06f1c372b088dac5b6d7ad7d49cd620d85cf72a15')
subject
expect(response).to have_gitlab_http_status(:ok)
end
end
end
end
......@@ -16,7 +16,7 @@ describe ContainerTagEntity do
before do
stub_container_registry_config(enabled: true)
stub_container_registry_tags(repository: /image/, tags: %w[test])
stub_container_registry_tags(repository: /image/, tags: %w[test], with_manifest: true)
allow(request).to receive(:project).and_return(project)
allow(request).to receive(:current_user).and_return(user)
end
......
# frozen_string_literal: true
require 'spec_helper'
describe Projects::ContainerRepository::CleanupTagsService do
set(:user) { create(:user) }
set(:project) { create(:project, :private) }
set(:repository) { create(:container_repository, :root, project: project) }
let(:service) { described_class.new(project, user, params) }
before do
project.add_maintainer(user)
stub_feature_flags(container_registry_cleanup: true)
stub_container_registry_config(enabled: true)
stub_container_registry_tags(
repository: repository.path,
tags: %w(latest A Ba Bb C D E))
stub_tag_digest('latest', 'sha256:configA')
stub_tag_digest('A', 'sha256:configA')
stub_tag_digest('Ba', 'sha256:configB')
stub_tag_digest('Bb', 'sha256:configB')
stub_tag_digest('C', 'sha256:configC')
stub_tag_digest('D', 'sha256:configD')
stub_tag_digest('E', nil)
stub_digest_config('sha256:configA', 1.hour.ago)
stub_digest_config('sha256:configB', 5.days.ago)
stub_digest_config('sha256:configC', 1.month.ago)
stub_digest_config('sha256:configD', nil)
end
describe '#execute' do
subject { service.execute(repository) }
context 'when no params are specified' do
let(:params) { {} }
it 'does not remove anything' do
expect_any_instance_of(ContainerRegistry::Client).not_to receive(:delete_repository_tag)
is_expected.to include(status: :success, deleted: [])
end
end
context 'when regex matching everything is specified' do
let(:params) do
{ 'name_regex' => '.*' }
end
it 'does remove B* and C' do
# The :A cannot be removed as config is shared with :latest
# The :E cannot be removed as it does not have valid manifest
expect_delete('sha256:configB').twice
expect_delete('sha256:configC')
expect_delete('sha256:configD')
is_expected.to include(status: :success, deleted: %w(D Bb Ba C))
end
end
context 'when regex matching specific tags is used' do
let(:params) do
{ 'name_regex' => 'C|D' }
end
it 'does remove C and D' do
expect_delete('sha256:configC')
expect_delete('sha256:configD')
is_expected.to include(status: :success, deleted: %w(D C))
end
end
context 'when removing a tagged image that is used by another tag' do
let(:params) do
{ 'name_regex' => 'Ba' }
end
it 'does not remove the tag' do
# Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
is_expected.to include(status: :success, deleted: [])
end
end
context 'when removing keeping only 3' do
let(:params) do
{ 'name_regex' => '.*',
'keep_n' => 3 }
end
it 'does remove C as it is oldest' do
expect_delete('sha256:configC')
is_expected.to include(status: :success, deleted: %w(C))
end
end
context 'when removing older than 1 day' do
let(:params) do
{ 'name_regex' => '.*',
'older_than' => '1 day' }
end
it 'does remove B* and C as they are older than 1 day' do
expect_delete('sha256:configB').twice
expect_delete('sha256:configC')
is_expected.to include(status: :success, deleted: %w(Bb Ba C))
end
end
context 'when combining all parameters' do
let(:params) do
{ 'name_regex' => '.*',
'keep_n' => 1,
'older_than' => '1 day' }
end
it 'does remove B* and C' do
expect_delete('sha256:configB').twice
expect_delete('sha256:configC')
is_expected.to include(status: :success, deleted: %w(Bb Ba C))
end
end
end
private
def stub_tag_digest(tag, digest)
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_tag_digest)
.with(repository.path, tag) { digest }
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_manifest)
.with(repository.path, tag) do
{ 'config' => { 'digest' => digest } } if digest
end
end
def stub_digest_config(digest, created_at)
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:blob)
.with(repository.path, digest, nil) do
{ 'created' => created_at.to_datetime.rfc3339 }.to_json if created_at
end
end
def expect_delete(digest)
expect_any_instance_of(ContainerRegistry::Client)
.to receive(:delete_repository_tag)
.with(repository.path, digest)
end
end
......@@ -38,31 +38,41 @@ module StubGitlabCalls
.to receive(:full_access_token).and_return('token')
end
def stub_container_registry_tags(repository: :any, tags:)
def stub_container_registry_tags(repository: :any, tags: [], with_manifest: false)
repository = any_args if repository == :any
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_tags).with(repository)
.and_return({ 'tags' => tags })
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_manifest).with(repository, anything)
.and_return(stub_container_registry_tag_manifest)
if with_manifest
tags.each do |tag|
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_tag_digest)
.with(repository, tag)
.and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3' \
'72b088dac5b6d7ad7d49cd620d85cf72a15')
end
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:blob).with(repository, anything, 'application/octet-stream')
.and_return(stub_container_registry_blob)
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_manifest).with(repository, anything)
.and_return(stub_container_registry_tag_manifest_content)
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:blob).with(repository, anything, 'application/octet-stream')
.and_return(stub_container_registry_blob_content)
end
end
private
def stub_container_registry_tag_manifest
def stub_container_registry_tag_manifest_content
fixture_path = 'spec/fixtures/container_registry/tag_manifest.json'
JSON.parse(File.read(Rails.root + fixture_path))
end
def stub_container_registry_blob
def stub_container_registry_blob_content
fixture_path = 'spec/fixtures/container_registry/config_blob.json'
File.read(Rails.root + fixture_path)
......
# frozen_string_literal: true
require 'spec_helper'
describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_state do
let(:repository) { create(:container_repository) }
let(:project) { repository.project }
let(:user) { project.owner }
let(:params) { { key: 'value' } }
subject { described_class.new }
describe '#perform' do
let(:service) { instance_double(Projects::ContainerRepository::CleanupTagsService) }
before do
allow(Projects::ContainerRepository::CleanupTagsService).to receive(:new)
.with(project, user, params).and_return(service)
end
it 'executes the destroy service' do
expect(service).to receive(:execute)
subject.perform(user.id, repository.id, params)
end
it 'does not raise error when user could not be found' do
expect do
subject.perform(-1, repository.id, params)
end.not_to raise_error
end
it 'does not raise error when repository could not be found' do
expect do
subject.perform(user.id, -1, params)
end.not_to raise_error
end
context 'when executed twice in short period' do
it 'executes service only for the first time' do
expect(service).to receive(:execute).once
2.times { subject.perform(user.id, repository.id, params) }
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment