Add Container Registry API

This includes a set of APIs to manipulate container registry.
This includes also an ability to delete tags based on requested
criteria, like keep-last-n, matching-name, older-than.

app/models/container_repository.rb

 # frozen_string_literal: true
 
 class ContainerRepository < ActiveRecord::Base
+  include Gitlab::Utils::StrongMemoize
+
   belongs_to :project
 
   validates :name, length: { minimum: 0, allow_nil: false }
@@ -8,6 +10,8 @@ class ContainerRepository < ActiveRecord::Base
 
   delegate :client, to: :registry
 
+  scope :ordered, -> { order(:name) }
+
   # rubocop: disable CodeReuse/ServiceClass
   def registry
     @registry ||= begin
@@ -39,11 +43,12 @@ class ContainerRepository < ActiveRecord::Base
   end
 
   def tags
-    return @tags if defined?(@tags)
     return [] unless manifest && manifest['tags']
 
-    @tags = manifest['tags'].map do |tag|
-      ContainerRegistry::Tag.new(self, tag)
+    strong_memoize(:tags) do
+      manifest['tags'].sort.map do |tag|
+        ContainerRegistry::Tag.new(self, tag)
+      end
     end
   end
 

app/policies/container_repository_policy.rb

+# frozen_string_literal: true
+
+class ContainerRepositoryPolicy < BasePolicy
+  delegate { @subject.project }
+end

app/serializers/container_repository_entity.rb

@@ -3,7 +3,7 @@
 class ContainerRepositoryEntity < Grape::Entity
   include RequestAwareEntity
 
-  expose :id, :path, :location
+  expose :id, :name, :path, :location, :created_at
 
   expose :tags_path do |repository|
     project_registry_repository_tags_path(project, repository, format: :json)

app/serializers/container_tag_entity.rb

@@ -3,7 +3,7 @@
 class ContainerTagEntity < Grape::Entity
   include RequestAwareEntity
 
-  expose :name, :location, :revision, :short_revision, :total_size, :created_at
+  expose :name, :path, :location, :digest, :revision, :short_revision, :total_size, :created_at
 
   expose :destroy_path, if: -> (*) { can_destroy? } do |tag|
     project_registry_repository_tag_path(project, tag.repository, tag.name)

app/services/concerns/exclusive_lease_guard.rb

@@ -6,9 +6,14 @@
 #
 # `#try_obtain_lease` takes a block which will be run if it was able to
 # obtain  the lease. Implement `#lease_timeout` to configure the timeout
-# for the exclusive lease. Optionally override `#lease_key` to set the
+# for the exclusive lease.
+#
+# Optionally override `#lease_key` to set the
 # lease key, it defaults to the class name with underscores.
 #
+# Optionally override `#lease_release?` to prevent the job to
+# be re-executed more often than LEASE_TIMEOUT.
+#
 module ExclusiveLeaseGuard
   extend ActiveSupport::Concern
 
@@ -23,7 +28,7 @@ module ExclusiveLeaseGuard
     begin
       yield lease
     ensure
-      release_lease(lease)
+      release_lease(lease) if lease_release?
     end
   end
 
@@ -40,6 +45,10 @@ module ExclusiveLeaseGuard
           "#{self.class.name} does not implement #{__method__}"
   end
 
+  def lease_release?
+    true
+  end
+
   def release_lease(uuid)
     Gitlab::ExclusiveLease.cancel(lease_key, uuid)
   end

app/services/projects/container_repository/cleanup_tags_service.rb

+# frozen_string_literal: true
+
+module Projects
+  module ContainerRepository
+    class CleanupTagsService < BaseService
+      def execute(container_repository)
+        return error('feature disabled') unless can_use?
+        return error('access denied') unless can_admin?
+
+        tags = container_repository.tags
+        tags_by_digest = group_by_digest(tags)
+
+        tags = without_latest(tags)
+        tags = filter_by_name(tags)
+        tags = with_manifest(tags)
+        tags = order_by_date(tags)
+        tags = filter_keep_n(tags)
+        tags = filter_by_older_than(tags)
+
+        deleted_tags = delete_tags(tags, tags_by_digest)
+
+        success(deleted: deleted_tags.map(&:name))
+      end
+
+      private
+
+      def delete_tags(tags_to_delete, tags_by_digest)
+        deleted_digests = group_by_digest(tags_to_delete).select do |digest, tags|
+          delete_tag_digest(digest, tags, tags_by_digest[digest])
+        end
+
+        deleted_digests.values.flatten
+      end
+
+      def delete_tag_digest(digest, tags, other_tags)
+        # Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
+        # we have to remove all tags due
+        # to Docker Distribution bug unable
+        # to delete single tag
+        return unless tags.count == other_tags.count
+
+        # delete all tags
+        tags.map(&:delete)
+      end
+
+      def group_by_digest(tags)
+        tags.group_by(&:digest)
+      end
+
+      def without_latest(tags)
+        tags.reject(&:latest?)
+      end
+
+      def with_manifest(tags)
+        tags.select(&:valid?)
+      end
+
+      def order_by_date(tags)
+        now = DateTime.now
+        tags.sort_by { |tag| tag.created_at || now }.reverse
+      end
+
+      def filter_by_name(tags)
+        regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z")
+
+        tags.select do |tag|
+          regex.scan(tag.name).any?
+        end
+      end
+
+      def filter_keep_n(tags)
+        tags.drop(params['keep_n'].to_i)
+      end
+
+      def filter_by_older_than(tags)
+        return tags unless params['older_than']
+
+        older_than = ChronicDuration.parse(params['older_than']).seconds.ago
+
+        tags.select do |tag|
+          tag.created_at && tag.created_at < older_than
+        end
+      end
+
+      def can_admin?
+        can?(current_user, :admin_container_image, project)
+      end
+
+      def can_use?
+        Feature.enabled?(:container_registry_cleanup, project, default_enabled: true)
+      end
+    end
+  end
+end

app/workers/all_queues.yml

@@ -90,13 +90,15 @@
 - object_pool:object_pool_join
 - object_pool:object_pool_destroy
 
+- container_repository:delete_container_repository
+- container_repository:cleanup_container_repository
+
 - default
 - mailers # ActionMailer::DeliveryJob.queue_name
 
 - authorized_projects
 - background_migration
 - create_gpg_signature
-- delete_container_repository
 - delete_merged_branches
 - delete_user
 - email_receiver

app/workers/cleanup_container_repository_worker.rb

+# frozen_string_literal: true
+
+class CleanupContainerRepositoryWorker
+  include ApplicationWorker
+  include ExclusiveLeaseGuard
+
+  queue_namespace :container_repository
+
+  LEASE_TIMEOUT = 1.hour
+
+  attr_reader :container_repository, :current_user
+
+  def perform(current_user_id, container_repository_id, params)
+    @current_user = User.find_by_id(current_user_id)
+    @container_repository = ContainerRepository.find_by_id(container_repository_id)
+
+    return unless valid?
+
+    try_obtain_lease do
+      Projects::ContainerRepository::CleanupTagsService
+        .new(project, current_user, params)
+        .execute(container_repository)
+    end
+  end
+
+  private
+
+  def valid?
+    current_user && container_repository && project
+  end
+
+  def project
+    container_repository&.project
+  end
+
+  # For ExclusiveLeaseGuard concern
+  def lease_key
+    @lease_key ||= "container_repository:cleanup_tags:#{container_repository.id}"
+  end
+
+  # For ExclusiveLeaseGuard concern
+  def lease_timeout
+    LEASE_TIMEOUT
+  end
+
+  # For ExclusiveLeaseGuard concern
+  def lease_release?
+    # we don't allow to execute this worker
+    # more often than LEASE_TIMEOUT
+    # for given container repository
+    false
+  end
+end

app/workers/delete_container_repository_worker.rb

@@ -4,6 +4,8 @@ class DeleteContainerRepositoryWorker
   include ApplicationWorker
   include ExclusiveLeaseGuard
 
+  queue_namespace :container_repository
+
   LEASE_TIMEOUT = 1.hour
 
   attr_reader :container_repository

changelogs/unreleased/container-repository-cleanup-api.yml

+---
+title: Add Container Registry API with cleanup function
+merge_request: 24303
+author:
+type: added

config/sidekiq_queues.yml

@@ -47,7 +47,6 @@
   - [project_service, 1]
   - [delete_user, 1]
   - [todos_destroyer, 1]
-  - [delete_container_repository, 1]
   - [delete_merged_branches, 1]
   - [authorized_projects, 1]
   - [expire_build_instance_artifacts, 1]
@@ -81,6 +80,7 @@
   - [delete_diff_files, 1]
   - [detect_repository_languages, 1]
   - [auto_devops, 2]
+  - [container_repository, 1]
   - [object_pool, 1]
   - [repository_cleanup, 1]
   - [delete_stored_files, 1]

db/post_migrate/20190115054215_migrate_delete_container_repository_worker.rb

+# frozen_string_literal: true
+
+class MigrateDeleteContainerRepositoryWorker < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    sidekiq_queue_migrate('delete_container_repository', to: 'container_repository:delete_container_repository')
+  end
+
+  def down
+    sidekiq_queue_migrate('container_repository:delete_container_repository', to: 'delete_container_repository')
+  end
+end

doc/api/README.md

@@ -16,6 +16,7 @@ The following API resources are available:
 - [Broadcast messages](broadcast_messages.md)
 - [Code snippets](snippets.md)
 - [Commits](commits.md)
+- [Container Registry](container_registry.md)
 - [Custom attributes](custom_attributes.md)
 - [Deploy keys](deploy_keys.md), and [deploy keys for multiple projects](deploy_key_multiple_projects.md)
 - [Deployments](deployments.md)

doc/api/container_registry.md

+# Container Registry API
+
+## List registry repositories
+
+Get a list of registry repositories in a project.
+
+```
+GET /projects/:id/registry/repositories
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id`      | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
+
+
+```bash
+curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories"
+```
+
+Example response:
+
+```json
+[
+  {
+    "id": 1,
+    "name": "",
+    "path": "group/project",
+    "location": "gitlab.example.com:5000/group/project",
+    "created_at": "2019-01-10T13:38:57.391Z"
+  },
+  {
+    "id": 2,
+    "name": "releases",
+    "path": "group/project/releases",
+    "location": "gitlab.example.com:5000/group/project/releases",
+    "created_at": "2019-01-10T13:39:08.229Z"
+  }
+]
+```
+
+## Delete registry repository
+
+Get a list of repository commits in a project.
+
+This operation is executed asynchronously and it might take
+time to get executed.
+
+```
+DELETE /projects/:id/registry/repositories/:repository_id
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id`      | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
+| `repository_id` | integer | yes | The ID of registry repository
+
+```bash
+curl -X DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2"
+```
+
+## List repository tags
+
+Get a list of tags for given registry repository.
+
+```
+GET /projects/:id/registry/repositories/:repository_id/tags
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id`      | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
+| `repository_id` | integer | yes | The ID of registry repository
+
+```bash
+curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
+```
+
+Example response:
+
+```json
+[
+  {
+    "name": "A",
+    "path": "group/project:A",
+    "location": "gitlab.example.com:5000/group/project:A"
+  },
+  {
+    "name": "latest",
+    "path": "group/project:latest",
+    "location": "gitlab.example.com:5000/group/project:latest"
+  }
+]
+```
+
+## Delete repository tags (in bulk)
+
+Delete repository tags in bulk based on given criteria.
+
+This API performs a following set of the operations:
+
+1. It schedules asynchronous job executed in background,
+1. It never removes tag named `latest`,
+1. It removes the tags matching given `name_regex` only,
+1. It orders all tags by creation date. The creation date is time of the manifest creation. It is not a time of tag push,
+1. It keeps N latest matching tags (if specified),
+1. It only removes tags that are older than (if specified).
+
+These operations are executed asynchronously and it might
+take time to get executed. This API can be run at most
+once an hour for given container repository.
+
+Due to [Docker Distribution deficiency](ce-21405) it does
+not remove tags whose manifest is shared by multiple tags
+
+```
+DELETE /projects/:id/registry/repositories/:repository_id/tags
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id`      | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
+| `repository_id` | integer | yes | The ID of registry repository
+| `name_regex` | string | yes | The regex of the name to delete. To delete all tags specify `.*`
+| `keep_n` | integer | no | The amount of latest tags of given name to keep
+| `older_than` | string | no | Tags to delete that are older than given timespec, written in human readable form `1h`, `1d`, `1month` |
+
+Examples:
+
+1. Remove tag names that are matching GIT SHA, keep always at least 5, and remove ones that are older than 2 days:
+
+    ```bash
+    curl -X DELETE -F 'name_regex=[0-9a-z]{40}' -F 'keep_n=5' -F 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
+    ```
+
+2. Remove all tags, but keep always latest 5:
+
+    ```bash
+    curl -X DELETE -F 'name_regex=.*' -F 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
+    ```
+
+3. Remove all tags that are older than 1 month:
+
+    ```bash
+    curl -X DELETE -F 'name_regex=.*' -F 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
+    ```
+
+## Get a details repository tag
+
+Get a details of registry repository tag
+
+```
+GET /projects/:id/registry/repositories/:repository_id/tags/:tag_name
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id`      | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
+| `repository_id` | integer | yes | The ID of registry repository
+| `tag_name` | string | yes | The name of tag
+
+```bash
+curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
+```
+
+Example response:
+
+```json
+{
+  "name": "v10.0.0",
+  "path": "group/project:latest",
+  "location": "gitlab.example.com:5000/group/project:latest",
+  "revision": "e9ed9d87c881d8c2fd3a31b41904d01ba0b836e7fd15240d774d811a1c248181",
+  "short_revision": "e9ed9d87c",
+  "digest": "sha256:c3490dcf10ffb6530c1303522a1405dfaf7daecd8f38d3e6a1ba19ea1f8a1751",
+  "created_at": "2019-01-06T16:49:51.272+00:00",
+  "total_size": 350224384
+}
+```
+
+## Delete a repository tag
+
+Delete a registry repository tag
+
+```
+DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id`      | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
+| `repository_id` | integer | yes | The ID of registry repository
+| `tag_name` | string | yes | The name of tag
+
+```bash
+curl -X DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
+```
+
+[ce-21405]: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405

lib/api/api.rb

@@ -100,6 +100,7 @@ module API
     mount ::API::CircuitBreakers
     mount ::API::Commits
     mount ::API::CommitStatuses
+    mount ::API::ContainerRegistry
     mount ::API::DeployKeys
     mount ::API::Deployments
     mount ::API::Environments

lib/api/container_registry.rb

+# frozen_string_literal: true
+
+module API
+  class ContainerRegistry < Grape::API
+    include PaginationParams
+
+    REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(
+      tag_name: API::NO_SLASH_URL_PART_REGEX)
+
+    before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) }
+    before { authorize_read_container_images! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc 'Get a project container repositories' do
+        detail 'This feature was introduced in GitLab 11.8.'
+        success Entities::ContainerRegistry::Repository
+      end
+      params do
+        use :pagination
+      end
+      get ':id/registry/repositories' do
+        repositories = user_project.container_repositories.ordered
+
+        present paginate(repositories), with: Entities::ContainerRegistry::Repository
+      end
+
+      desc 'Delete repository' do
+        detail 'This feature was introduced in GitLab 11.8.'
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+      end
+      delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_admin_container_image!
+
+        DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id)
+
+        status :accepted
+      end
+
+      desc 'Get a list of repositories tags' do
+        detail 'This feature was introduced in GitLab 11.8.'
+        success Entities::ContainerRegistry::Tag
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+        use :pagination
+      end
+      get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_read_container_image!
+
+        tags = Kaminari.paginate_array(repository.tags)
+        present paginate(tags), with: Entities::ContainerRegistry::Tag
+      end
+
+      desc 'Delete repository tags (in bulk)' do
+        detail 'This feature was introduced in GitLab 11.8.'
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+        requires :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all'
+        optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name'
+        optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month'
+      end
+      delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_admin_container_image!
+
+        CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id,
+          declared_params.except(:repository_id)) # rubocop: disable CodeReuse/ActiveRecord
+
+        status :accepted
+      end
+
+      desc 'Get a details about repository tag' do
+        detail 'This feature was introduced in GitLab 11.8.'
+        success Entities::ContainerRegistry::TagDetails
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+        requires :tag_name, type: String, desc: 'The name of the tag'
+      end
+      get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_read_container_image!
+        validate_tag!
+
+        present tag, with: Entities::ContainerRegistry::TagDetails
+      end
+
+      desc 'Delete repository tag' do
+        detail 'This feature was introduced in GitLab 11.8.'
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+        requires :tag_name, type: String, desc: 'The name of the tag'
+      end
+      delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_destroy_container_image!
+        validate_tag!
+
+        tag.delete
+
+        status :ok
+      end
+    end
+
+    helpers do
+      def authorize_read_container_images!
+        authorize! :read_container_image, user_project
+      end
+
+      def authorize_read_container_image!
+        authorize! :read_container_image, repository
+      end
+
+      def authorize_update_container_image!
+        authorize! :update_container_image, repository
+      end
+
+      def authorize_destroy_container_image!
+        authorize! :admin_container_image, repository
+      end
+
+      def authorize_admin_container_image!
+        authorize! :admin_container_image, repository
+      end
+
+      def repository
+        @repository ||= user_project.container_repositories.find(params[:repository_id])
+      end
+
+      def tag
+        @tag ||= repository.tag(params[:tag_name])
+      end
+
+      def validate_tag!
+        not_found!('Tag') unless tag.valid?
+      end
+    end
+  end
+end

lib/api/entities/container_registry.rb

+# frozen_string_literal: true
+
+module API
+  module Entities
+    module ContainerRegistry
+      class Repository < Grape::Entity
+        expose :id
+        expose :name
+        expose :path
+        expose :location
+        expose :created_at
+      end
+
+      class Tag < Grape::Entity
+        expose :name
+        expose :path
+        expose :location
+      end
+
+      class TagDetails < Tag
+        expose :revision
+        expose :short_revision
+        expose :digest
+        expose :created_at
+        expose :total_size
+      end
+    end
+  end
+end

lib/container_registry/tag.rb

@@ -2,6 +2,8 @@
 
 module ContainerRegistry
   class Tag
+    include Gitlab::Utils::StrongMemoize
+
     attr_reader :repository, :name
 
     delegate :registry, :client, to: :repository
@@ -15,6 +17,10 @@ module ContainerRegistry
       manifest.present?
     end
 
+    def latest?
+      name == "latest"
+    end
+
     def v1?
       manifest && manifest['schemaVersion'] == 1
     end
@@ -24,7 +30,9 @@ module ContainerRegistry
     end
 
     def manifest
-      @manifest ||= client.repository_manifest(repository.path, name)
+      strong_memoize(:manifest) do
+        client.repository_manifest(repository.path, name)
+      end
     end
 
     def path
@@ -42,36 +50,44 @@ module ContainerRegistry
     end
 
     def digest
-      @digest ||= client.repository_tag_digest(repository.path, name)
+      strong_memoize(:digest) do
+        client.repository_tag_digest(repository.path, name)
+      end
     end
 
     def config_blob
-      return @config_blob if defined?(@config_blob)
       return unless manifest && manifest['config']
 
-      @config_blob = repository.blob(manifest['config'])
+      strong_memoize(:config_blob) do
+        repository.blob(manifest['config'])
+      end
     end
 
     def config
-      return unless config_blob
+      return unless config_blob&.data
 
-      @config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data
+      strong_memoize(:config) do
+        ContainerRegistry::Config.new(self, config_blob)
+      end
     end
 
     def created_at
       return unless config
 
-      @created_at ||= DateTime.rfc3339(config['created'])
+      strong_memoize(:created_at) do
+        DateTime.rfc3339(config['created'])
+      end
     end
 
     def layers
-      return @layers if defined?(@layers)
       return unless manifest
 
-      layers = manifest['layers'] || manifest['fsLayers']
+      strong_memoize(:layers) do
+        layers = manifest['layers'] || manifest['fsLayers']
 
-      @layers = layers.map do |layer|
-        repository.blob(layer)
+        layers.map do |layer|
+          repository.blob(layer)
+        end
       end
     end
 

spec/controllers/projects/registry/tags_controller_spec.rb

@@ -19,7 +19,7 @@ describe Projects::Registry::TagsController do
     end
 
     before do
-      stub_container_registry_tags(repository: /image/, tags: tags)
+      stub_container_registry_tags(repository: /image/, tags: tags, with_manifest: true)
     end
 
     context 'when user can control the registry' do

spec/factories/container_repositories.rb

 FactoryBot.define do
   factory :container_repository do
-    name 'test_container_image'
+    name 'test_image'
     project
 
     transient do

spec/features/container_registry_spec.rb

@@ -25,7 +25,7 @@ describe "Container Registry", :js do
 
   context 'when there are image repositories' do
     before do
-      stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest])
+      stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest], with_manifest: true)
       project.container_repositories << container_repository
     end
 

spec/fixtures/api/schemas/registry/repository.json

@@ -2,20 +2,27 @@
   "type": "object",
   "required" : [
     "id",
+    "name",
     "path",
     "location",
-    "tags_path"
+    "created_at"
   ],
   "properties" : {
     "id": {
       "type": "integer"
     },
+    "name": {
+      "type": "string"
+    },
     "path": {
       "type": "string"
     },
     "location": {
       "type": "string"
     },
+    "created_at": {
+      "type": "date-time"
+    },
     "tags_path": {
       "type": "string"
     },

spec/fixtures/api/schemas/registry/tag.json

@@ -2,15 +2,22 @@
   "type": "object",
   "required" : [
     "name",
+    "path",
     "location"
   ],
   "properties" : {
     "name": {
       "type": "string"
     },
+    "path": {
+      "type": "string"
+    },
     "location": {
       "type": "string"
     },
+    "digest": {
+      "type": "string"
+    },
     "revision": {
       "type": "string"
     },

spec/requests/api/container_registry_spec.rb

+require 'spec_helper'
+
+describe API::ContainerRegistry do
+  set(:project) { create(:project, :private) }
+  set(:maintainer) { create(:user) }
+  set(:developer) { create(:user) }
+  set(:reporter) { create(:user) }
+  set(:guest) { create(:user) }
+
+  let(:root_repository) { create(:container_repository, :root, project: project) }
+  let(:test_repository) { create(:container_repository, project: project) }
+
+  let(:api_user) { maintainer }
+
+  before do
+    project.add_maintainer(maintainer)
+    project.add_developer(developer)
+    project.add_reporter(reporter)
+    project.add_guest(guest)
+
+    stub_feature_flags(container_registry_api: true)
+    stub_container_registry_config(enabled: true)
+
+    root_repository
+    test_repository
+  end
+
+  shared_examples 'being disallowed' do |param|
+    context "for #{param}" do
+      let(:api_user) { public_send(param) }
+
+      it 'returns access denied' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+
+    context "for anonymous" do
+      let(:api_user) { nil }
+
+      it 'returns not found' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+  end
+
+  describe 'GET /projects/:id/registry/repositories' do
+    subject { get api("/projects/#{project.id}/registry/repositories", api_user) }
+
+    it_behaves_like 'being disallowed', :guest
+
+    context 'for reporter' do
+      let(:api_user) { reporter }
+
+      it 'returns a list of repositories' do
+        subject
+
+        expect(json_response.length).to eq(2)
+        expect(json_response.map { |repository| repository['id'] }).to contain_exactly(
+          root_repository.id, test_repository.id)
+      end
+
+      it 'returns a matching schema' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(response).to match_response_schema('registry/repositories')
+      end
+    end
+  end
+
+  describe 'DELETE /projects/:id/registry/repositories/:repository_id' do
+    subject { delete api("/projects/#{project.id}/registry/repositories/#{root_repository.id}", api_user) }
+
+    it_behaves_like 'being disallowed', :developer
+
+    context 'for maintainer' do
+      let(:api_user) { maintainer }
+
+      it 'schedules removal of repository' do
+        expect(DeleteContainerRepositoryWorker).to receive(:perform_async)
+          .with(maintainer.id, root_repository.id)
+
+        subject
+
+        expect(response).to have_gitlab_http_status(:accepted)
+      end
+    end
+  end
+
+  describe 'GET /projects/:id/registry/repositories/:repository_id/tags' do
+    subject { get api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags", api_user) }
+
+    it_behaves_like 'being disallowed', :guest
+
+    context 'for reporter' do
+      let(:api_user) { reporter }
+
+      before do
+        stub_container_registry_tags(repository: root_repository.path, tags: %w(rootA latest))
+      end
+
+      it 'returns a list of tags' do
+        subject
+
+        expect(json_response.length).to eq(2)
+        expect(json_response.map { |repository| repository['name'] }).to eq %w(latest rootA)
+      end
+
+      it 'returns a matching schema' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(response).to match_response_schema('registry/tags')
+      end
+    end
+  end
+
+  describe 'DELETE /projects/:id/registry/repositories/:repository_id/tags' do
+    subject { delete api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags", api_user), params: params }
+
+    it_behaves_like 'being disallowed', :developer do
+      let(:params) do
+        { name_regex: 'v10.*' }
+      end
+    end
+
+    context 'for maintainer' do
+      let(:api_user) { maintainer }
+
+      context 'without required parameters' do
+        let(:params) { }
+
+        it 'returns bad request' do
+          subject
+
+          expect(response).to have_gitlab_http_status(:bad_request)
+        end
+      end
+
+      context 'passes all declared parameters' do
+        let(:params) do
+          { name_regex: 'v10.*',
+            keep_n: 100,
+            older_than: '1 day',
+            other: 'some value' }
+        end
+
+        let(:worker_params) do
+          { name_regex: 'v10.*',
+            keep_n: 100,
+            older_than: '1 day' }
+        end
+
+        it 'schedules cleanup of tags repository' do
+          expect(CleanupContainerRepositoryWorker).to receive(:perform_async)
+            .with(maintainer.id, root_repository.id, worker_params)
+
+          subject
+
+          expect(response).to have_gitlab_http_status(:accepted)
+        end
+      end
+    end
+  end
+
+  describe 'GET /projects/:id/registry/repositories/:repository_id/tags/:tag_name' do
+    subject { get api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags/rootA", api_user) }
+
+    it_behaves_like 'being disallowed', :guest
+
+    context 'for reporter' do
+      let(:api_user) { reporter }
+
+      before do
+        stub_container_registry_tags(repository: root_repository.path, tags: %w(rootA), with_manifest: true)
+      end
+
+      it 'returns a details of tag' do
+        subject
+
+        expect(json_response).to include(
+          'name' => 'rootA',
+          'digest' => 'sha256:4c8e63ca4cb663ce6c688cb06f1c372b088dac5b6d7ad7d49cd620d85cf72a15',
+          'revision' => 'd7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac',
+          'total_size' => 2319870)
+      end
+
+      it 'returns a matching schema' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(response).to match_response_schema('registry/tag')
+      end
+    end
+  end
+
+  describe 'DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name' do
+    subject { delete api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags/rootA", api_user) }
+
+    it_behaves_like 'being disallowed', :developer
+
+    context 'for maintainer' do
+      let(:api_user) { maintainer }
+
+      before do
+        stub_container_registry_tags(repository: root_repository.path, tags: %w(rootA), with_manifest: true)
+      end
+
+      it 'properly removes tag' do
+        expect_any_instance_of(ContainerRegistry::Client)
+          .to receive(:delete_repository_tag).with(root_repository.path,
+            'sha256:4c8e63ca4cb663ce6c688cb06f1c372b088dac5b6d7ad7d49cd620d85cf72a15')
+
+        subject
+
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+    end
+  end
+end

spec/serializers/container_tag_entity_spec.rb

@@ -16,7 +16,7 @@ describe ContainerTagEntity do
 
   before do
     stub_container_registry_config(enabled: true)
-    stub_container_registry_tags(repository: /image/, tags: %w[test])
+    stub_container_registry_tags(repository: /image/, tags: %w[test], with_manifest: true)
     allow(request).to receive(:project).and_return(project)
     allow(request).to receive(:current_user).and_return(user)
   end

spec/services/projects/container_repository/cleanup_tags_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Projects::ContainerRepository::CleanupTagsService do
+  set(:user) { create(:user) }
+  set(:project) { create(:project, :private) }
+  set(:repository) { create(:container_repository, :root, project: project) }
+
+  let(:service) { described_class.new(project, user, params) }
+
+  before do
+    project.add_maintainer(user)
+
+    stub_feature_flags(container_registry_cleanup: true)
+
+    stub_container_registry_config(enabled: true)
+
+    stub_container_registry_tags(
+      repository: repository.path,
+      tags: %w(latest A Ba Bb C D E))
+
+    stub_tag_digest('latest', 'sha256:configA')
+    stub_tag_digest('A', 'sha256:configA')
+    stub_tag_digest('Ba', 'sha256:configB')
+    stub_tag_digest('Bb', 'sha256:configB')
+    stub_tag_digest('C', 'sha256:configC')
+    stub_tag_digest('D', 'sha256:configD')
+    stub_tag_digest('E', nil)
+
+    stub_digest_config('sha256:configA', 1.hour.ago)
+    stub_digest_config('sha256:configB', 5.days.ago)
+    stub_digest_config('sha256:configC', 1.month.ago)
+    stub_digest_config('sha256:configD', nil)
+  end
+
+  describe '#execute' do
+    subject { service.execute(repository) }
+
+    context 'when no params are specified' do
+      let(:params) { {} }
+
+      it 'does not remove anything' do
+        expect_any_instance_of(ContainerRegistry::Client).not_to receive(:delete_repository_tag)
+
+        is_expected.to include(status: :success, deleted: [])
+      end
+    end
+
+    context 'when regex matching everything is specified' do
+      let(:params) do
+        { 'name_regex' => '.*' }
+      end
+
+      it 'does remove B* and C' do
+        # The :A cannot be removed as config is shared with :latest
+        # The :E cannot be removed as it does not have valid manifest
+
+        expect_delete('sha256:configB').twice
+        expect_delete('sha256:configC')
+        expect_delete('sha256:configD')
+
+        is_expected.to include(status: :success, deleted: %w(D Bb Ba C))
+      end
+    end
+
+    context 'when regex matching specific tags is used' do
+      let(:params) do
+        { 'name_regex' => 'C|D' }
+      end
+
+      it 'does remove C and D' do
+        expect_delete('sha256:configC')
+        expect_delete('sha256:configD')
+
+        is_expected.to include(status: :success, deleted: %w(D C))
+      end
+    end
+
+    context 'when removing a tagged image that is used by another tag' do
+      let(:params) do
+        { 'name_regex' => 'Ba' }
+      end
+
+      it 'does not remove the tag' do
+        # Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
+
+        is_expected.to include(status: :success, deleted: [])
+      end
+    end
+
+    context 'when removing keeping only 3' do
+      let(:params) do
+        { 'name_regex' => '.*',
+          'keep_n' => 3 }
+      end
+
+      it 'does remove C as it is oldest' do
+        expect_delete('sha256:configC')
+
+        is_expected.to include(status: :success, deleted: %w(C))
+      end
+    end
+
+    context 'when removing older than 1 day' do
+      let(:params) do
+        { 'name_regex' => '.*',
+          'older_than' => '1 day' }
+      end
+
+      it 'does remove B* and C as they are older than 1 day' do
+        expect_delete('sha256:configB').twice
+        expect_delete('sha256:configC')
+
+        is_expected.to include(status: :success, deleted: %w(Bb Ba C))
+      end
+    end
+
+    context 'when combining all parameters' do
+      let(:params) do
+        { 'name_regex' => '.*',
+          'keep_n' => 1,
+          'older_than' => '1 day' }
+      end
+
+      it 'does remove B* and C' do
+        expect_delete('sha256:configB').twice
+        expect_delete('sha256:configC')
+
+        is_expected.to include(status: :success, deleted: %w(Bb Ba C))
+      end
+    end
+  end
+
+  private
+
+  def stub_tag_digest(tag, digest)
+    allow_any_instance_of(ContainerRegistry::Client)
+      .to receive(:repository_tag_digest)
+      .with(repository.path, tag) { digest }
+
+    allow_any_instance_of(ContainerRegistry::Client)
+      .to receive(:repository_manifest)
+      .with(repository.path, tag) do
+      { 'config' => { 'digest' => digest } } if digest
+    end
+  end
+
+  def stub_digest_config(digest, created_at)
+    allow_any_instance_of(ContainerRegistry::Client)
+      .to receive(:blob)
+      .with(repository.path, digest, nil) do
+      { 'created' => created_at.to_datetime.rfc3339 }.to_json if created_at
+    end
+  end
+
+  def expect_delete(digest)
+    expect_any_instance_of(ContainerRegistry::Client)
+      .to receive(:delete_repository_tag)
+      .with(repository.path, digest)
+  end
+end

spec/support/helpers/stub_gitlab_calls.rb

@@ -36,31 +36,41 @@ module StubGitlabCalls
       .to receive(:full_access_token).and_return('token')
   end
 
-  def stub_container_registry_tags(repository: :any, tags:)
+  def stub_container_registry_tags(repository: :any, tags: [], with_manifest: false)
     repository = any_args if repository == :any
 
     allow_any_instance_of(ContainerRegistry::Client)
       .to receive(:repository_tags).with(repository)
       .and_return({ 'tags' => tags })
 
-    allow_any_instance_of(ContainerRegistry::Client)
-      .to receive(:repository_manifest).with(repository, anything)
-      .and_return(stub_container_registry_tag_manifest)
+    if with_manifest
+      tags.each do |tag|
+        allow_any_instance_of(ContainerRegistry::Client)
+          .to receive(:repository_tag_digest)
+          .with(repository, tag)
+          .and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3' \
+                      '72b088dac5b6d7ad7d49cd620d85cf72a15')
+      end
 
-    allow_any_instance_of(ContainerRegistry::Client)
-      .to receive(:blob).with(repository, anything, 'application/octet-stream')
-      .and_return(stub_container_registry_blob)
+      allow_any_instance_of(ContainerRegistry::Client)
+        .to receive(:repository_manifest).with(repository, anything)
+        .and_return(stub_container_registry_tag_manifest_content)
+
+      allow_any_instance_of(ContainerRegistry::Client)
+        .to receive(:blob).with(repository, anything, 'application/octet-stream')
+        .and_return(stub_container_registry_blob_content)
+    end
   end
 
   private
 
-  def stub_container_registry_tag_manifest
+  def stub_container_registry_tag_manifest_content
     fixture_path = 'spec/fixtures/container_registry/tag_manifest.json'
 
     JSON.parse(File.read(Rails.root + fixture_path))
   end
 
-  def stub_container_registry_blob
+  def stub_container_registry_blob_content
     fixture_path = 'spec/fixtures/container_registry/config_blob.json'
 
     File.read(Rails.root + fixture_path)

spec/workers/cleanup_container_repository_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_state do
+  let(:repository) { create(:container_repository) }
+  let(:project) { repository.project }
+  let(:user) { project.owner }
+  let(:params) { { key: 'value' } }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    let(:service) { instance_double(Projects::ContainerRepository::CleanupTagsService) }
+
+    before do
+      allow(Projects::ContainerRepository::CleanupTagsService).to receive(:new)
+        .with(project, user, params).and_return(service)
+    end
+
+    it 'executes the destroy service' do
+      expect(service).to receive(:execute)
+
+      subject.perform(user.id, repository.id, params)
+    end
+
+    it 'does not raise error when user could not be found' do
+      expect do
+        subject.perform(-1, repository.id, params)
+      end.not_to raise_error
+    end
+
+    it 'does not raise error when repository could not be found' do
+      expect do
+        subject.perform(user.id, -1, params)
+      end.not_to raise_error
+    end
+
+    context 'when executed twice in short period' do
+      it 'executes service only for the first time' do
+        expect(service).to receive(:execute).once
+
+        2.times { subject.perform(user.id, repository.id, params) }
+      end
+    end
+  end
+end