Merge branch '347349-expose-deduplicated-size-of-container-images' into 'master'

Expose container repository sizes

See merge request gitlab-org/gitlab!80412

app/graphql/types/container_repository_details_type.rb

@@ -15,8 +15,19 @@ module Types
           max_page_size: 20,
           resolver: Resolvers::ContainerRepositoryTagsResolver
 
+    field :size,
+          GraphQL::Types::Float,
+          null: true,
+          description: 'Deduplicated size of the image repository in bytes. This is only available on GitLab.com for repositories created after `2021-11-04`.'
+
     def can_delete
       Ability.allowed?(current_user, :destroy_container_image, object)
     end
+
+    def size
+      object.size
+    rescue Faraday::Error
+      raise ::Gitlab::Graphql::Errors::ResourceNotAvailable, "Can't connect to the Container Registry. If this error persists, please review the troubleshooting documentation."
+    end
   end
 end

app/models/container_repository.rb

@@ -14,6 +14,8 @@ class ContainerRepository < ApplicationRecord
   ABORTABLE_MIGRATION_STATES = (ACTIVE_MIGRATION_STATES + %w[pre_import_done default]).freeze
   MIGRATION_STATES = (IDLE_MIGRATION_STATES + ACTIVE_MIGRATION_STATES).freeze
 
+  MIGRATION_PHASE_1_STARTED_AT = Date.new(2021, 11, 4).freeze
+
   TooManyImportsError = Class.new(StandardError)
   NativeImportError = Class.new(StandardError)
 
@@ -408,6 +410,16 @@ class ContainerRepository < ApplicationRecord
     update!(expiration_policy_started_at: Time.zone.now)
   end
 
+  def size
+    strong_memoize(:size) do
+      next unless Gitlab.com?
+      next if self.created_at.before?(MIGRATION_PHASE_1_STARTED_AT)
+      next unless gitlab_api_client.supports_gitlab_api?
+
+      gitlab_api_client.repository_details(self.path, with_size: true)['size_bytes']
+    end
+  end
+
   def migration_in_active_state?
     migration_state.in?(ACTIVE_MIGRATION_STATES)
   end

doc/api/container_registry.md

@@ -210,10 +210,11 @@ GET /registry/repositories/:id
 | `id`      | integer/string | yes | The ID of the registry repository accessible by the authenticated user. |
 | `tags`      | boolean | no | If the parameter is included as `true`, the response includes an array of `"tags"`. |
 | `tags_count` | boolean | no | If the parameter is included as `true`, the response includes `"tags_count"`. |
+| `size` | boolean | no | If the parameter is included as `true`, the response includes `"size"`. This is the deduplicated size of all images within the repository. Deduplication eliminates extra copies of identical data. For example, if you upload the same image twice, the Container Registry stores only one copy. This field is only available on GitLab.com for repositories created after `2021-11-04`. |
 
 ```shell
 curl --header "PRIVATE-TOKEN: <your_access_token>" \
-     "https://gitlab.example.com/api/v4/registry/repositories/2?tags=true&tags_count=true"
+     "https://gitlab.example.com/api/v4/registry/repositories/2?tags=true&tags_count=true&size=true"
 ```
 
 Example response:
@@ -234,7 +235,8 @@ Example response:
       "path": "group/project:0.0.1",
       "location": "gitlab.example.com:5000/group/project:0.0.1"
     }
-  ]
+  ],
+  "size": 2818413
 }
 ```
 

doc/api/graphql/reference/index.md

@@ -9577,6 +9577,7 @@ Details of a container repository.
 | <a id="containerrepositorydetailsname"></a>`name` | [`String!`](#string) | Name of the container repository. |
 | <a id="containerrepositorydetailspath"></a>`path` | [`String!`](#string) | Path of the container repository. |
 | <a id="containerrepositorydetailsproject"></a>`project` | [`Project!`](#project) | Project of the container registry. |
+| <a id="containerrepositorydetailssize"></a>`size` | [`Float`](#float) | Deduplicated size of the image repository in bytes. This is only available on GitLab.com for repositories created after `2021-11-04`. |
 | <a id="containerrepositorydetailsstatus"></a>`status` | [`ContainerRepositoryStatus`](#containerrepositorystatus) | Status of the container repository. |
 | <a id="containerrepositorydetailstagscount"></a>`tagsCount` | [`Int!`](#int) | Number of tags associated with this image. |
 | <a id="containerrepositorydetailsupdatedat"></a>`updatedAt` | [`Time!`](#time) | Timestamp when the container repository was updated. |

lib/api/container_repositories.rb

@@ -23,11 +23,17 @@ module API
         params do
           optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included'
           optional :tags_count, type: Boolean, default: false, desc: 'Determines if the tags count should be included'
+          optional :size, type: Boolean, default: false, desc: 'Determines if the size should be included'
         end
         get ':id' do
           authorize!(:read_container_image, repository)
 
-          present repository, with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count], user: current_user
+          present repository,
+                  with: Entities::ContainerRegistry::Repository,
+                  tags: params[:tags],
+                  tags_count: params[:tags_count],
+                  size: params[:size],
+                  user: current_user
         end
       end
     end

lib/api/entities/container_registry.rb

@@ -22,6 +22,7 @@ module API
         expose :tags_count, if: -> (_, options) { options[:tags_count] }
         expose :tags, using: Tag, if: -> (_, options) { options[:tags] }
         expose :delete_api_path, if: ->(object, options) { Ability.allowed?(options[:user], :admin_container_image, object) }
+        expose :size, if: -> (_, options) { options[:size] }
 
         private
 

lib/container_registry/gitlab_api_client.rb

@@ -31,8 +31,10 @@ module ContainerRegistry
         registry_features = Gitlab::CurrentSettings.container_registry_features || []
         next true if ::Gitlab.com? && registry_features.include?(REGISTRY_GITLAB_V1_API_FEATURE)
 
-        response = faraday.get('/gitlab/v1/')
-        response.success? || response.status == 401
+        with_token_faraday do |faraday_client|
+          response = faraday_client.get('/gitlab/v1/')
+          response.success? || response.status == 401
+        end
       end
     end
 
@@ -50,15 +52,46 @@ module ContainerRegistry
 
     # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#get-repository-import-status
     def import_status(path)
-      body_hash = response_body(faraday.get(import_url_for(path)))
-      body_hash['status'] || 'error'
+      with_import_token_faraday do |faraday_client|
+        body_hash = response_body(faraday_client.get(import_url_for(path)))
+        body_hash['status'] || 'error'
+      end
+    end
+
+    def repository_details(path, with_size: false)
+      with_token_faraday do |faraday_client|
+        req = faraday_client.get("/gitlab/v1/repositories/#{path}/") do |req|
+          req.params['size'] = 'self' if with_size
+        end
+
+        break {} unless req.success?
+
+        response_body(req)
+      end
     end
 
     private
 
     def start_import_for(path, pre:)
-      faraday.put(import_url_for(path)) do |req|
-        req.params['pre'] = pre.to_s
+      with_import_token_faraday do |faraday_client|
+        faraday_client.put(import_url_for(path)) do |req|
+          req.params['pre'] = pre.to_s
+        end
+      end
+    end
+
+    def with_token_faraday
+      yield faraday
+    end
+
+    def with_import_token_faraday
+      yield faraday_with_import_token
+    end
+
+    def faraday_with_import_token(timeout_enabled: true)
+      @faraday_with_import_token ||= faraday_base(timeout_enabled: timeout_enabled) do |conn|
+        # initialize the connection with the :import_token instead of :token
+        initialize_connection(conn, @options.merge(token: @options[:import_token]), &method(:configure_connection))
       end
     end
 

lib/container_registry/registry.rb

@@ -2,26 +2,16 @@
 
 module ContainerRegistry
   class Registry
-    include Gitlab::Utils::StrongMemoize
-
-    attr_reader :uri, :client, :path
+    attr_reader :uri, :client, :gitlab_api_client, :path
 
     def initialize(uri, options = {})
       @uri = uri
       @options = options
       @path = @options[:path] || default_path
       @client = ContainerRegistry::Client.new(@uri, @options)
-    end
-
-    def gitlab_api_client
-      strong_memoize(:gitlab_api_client) do
-        token = Auth::ContainerRegistryAuthenticationService.import_access_token
-
-        url = Gitlab.config.registry.api_url
-        host_port = Gitlab.config.registry.host_port
 
-        ContainerRegistry::GitlabApiClient.new(url, token: token, path: host_port)
-      end
+      import_token = Auth::ContainerRegistryAuthenticationService.import_access_token
+      @gitlab_api_client = ContainerRegistry::GitlabApiClient.new(@uri, @options.merge(import_token: import_token))
     end
 
     private

spec/graphql/types/container_repository_details_type_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 RSpec.describe GitlabSchema.types['ContainerRepositoryDetails'] do
-  fields = %i[id name path location created_at updated_at expiration_policy_started_at status tags_count can_delete expiration_policy_cleanup_status tags project]
+  fields = %i[id name path location created_at updated_at expiration_policy_started_at status tags_count can_delete expiration_policy_cleanup_status tags size project]
 
   it { expect(described_class.graphql_name).to eq('ContainerRepositoryDetails') }
 

spec/lib/container_registry/gitlab_api_client_spec.rb

@@ -6,8 +6,11 @@ RSpec.describe ContainerRegistry::GitlabApiClient do
   using RSpec::Parameterized::TableSyntax
 
   include_context 'container registry client'
+  include_context 'container registry client stubs'
 
   let(:path) { 'namespace/path/to/repository' }
+  let(:import_token) { 'import_token' }
+  let(:options) { { token: token, import_token: import_token } }
 
   describe '#supports_gitlab_api?' do
     subject { client.supports_gitlab_api? }
@@ -121,6 +124,40 @@ RSpec.describe ContainerRegistry::GitlabApiClient do
     end
   end
 
+  describe '#repository_details' do
+    let(:path) { 'namespace/path/to/repository' }
+    let(:response) { { foo: :bar, this: :is_a_test } }
+    let(:with_size) { true }
+
+    subject { client.repository_details(path, with_size: with_size) }
+
+    context 'with size' do
+      before do
+        stub_repository_details(path, with_size: with_size, respond_with: response)
+      end
+
+      it { is_expected.to eq(response.stringify_keys.deep_transform_values(&:to_s)) }
+    end
+
+    context 'without_size' do
+      let(:with_size) { false }
+
+      before do
+        stub_repository_details(path, with_size: with_size, respond_with: response)
+      end
+
+      it { is_expected.to eq(response.stringify_keys.deep_transform_values(&:to_s)) }
+    end
+
+    context 'with non successful response' do
+      before do
+        stub_repository_details(path, with_size: with_size, status_code: 404)
+      end
+
+      it { is_expected.to eq({}) }
+    end
+  end
+
   describe '.supports_gitlab_api?' do
     subject { described_class.supports_gitlab_api? }
 
@@ -181,7 +218,7 @@ RSpec.describe ContainerRegistry::GitlabApiClient do
 
   def stub_pre_import(path, status_code, pre:)
     stub_request(:put, "#{registry_api_url}/gitlab/v1/import/#{path}/?pre=#{pre}")
-      .with(headers: { 'Accept' => described_class::JSON_TYPE })
+      .with(headers: { 'Accept' => described_class::JSON_TYPE, 'Authorization' => "bearer #{import_token}" })
       .to_return(status: status_code, body: '')
   end
 
@@ -194,11 +231,19 @@ RSpec.describe ContainerRegistry::GitlabApiClient do
 
   def stub_import_status(path, status)
     stub_request(:get, "#{registry_api_url}/gitlab/v1/import/#{path}/")
-      .with(headers: { 'Accept' => described_class::JSON_TYPE })
+      .with(headers: { 'Accept' => described_class::JSON_TYPE, 'Authorization' => "bearer #{import_token}" })
       .to_return(
         status: 200,
         body: { status: status }.to_json,
         headers: { content_type: 'application/json' }
       )
   end
+
+  def stub_repository_details(path, with_size: true, status_code: 200, respond_with: {})
+    url = "#{registry_api_url}/gitlab/v1/repositories/#{path}/"
+    url += "?size=self" if with_size
+    stub_request(:get, url)
+      .with(headers: { 'Accept' => described_class::JSON_TYPE, 'Authorization' => "bearer #{token}" })
+      .to_return(status: status_code, body: respond_with.to_json, headers: { 'Content-Type' => described_class::JSON_TYPE })
+  end
 end

spec/lib/container_registry/registry_spec.rb

@@ -4,10 +4,15 @@ require 'spec_helper'
 
 RSpec.describe ContainerRegistry::Registry do
   let(:path) { nil }
-  let(:registry) { described_class.new('http://example.com', path: path) }
+  let(:registry_api_url) { 'http://example.com' }
+  let(:registry) { described_class.new(registry_api_url, path: path) }
 
   subject { registry }
 
+  before do
+    stub_container_registry_config(enabled: true, api_url: registry_api_url, key: 'spec/fixtures/x509_certificate_pk.key')
+  end
+
   it { is_expected.to respond_to(:client) }
   it { is_expected.to respond_to(:uri) }
   it { is_expected.to respond_to(:path) }

spec/models/container_repository_spec.rb

@@ -653,6 +653,58 @@ RSpec.describe ContainerRepository, :aggregate_failures do
     end
   end
 
+  describe '#size' do
+    let(:on_com) { true }
+    let(:created_at) { described_class::MIGRATION_PHASE_1_STARTED_AT + 3.months }
+
+    subject { repository.size }
+
+    before do
+      allow(::Gitlab).to receive(:com?).and_return(on_com)
+      allow(repository).to receive(:created_at).and_return(created_at)
+    end
+
+    context 'supports gitlab api on .com with a recent repository' do
+      before do
+        expect(repository.gitlab_api_client).to receive(:supports_gitlab_api?).and_return(true)
+        expect(repository.gitlab_api_client).to receive(:repository_details).with(repository.path, with_size: true).and_return(response)
+      end
+
+      context 'with a size_bytes field' do
+        let(:response) { { 'size_bytes' => 12345 } }
+
+        it { is_expected.to eq(12345) }
+      end
+
+      context 'without a size_bytes field' do
+        let(:response) { { 'foo' => 'bar' } }
+
+        it { is_expected.to eq(nil) }
+      end
+    end
+
+    context 'does not support gitlab api' do
+      before do
+        expect(repository.gitlab_api_client).to receive(:supports_gitlab_api?).and_return(false)
+        expect(repository.gitlab_api_client).not_to receive(:repository_details)
+      end
+
+      it { is_expected.to eq(nil) }
+    end
+
+    context 'not on .com' do
+      let(:on_com) { false }
+
+      it { is_expected.to eq(nil) }
+    end
+
+    context 'with an old repository' do
+      let(:created_at) { described_class::MIGRATION_PHASE_1_STARTED_AT - 3.months }
+
+      it { is_expected.to eq(nil) }
+    end
+  end
+
   describe '#reset_expiration_policy_started_at!' do
     subject { repository.reset_expiration_policy_started_at! }
 

spec/requests/api/container_repositories_spec.rb

@@ -3,6 +3,8 @@
 require 'spec_helper'
 
 RSpec.describe API::ContainerRepositories do
+  include_context 'container registry client stubs'
+
   let_it_be(:project) { create(:project, :private) }
   let_it_be(:reporter) { create(:user) }
   let_it_be(:guest) { create(:user) }
@@ -103,6 +105,68 @@ RSpec.describe API::ContainerRepositories do
           expect(json_response['tags_count']).to eq(2)
         end
       end
+
+      context 'with size param' do
+        let(:url) { "/registry/repositories/#{repository.id}?size=true" }
+        let(:on_com) { true }
+        let(:created_at) { ::ContainerRepository::MIGRATION_PHASE_1_STARTED_AT + 3.months }
+
+        before do
+          allow(::Gitlab).to receive(:com?).and_return(on_com)
+          repository.update_column(:created_at, created_at)
+        end
+
+        it 'returns a repository and its size' do
+          stub_container_registry_gitlab_api_support(supported: true) do |client|
+            stub_container_registry_gitlab_api_repository_details(client, path: repository.path, size_bytes: 12345)
+          end
+
+          subject
+
+          expect(json_response['size']).to eq(12345)
+        end
+
+        context 'with a network error' do
+          it 'returns an error message' do
+            stub_container_registry_gitlab_api_network_error
+
+            subject
+
+            expect(response).to have_gitlab_http_status(:service_unavailable)
+            expect(json_response['message']).to include('We are having trouble connecting to the Container Registry')
+          end
+        end
+
+        context 'with not supporting the gitlab api' do
+          it 'returns nil' do
+            stub_container_registry_gitlab_api_support(supported: false)
+
+            subject
+
+            expect(json_response['size']).to eq(nil)
+          end
+        end
+
+        context 'not on .com' do
+          let(:on_com) { false }
+
+          it 'returns nil' do
+            subject
+
+            expect(json_response['size']).to eq(nil)
+          end
+        end
+
+        context 'with an older container repository' do
+          let(:created_at) { ::ContainerRepository::MIGRATION_PHASE_1_STARTED_AT - 3.months }
+
+          it 'returns nil' do
+            subject
+
+            expect(json_response['size']).to eq(nil)
+          end
+        end
+      end
     end
 
     context 'with invalid repository id' do

spec/requests/api/graphql/container_repository/container_repository_details_spec.rb

@@ -3,17 +3,19 @@ require 'spec_helper'
 
 RSpec.describe 'container repository details' do
   include_context 'container registry tags'
+  include_context 'container registry client stubs'
+
   using RSpec::Parameterized::TableSyntax
   include GraphqlHelpers
 
   let_it_be_with_reload(:project) { create(:project) }
-  let_it_be(:container_repository) { create(:container_repository, project: project) }
+  let_it_be_with_reload(:container_repository) { create(:container_repository, project: project) }
 
   let(:query) do
     graphql_query_for(
       'containerRepository',
       { id: container_repository_global_id },
-      all_graphql_fields_for('ContainerRepositoryDetails', excluded: ['pipeline'])
+      all_graphql_fields_for('ContainerRepositoryDetails', excluded: %w[pipeline size])
     )
   end
 
@@ -220,6 +222,80 @@ RSpec.describe 'container repository details' do
     end
   end
 
+  context 'size field' do
+    let(:size_response) { container_repository_details_response.dig('size') }
+    let(:on_com) { true }
+    let(:created_at) { ::ContainerRepository::MIGRATION_PHASE_1_STARTED_AT + 3.months }
+    let(:variables) do
+      { id: container_repository_global_id }
+    end
+
+    let(:query) do
+      <<~GQL
+        query($id: ID!) {
+          containerRepository(id: $id) {
+            size
+          }
+        }
+      GQL
+    end
+
+    before do
+      allow(::Gitlab).to receive(:com?).and_return(on_com)
+      container_repository.update_column(:created_at, created_at)
+    end
+
+    it 'returns the size' do
+      stub_container_registry_gitlab_api_support(supported: true) do |client|
+        stub_container_registry_gitlab_api_repository_details(client, path: container_repository.path, size_bytes: 12345)
+      end
+
+      subject
+
+      expect(size_response).to eq(12345)
+    end
+
+    context 'with a network error' do
+      it 'returns an error' do
+        stub_container_registry_gitlab_api_network_error
+
+        subject
+
+        expect_graphql_errors_to_include("Can't connect to the Container Registry. If this error persists, please review the troubleshooting documentation.")
+      end
+    end
+
+    context 'with not supporting the gitlab api' do
+      it 'returns nil' do
+        stub_container_registry_gitlab_api_support(supported: false)
+
+        subject
+
+        expect(size_response).to eq(nil)
+      end
+    end
+
+    context 'not on .com' do
+      let(:on_com) { false }
+
+      it 'returns nil' do
+        subject
+
+        expect(size_response).to eq(nil)
+      end
+    end
+
+    context 'with an older container repository' do
+      let(:created_at) { ::ContainerRepository::MIGRATION_PHASE_1_STARTED_AT - 3.months }
+
+      it 'returns nil' do
+        subject
+
+        expect(size_response).to eq(nil)
+      end
+    end
+  end
+
   context 'with tags with a manifest containing nil fields' do
     let(:tags_response) { container_repository_details_response.dig('tags', 'nodes') }
     let(:errors) { container_repository_details_response.dig('errors') }

spec/support/shared_contexts/lib/container_registry/client_stubs_shared_context.rb

+# frozen_string_literal: true
+
+RSpec.shared_context 'container registry client stubs' do
+  def stub_container_registry_gitlab_api_support(supported: true)
+    allow_next_instance_of(ContainerRegistry::GitlabApiClient) do |client|
+      allow(client).to receive(:supports_gitlab_api?).and_return(supported)
+      yield client if block_given?
+    end
+  end
+
+  def stub_container_registry_gitlab_api_repository_details(client, path:, size_bytes:)
+    allow(client).to receive(:repository_details).with(path, with_size: true).and_return('size_bytes' => size_bytes)
+  end
+
+  def stub_container_registry_gitlab_api_network_error(client_method: :supports_gitlab_api?)
+    allow_next_instance_of(ContainerRegistry::GitlabApiClient) do |client|
+      allow(client).to receive(client_method).and_raise(::Faraday::Error, nil, nil)
+    end
+  end
+end