Merge branch 'dz-registry-proxy-models' into 'master'

Add depedency proxy models

See merge request gitlab-org/gitlab-ee!10681

ee/app/models/dependency_proxy.rb

+# frozen_string_literal: true
+module DependencyProxy
+  def self.table_name_prefix
+    'dependency_proxy_'
+  end
+end

ee/app/models/dependency_proxy/blob.rb

+# frozen_string_literal: true
+
+class DependencyProxy::Blob < ApplicationRecord
+  belongs_to :group
+
+  validates :group, presence: true
+  validates :file, presence: true
+  validates :file_name, presence: true
+
+  mount_uploader :file, DependencyProxy::FileUploader
+
+  def self.total_size
+    sum(:size)
+  end
+
+  def self.find_or_build(file_name)
+    find_or_initialize_by(file_name: file_name)
+  end
+end

ee/app/models/dependency_proxy/group_setting.rb

+# frozen_string_literal: true
+
+class DependencyProxy::GroupSetting < ApplicationRecord
+  belongs_to :group
+
+  validates :group, presence: true
+end

ee/app/models/dependency_proxy/registry.rb

+# frozen_string_literal: true
+
+class DependencyProxy::Registry
+  AUTH_URL = 'https://auth.docker.io'.freeze
+  LIBRARY_URL = 'https://registry-1.docker.io/v2/library'.freeze
+
+  class << self
+    def auth_url(image)
+      "#{AUTH_URL}/token?service=registry.docker.io&scope=repository:library/#{image}:pull"
+    end
+
+    def manifest_url(image, tag)
+      "#{LIBRARY_URL}/#{image}/manifests/#{tag}"
+    end
+
+    def blob_url(image, blob_sha)
+      "#{LIBRARY_URL}/#{image}/blobs/#{blob_sha}"
+    end
+  end
+end

ee/app/models/ee/group.rb

@@ -25,6 +25,9 @@ module EE
       has_many :ldap_group_links, foreign_key: 'group_id', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
       has_many :hooks, dependent: :destroy, class_name: 'GroupHook' # rubocop:disable Cop/ActiveRecordDependent
 
+      has_one :dependency_proxy_setting, class_name: 'DependencyProxy::GroupSetting'
+      has_many :dependency_proxy_blobs, class_name: 'DependencyProxy::Blob'
+
       # We cannot simply set `has_many :audit_events, as: :entity, dependent: :destroy`
       # here since Group inherits from Namespace, the entity_type would be set to `Namespace`.
       has_many :audit_events, -> { where(entity_type: ::Group.name) }, foreign_key: 'entity_id'

ee/app/models/license.rb

@@ -77,6 +77,7 @@ class License < ApplicationRecord
     merge_pipelines
     design_management
     operations_dashboard
+    dependency_proxy
   ]
   EEP_FEATURES.freeze
 

ee/app/uploaders/dependency_proxy/file_uploader.rb

+# frozen_string_literal: true
+
+class DependencyProxy::FileUploader < GitlabUploader
+  include ObjectStorage::Concern
+
+  storage_options Gitlab.config.dependency_proxy
+
+  alias_method :upload, :model
+
+  def filename
+    model.file_name
+  end
+
+  def store_dir
+    dynamic_segment
+  end
+
+  private
+
+  def dynamic_segment
+    File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
+              'dependency_proxy', model.group_id.to_s, 'files', model.id.to_s)
+  end
+
+  def disk_hash
+    @disk_hash ||= Digest::SHA2.hexdigest(model.group_id.to_s)
+  end
+end

ee/spec/factories/dependency_proxy.rb

+# frozen_string_literal: true
+FactoryBot.define do
+  factory :dependency_proxy_blob, class: DependencyProxy::Blob do
+    group
+    file { fixture_file_upload('ee/spec/fixtures/dependency_proxy/a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4.gz') }
+    file_name 'a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4.gz'
+  end
+end

ee/spec/models/dependency_proxy/blob_spec.rb

+# frozen_string_literal: true
+require 'rails_helper'
+
+RSpec.describe DependencyProxy::Blob, type: :model do
+  describe 'relationships' do
+    it { is_expected.to belong_to(:group) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:group) }
+    it { is_expected.to validate_presence_of(:file) }
+    it { is_expected.to validate_presence_of(:file_name) }
+  end
+
+  describe '.total_size' do
+    it 'returns 0 if no files' do
+      expect(described_class.total_size).to eq(0)
+    end
+
+    it 'returns a correct sum of all files sizes' do
+      create(:dependency_proxy_blob, size: 10)
+      create(:dependency_proxy_blob, size: 20)
+
+      expect(described_class.total_size).to eq(30)
+    end
+  end
+
+  describe '.find_or_build' do
+    let!(:blob) { create(:dependency_proxy_blob) }
+
+    it 'builds new instance if not found' do
+      expect(described_class.find_or_build('foo.gz')).not_to be_persisted
+    end
+
+    it 'finds an existing blob' do
+      expect(described_class.find_or_build(blob.file_name)).to eq(blob)
+    end
+  end
+end

ee/spec/models/dependency_proxy/group_setting_spec.rb

+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe DependencyProxy::GroupSetting, type: :model do
+  describe 'relationships' do
+    it { is_expected.to belong_to(:group) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:group) }
+  end
+end

ee/spec/models/dependency_proxy/registry_spec.rb

+# frozen_string_literal: true
+require 'rails_helper'
+
+RSpec.describe DependencyProxy::Registry, type: :model do
+  let(:image)    { 'ruby' }
+  let(:tag)      { '2.3.5-alpine' }
+  let(:blob_sha) { '40bd001563085fc35165329ea1ff5c5ecbdbbeef' }
+
+  describe '#auth_url' do
+    it 'returns a correct auth url' do
+      expect(described_class.auth_url(image))
+        .to eq('https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/ruby:pull')
+    end
+  end
+
+  describe '#manifest_url' do
+    it 'returns a correct manifest url' do
+      expect(described_class.manifest_url(image, tag))
+        .to eq('https://registry-1.docker.io/v2/library/ruby/manifests/2.3.5-alpine')
+    end
+  end
+
+  describe '#blob_url' do
+    it 'returns a correct blob url' do
+      expect(described_class.blob_url(image, blob_sha))
+        .to eq('https://registry-1.docker.io/v2/library/ruby/blobs/40bd001563085fc35165329ea1ff5c5ecbdbbeef')
+    end
+  end
+end

ee/spec/models/group_spec.rb

@@ -8,6 +8,8 @@ describe Group do
   describe 'associations' do
     it { is_expected.to have_many(:audit_events).dependent(false) }
     it { is_expected.to belong_to(:file_template_project) }
+    it { is_expected.to have_many(:dependency_proxy_blobs) }
+    it { is_expected.to have_one(:dependency_proxy_setting) }
   end
 
   describe 'scopes' do

ee/spec/support/helpers/ee/stub_object_storage.rb

@@ -8,6 +8,13 @@ module EE
                                    **params)
     end
 
+    def stub_dependency_proxy_object_storage(**params)
+      stub_object_storage_uploader(config: ::Gitlab.config.dependency_proxy.object_store,
+                                   uploader: ::DependencyProxy::FileUploader,
+                                   remote_directory: 'dependency_proxy',
+                                   **params)
+    end
+
     def stub_object_storage_pseudonymizer
       stub_object_storage(connection_params: Pseudonymizer::Uploader.object_store_credentials,
                           remote_directory: Pseudonymizer::Uploader.remote_directory)

ee/spec/uploaders/dependency_proxy/file_uploader_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe DependencyProxy::FileUploader do
+  let(:blob) { create(:dependency_proxy_blob) }
+  let(:uploader) { described_class.new(blob, :file) }
+  let(:path) { Gitlab.config.dependency_proxy.storage_path }
+
+  subject { uploader }
+
+  it_behaves_like "builds correct paths",
+                  store_dir: %r[\h{2}/\h{2}],
+                  cache_dir: %r[/dependency_proxy/tmp/cache],
+                  work_dir: %r[/dependency_proxy/tmp/work]
+
+  context 'object store is remote' do
+    before do
+      stub_dependency_proxy_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like "builds correct paths",
+                    store_dir: %r[\h{2}/\h{2}]
+  end
+end