Add dependency proxy for container images

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

ee/app/controllers/groups/dependency_proxy_for_containers_controller.rb

+# frozen_string_literal: true
+
+class Groups::DependencyProxyForContainersController < Groups::ApplicationController
+  include SendFileUpload
+
+  before_action :ensure_feature_enabled!
+
+  def manifest
+    output = DependencyProxy::PullManifestService.new(image, tag, token).execute
+
+    render json: output
+  end
+
+  def blob
+    blob = DependencyProxy::FindOrCreateBlobService
+      .new(group, image, token, params[:sha]).execute
+
+    send_upload(blob.file)
+  end
+
+  private
+
+  def image
+    params[:image]
+  end
+
+  def tag
+    params[:tag]
+  end
+
+  def token
+    @token ||= request_token
+  end
+
+  def request_token
+    DependencyProxy::RequestTokenService.new(image).execute
+  end
+
+  def ensure_feature_enabled!
+    render_404 unless Gitlab.config.dependency_proxy.enabled &&
+        group.feature_available?(:dependency_proxy) &&
+        group.dependency_proxy_setting&.enabled
+  end
+end

ee/app/services/dependency_proxy/base_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class BaseService
+    private
+
+    def registry
+      DependencyProxy::Registry.new
+    end
+
+    def auth_headers
+      {
+        Authorization: "Bearer #{@token}"
+      }
+    end
+  end
+end

ee/app/services/dependency_proxy/download_blob_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class DownloadBlobService < DependencyProxy::BaseService
+    DownloadError = Class.new(StandardError)
+
+    def initialize(image, blob_sha, token, file_path)
+      @image = image
+      @blob_sha = blob_sha
+      @token = token
+      @file_path = file_path
+    end
+
+    def execute
+      File.open(@file_path, "wb") do |file|
+        Gitlab::HTTP.get(blob_url, headers: auth_headers, stream_body: true) do |fragment|
+          if [301, 302, 307].include?(fragment.code)
+            # do nothing
+          elsif fragment.code == 200
+            file.write(fragment)
+          else
+            raise DownloadError, "Non-success status code while downloading a blob. #{fragment.code}"
+          end
+        end
+      end
+
+      true
+    rescue DownloadError
+      false
+    end
+
+    private
+
+    def blob_url
+      registry.blob_url(@image, @blob_sha)
+    end
+  end
+end

ee/app/services/dependency_proxy/find_or_create_blob_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class FindOrCreateBlobService < DependencyProxy::BaseService
+    def initialize(group, image, token, blob_sha)
+      @group = group
+      @image = image
+      @token = token
+      @blob_sha = blob_sha
+    end
+
+    def execute
+      file_name = @blob_sha.sub('sha256:', '') + '.gz'
+      blob = @group.dependency_proxy_blobs.find_or_build(file_name)
+
+      unless blob.persisted?
+        temp_file = Tempfile.new
+
+        success = DependencyProxy::DownloadBlobService
+          .new(@image, @blob_sha, @token, temp_file.path).execute
+
+        return unless success
+
+        blob.file = temp_file
+        blob.size = temp_file.size
+        blob.save!
+      end
+
+      blob
+    end
+  end
+end

ee/app/services/dependency_proxy/pull_manifest_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class PullManifestService < DependencyProxy::BaseService
+    def initialize(image, tag, token)
+      @image = image
+      @tag = tag
+      @token = token
+    end
+
+    def execute
+      response = Gitlab::HTTP.get(manifest_url, headers: auth_headers)
+      response.body
+    end
+
+    private
+
+    def manifest_url
+      registry.manifest_url(@image, @tag)
+    end
+  end
+end

ee/app/services/dependency_proxy/request_token_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class RequestTokenService < DependencyProxy::BaseService
+    def initialize(image)
+      @image = image
+    end
+
+    def execute
+      response = Gitlab::HTTP.get(auth_url)
+
+      JSON.parse(response.body)['token']
+    end
+
+    private
+
+    def auth_url
+      registry.auth_url(@image)
+    end
+  end
+end

ee/changelogs/unreleased/dz-registry-proxy.yml

+---
+title: Add dependency proxy for containers
+merge_request: 9750
+author:
+type: added

ee/config/routes/group.rb

@@ -114,3 +114,11 @@ constraints(::Constraints::GroupUrlConstrainer.new) do
                                           :pipeline_quota, :hooks, :boards)
   end
 end
+
+# Dependency proxy for containers
+# Because docker adds v2 prefix to URI this need to be outside of usual group routes
+scope constraints: { format: nil } do
+  get 'v2', to: proc { [200, {}, ['']] }
+  get 'v2/*group_id/dependency_proxy/containers/:image/manifests/*tag' => 'groups/dependency_proxy_for_containers#manifest'
+  get 'v2/*group_id/dependency_proxy/containers/:image/blobs/:sha' => 'groups/dependency_proxy_for_containers#blob'
+end

ee/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Groups::DependencyProxyForContainersController do
+  let(:group) { create(:group) }
+
+  before do
+    allow(Gitlab.config.dependency_proxy)
+      .to receive(:enabled).and_return(true)
+
+    allow_any_instance_of(DependencyProxy::RequestTokenService)
+      .to receive(:execute).and_return('abcd1234')
+  end
+
+  describe 'GET #manifest' do
+    let(:manifest) { { foo: 'bar' }.to_json }
+
+    before do
+      allow_any_instance_of(DependencyProxy::PullManifestService)
+        .to receive(:execute).and_return(manifest)
+    end
+
+    it 'returns 200 with manifest file' do
+      enable_dependency_proxy
+
+      get_manifest
+
+      expect(response).to have_gitlab_http_status(200)
+      expect(response.body).to eq(manifest)
+    end
+
+    it 'returns 404 when feature is disabled' do
+      get_manifest
+
+      expect(response).to have_gitlab_http_status(404)
+    end
+
+    def get_manifest
+      get :manifest, params: { group_id: group.to_param, image: 'alpine', tag: '3.9.2' }
+    end
+  end
+
+  describe 'GET #blob' do
+    let(:blob) { create(:dependency_proxy_blob) }
+    let(:blob_sha) { blob.file_name.sub('.gz', '') }
+
+    before do
+      allow_any_instance_of(DependencyProxy::FindOrCreateBlobService)
+        .to receive(:execute).and_return(blob)
+    end
+
+    context 'feature enabled' do
+      before do
+        enable_dependency_proxy
+      end
+
+      it 'sends a file' do
+        expect(controller).to receive(:send_file).with(blob.file.path, {})
+
+        get_blob
+      end
+
+      it 'returns Content-Disposition: attachment' do
+        get_blob
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(response.headers['Content-Disposition']).to match(/^attachment/)
+      end
+    end
+
+    it 'returns 404 when feature is disabled' do
+      get_blob
+
+      expect(response).to have_gitlab_http_status(404)
+    end
+
+    def get_blob
+      get :blob, params: { group_id: group.to_param, image: 'alpine', sha: blob_sha }
+    end
+  end
+
+  def enable_dependency_proxy
+    stub_licensed_features(dependency_proxy: true)
+    group.create_dependency_proxy_setting!(enabled: true)
+  end
+end

ee/spec/routing/group_routing_spec.rb

@@ -87,4 +87,20 @@ describe 'Group routing', "routing" do
       expect(get('/groups/gitlabhq/-/security/vulnerabilities/history')).to route_to('groups/security/vulnerabilities#history', group_id: 'gitlabhq')
     end
   end
+
+  describe 'dependency proxy for containers' do
+    before do
+      allow(Group).to receive(:find_by_full_path).with('gitlabhq', any_args).and_return(true)
+    end
+
+    it 'routes to #manifest' do
+      expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/manifests/2.3.6'))
+        .to route_to('groups/dependency_proxy_for_containers#manifest', group_id: 'gitlabhq', image: 'ruby', tag: '2.3.6')
+    end
+
+    it 'routes to #blob' do
+      expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/blobs/abc12345'))
+        .to route_to('groups/dependency_proxy_for_containers#blob', group_id: 'gitlabhq', image: 'ruby', sha: 'abc12345')
+    end
+  end
 end

ee/spec/services/dependency_proxy/download_blob_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe DependencyProxy::DownloadBlobService do
+  include EE::DependencyProxyHelpers
+
+  let(:image) { 'alpine' }
+  let(:token) { Digest::SHA256.hexdigest('123') }
+  let(:blob_sha) { Digest::SHA256.hexdigest('ruby:2.3.9') }
+  let(:file) { Tempfile.new }
+
+  subject { described_class.new(image, blob_sha, token, file.path) }
+
+  before do
+    stub_blob_download(image, blob_sha)
+  end
+
+  it 'downloads blob and writes it into the file' do
+    expect { subject.execute }.to change { file.size }.from(0).to(6)
+  end
+end

ee/spec/services/dependency_proxy/find_or_create_blob_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe DependencyProxy::FindOrCreateBlobService do
+  include EE::DependencyProxyHelpers
+
+  let(:blob)  { create(:dependency_proxy_blob) }
+  let(:group) { blob.group }
+  let(:image) { 'alpine' }
+  let(:tag)   { '3.9' }
+  let(:token) { Digest::SHA256.hexdigest('123') }
+  let(:blob_sha) { '40bd001563085fc35165329ea1ff5c5ecbdbbeef' }
+
+  subject { described_class.new(group, image, token, blob_sha).execute }
+
+  before do
+    stub_registry_auth(image, token)
+  end
+
+  context 'no cache' do
+    before do
+      stub_blob_download(image, blob_sha)
+    end
+
+    it 'downloads blob from remote registry if there is no cached one' do
+      is_expected.to be_a(DependencyProxy::Blob)
+      is_expected.to be_persisted
+    end
+  end
+
+  context 'cached blob' do
+    let(:blob_sha) { blob.file_name.sub('.gz', '') }
+
+    it 'uses cached blob instead of downloading one' do
+      is_expected.to be_a(DependencyProxy::Blob)
+      is_expected.to eq(blob)
+    end
+  end
+
+  context 'no such blob exists remotely' do
+    before do
+      stub_blob_download_not_found(image, blob_sha)
+    end
+
+    it { is_expected.to be nil }
+  end
+end

ee/spec/services/dependency_proxy/pull_manifest_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe DependencyProxy::PullManifestService do
+  include EE::DependencyProxyHelpers
+
+  let(:image) { 'alpine' }
+  let(:tag) { '3.9' }
+  let(:token) { Digest::SHA256.hexdigest('123') }
+  let(:manifest) { { foo: 'bar' }.to_json }
+
+  subject { described_class.new(image, tag, token) }
+
+  before do
+    stub_manifest_download(image, tag)
+  end
+
+  it 'downloads blob and writes it into the file' do
+    expect(subject.execute).to eq(manifest)
+  end
+end

ee/spec/services/dependency_proxy/request_token_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe DependencyProxy::RequestTokenService do
+  let(:image) { 'alpine:3.9' }
+  let(:token) { Digest::SHA256.hexdigest('123') }
+  let(:registry) { DependencyProxy::Registry.new }
+
+  subject { described_class.new(image).execute }
+
+  before do
+    auth_body = { 'token' => token }.to_json
+    auth_link = registry.auth_url(image)
+
+    stub_request(:get, auth_link)
+      .to_return(status: 200, body: auth_body)
+  end
+
+  it 'requests an access token from auth service' do
+    is_expected.to eq(token)
+  end
+end

ee/spec/support/helpers/ee/dependency_proxy_helpers.rb

+module EE
+  module DependencyProxyHelpers
+    def stub_registry_auth(image, token)
+      auth_body = { 'token' => token }.to_json
+      auth_link = registry.auth_url(image)
+
+      stub_request(:get, auth_link)
+        .to_return(status: 200, body: auth_body)
+    end
+
+    def stub_manifest_download(image, tag)
+      manifest_url = registry.manifest_url(image, tag)
+
+      stub_request(:get, manifest_url)
+        .to_return(status: 200, body: manifest)
+    end
+
+    def stub_blob_download(image, blob_sha)
+      download_link = registry.blob_url(image, blob_sha)
+
+      stub_request(:get, download_link)
+        .to_return(status: 200, body: '123456')
+    end
+
+    def stub_blob_download_not_found(image, blob_sha)
+      download_link = registry.blob_url(image, blob_sha)
+
+      stub_request(:get, download_link)
+        .to_return(status: 404)
+    end
+
+    private
+
+    def registry
+      @registry ||= DependencyProxy::Registry.new
+    end
+  end
+end