Merge branch '13345-conan-upload-file-endpoints' into 'master'

Conan package registry file upload endpoints

See merge request gitlab-org/gitlab!17791

ee/app/services/packages/conan/create_package_file_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module Conan
+    class CreatePackageFileService
+      attr_reader :package, :file, :params
+
+      def initialize(package, file, params)
+        @package = package
+        @file = file
+        @params = params
+      end
+
+      def execute
+        package.package_files.create!(
+          file:      file,
+          size:      params['file.size'],
+          file_name: params[:file_name],
+          file_type: params['file.type'],
+          file_sha1: params['file.sha1'],
+          file_md5:  params['file.md5'],
+          conan_file_metadatum_attributes: {
+            recipe_revision: params[:recipe_revision],
+            package_revision: params[:package_revision],
+            conan_package_reference: params[:conan_package_reference],
+            conan_file_type: params[:conan_file_type]
+          }
+        )
+      end
+    end
+  end
+end

ee/app/services/packages/conan/create_package_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module Conan
+    class CreatePackageService < BaseService
+      def execute
+        project.packages.create!(
+          name: params[:package_name],
+          version: params[:package_version],
+          package_type: :conan,
+          conan_metadatum_attributes: {
+            package_username: params[:package_username],
+            package_channel: params[:package_channel]
+          }
+        )
+      end
+    end
+  end
+end

ee/app/uploaders/packages/package_file_uploader.rb

@@ -5,6 +5,8 @@ class Packages::PackageFileUploader < GitlabUploader
 
   storage_options Gitlab.config.packages
 
+  after :store, :schedule_background_upload
+
   alias_method :upload, :model
 
   def filename

ee/lib/api/conan_packages.rb

@@ -24,6 +24,7 @@ module API
     }.freeze
 
     PACKAGE_COMPONENT_REGEX = Gitlab::Regex.conan_recipe_component_regex
+    CONAN_REVISION_REGEX = Gitlab::Regex.conan_revision_regex
 
     before do
       not_found! unless Feature.enabled?(:conan_package_registry)
@@ -195,7 +196,7 @@ module API
         requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version'
         requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username'
         requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel'
-        requires :recipe_revision, type: String, desc: 'Conan Recipe Revision'
+        requires :recipe_revision, type: String, regexp: CONAN_REVISION_REGEX, desc: 'Conan Recipe Revision'
       end
       namespace 'files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision', requirements: PACKAGE_REQUIREMENTS do
         before do
@@ -203,21 +204,44 @@ module API
         end
 
         params do
-          requires :file_name, type: String, desc: 'Package file name'
+          requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.conan_file_name_regex
         end
         namespace 'export/:file_name', requirements: FILE_NAME_REQUIREMENTS do
           desc 'Download recipe files' do
-            detail 'This feature was introduced in GitLab 12.5'
+            detail 'This feature was introduced in GitLab 12.6'
           end
           get do
             download_package_file(:recipe_file)
           end
+
+          desc 'Upload recipe package files' do
+            detail 'This feature was introduced in GitLab 12.6'
+          end
+          params do
+            optional 'file.path', type: String, desc: 'Path to locally stored body (generated by Workhorse)'
+            optional 'file.name', type: String, desc: 'Real filename as send in Content-Disposition (generated by Workhorse)'
+            optional 'file.type', type: String, desc: 'Real content type as send in Content-Type (generated by Workhorse)'
+            optional 'file.size', type: Integer, desc: 'Real size of file (generated by Workhorse)'
+            optional 'file.md5', type: String, desc: 'MD5 checksum of the file (generated by Workhorse)'
+            optional 'file.sha1', type: String, desc: 'SHA1 checksum of the file (generated by Workhorse)'
+            optional 'file.sha256', type: String, desc: 'SHA256 checksum of the file (generated by Workhorse)'
+          end
+          put do
+            upload_package_file(:recipe_file)
+          end
+
+          desc 'Workhorse authorize the conan recipe file' do
+            detail 'This feature was introduced in GitLab 12.6'
+          end
+          put 'authorize' do
+            authorize_workhorse
+          end
         end
 
         params do
           requires :conan_package_reference, type: String, desc: 'Conan Package ID'
           requires :package_revision, type: String, desc: 'Conan Package Revision'
-          requires :file_name, type: String, desc: 'Package file name'
+          requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.conan_file_name_regex
         end
         namespace 'package/:conan_package_reference/:package_revision/:file_name', requirements: FILE_NAME_REQUIREMENTS do
           desc 'Download package files' do
@@ -226,6 +250,29 @@ module API
           get do
             download_package_file(:package_file)
           end
+
+          desc 'Workhorse authorize the conan package file' do
+            detail 'This feature was introduced in GitLab 12.6'
+          end
+          put 'authorize' do
+            authorize_workhorse
+          end
+
+          desc 'Upload package files' do
+            detail 'This feature was introduced in GitLab 12.6'
+          end
+          params do
+            optional 'file.path', type: String, desc: 'Path to locally stored body (generated by Workhorse)'
+            optional 'file.name', type: String, desc: 'Real filename as send in Content-Disposition (generated by Workhorse)'
+            optional 'file.type', type: String, desc: 'Real content type as send in Content-Type (generated by Workhorse)'
+            optional 'file.size', type: Integer, desc: 'Real size of file (generated by Workhorse)'
+            optional 'file.md5', type: String, desc: 'MD5 checksum of the file (generated by Workhorse)'
+            optional 'file.sha1', type: String, desc: 'SHA1 checksum of the file (generated by Workhorse)'
+            optional 'file.sha256', type: String, desc: 'SHA256 checksum of the file (generated by Workhorse)'
+          end
+          put do
+            upload_package_file(:package_file)
+          end
         end
       end
     end
@@ -329,6 +376,37 @@ module API
         present_carrierwave_file!(package_file.file)
       end
 
+      def upload_package_file(file_type)
+        authorize_upload
+
+        uploaded_file = UploadedFile.from_params(params, :file, ::Packages::PackageFileUploader.workhorse_local_upload_path)
+        bad_request!('Missing package file!') unless uploaded_file
+
+        current_package = package || ::Packages::Conan::CreatePackageService.new(project, current_user, params).execute
+
+        # conan sends two upload requests, the first has no file, so we skip record creation if file.size == 0
+        ::Packages::Conan::CreatePackageFileService.new(current_package, uploaded_file, params.merge(conan_file_type: file_type)).execute unless params['file.size'] == 0
+      rescue ObjectStorage::RemoteStoreError => e
+        Gitlab::Sentry.track_acceptable_exception(e, extra: { file_name: params[:file_name], project_id: project.id })
+
+        forbidden!
+      end
+
+      def authorize_workhorse
+        authorize_upload
+
+        Gitlab::Workhorse.verify_api_request!(headers)
+
+        status 200
+        content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
+        ::Packages::PackageFileUploader.workhorse_authorize(has_length: true)
+      end
+
+      def authorize_upload
+        authorize!(:create_package, project)
+        require_gitlab_workhorse!
+      end
+
       def find_personal_access_token
         personal_access_token = find_personal_access_token_from_conan_jwt ||
           find_personal_access_token_from_conan_http_basic_auth

ee/spec/models/packages/package_spec.rb

@@ -110,4 +110,14 @@ RSpec.describe Packages::Package, type: :model do
       end
     end
   end
+
+  describe '.with_conan_channel' do
+    let!(:package) { create(:conan_package) }
+
+    subject { described_class.with_conan_channel('stable') }
+
+    it 'includes only packages with specified version' do
+      is_expected.to eq([package])
+    end
+  end
 end

ee/spec/services/packages/conan/create_package_file_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Packages::Conan::CreatePackageFileService do
+  include WorkhorseHelpers
+
+  let_it_be(:package) { create(:conan_package) }
+
+  describe '#execute' do
+    let(:file_name) { 'foo.tgz' }
+
+    subject { described_class.new(package, file, params) }
+
+    shared_examples 'a valid package_file' do
+      let(:params) do
+        {
+          file_name: file_name,
+          'file.md5': '12345',
+          'file.sha1': '54321',
+          'file.size': '128',
+          'file.type': 'txt',
+          recipe_revision: '0',
+          package_revision: '0',
+          conan_package_reference: '123456789',
+          conan_file_type: :package_file
+        }.with_indifferent_access
+      end
+
+      it 'creates a new package file' do
+        package_file = subject.execute
+
+        expect(package_file).to be_valid
+        expect(package_file.file_name).to eq(file_name)
+        expect(package_file.file_md5).to eq('12345')
+        expect(package_file.size).to eq(128)
+        expect(package_file.conan_file_metadatum).to be_valid
+        expect(package_file.conan_file_metadatum.recipe_revision).to eq('0')
+        expect(package_file.conan_file_metadatum.package_revision).to eq('0')
+        expect(package_file.conan_file_metadatum.conan_package_reference).to eq('123456789')
+        expect(package_file.conan_file_metadatum.conan_file_type).to eq('package_file')
+        expect(package_file.file.read).to eq('content')
+      end
+    end
+
+    shared_examples 'a valid recipe_file' do
+      let(:params) do
+        {
+          file_name: file_name,
+          'file.md5': '12345',
+          'file.sha1': '54321',
+          'file.size': '128',
+          'file.type': 'txt',
+          recipe_revision: '0',
+          conan_file_type: :recipe_file
+        }.with_indifferent_access
+      end
+
+      it 'creates a new recipe file' do
+        package_file = subject.execute
+
+        expect(package_file).to be_valid
+        expect(package_file.file_name).to eq(file_name)
+        expect(package_file.file_md5).to eq('12345')
+        expect(package_file.size).to eq(128)
+        expect(package_file.conan_file_metadatum).to be_valid
+        expect(package_file.conan_file_metadatum.recipe_revision).to eq('0')
+        expect(package_file.conan_file_metadatum.package_revision).to be_nil
+        expect(package_file.conan_file_metadatum.conan_package_reference).to be_nil
+        expect(package_file.conan_file_metadatum.conan_file_type).to eq('recipe_file')
+        expect(package_file.file.read).to eq('content')
+      end
+    end
+
+    context 'with temp file' do
+      let!(:file) do
+        upload_path = ::Packages::PackageFileUploader.workhorse_local_upload_path
+        file_path = upload_path + '/' + file_name
+
+        FileUtils.mkdir_p(upload_path)
+        File.write(file_path, 'content')
+
+        UploadedFile.new(file_path, filename: File.basename(file_path))
+      end
+
+      it_behaves_like 'a valid package_file'
+      it_behaves_like 'a valid recipe_file'
+    end
+
+    context 'with remote file' do
+      let!(:fog_connection) do
+        stub_package_file_object_storage(direct_upload: true)
+      end
+
+      let(:tmp_object) do
+        fog_connection.directories.new(key: 'packages').files.create(
+          key: "tmp/uploads/#{file_name}",
+          body: 'content'
+        )
+      end
+
+      let(:file) { fog_to_uploaded_file(tmp_object) }
+
+      it_behaves_like 'a valid package_file'
+      it_behaves_like 'a valid recipe_file'
+    end
+
+    context 'file is missing' do
+      let(:file) { nil }
+      let(:params) do
+        {
+          file_name: file_name,
+          recipe_revision: '0',
+          conan_file_type: :recipe_file
+        }
+      end
+
+      it 'raises an error' do
+        expect { subject.execute }.to raise_error(ActiveRecord::RecordInvalid)
+      end
+    end
+  end
+end

ee/spec/services/packages/conan/create_package_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Packages::Conan::CreatePackageService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+
+  subject { described_class.new(project, user, params) }
+
+  describe '#execute' do
+    context 'valid params' do
+      let(:params) do
+        {
+          package_name: 'my-pkg',
+          package_version: '1.0.0',
+          package_username: ::Packages::ConanMetadatum.package_username_from(full_path: project.full_path),
+          package_channel: 'stable'
+        }
+      end
+
+      it 'creates a new package' do
+        package = subject.execute
+
+        expect(package).to be_valid
+        expect(package.name).to eq(params[:package_name])
+        expect(package.version).to eq(params[:package_version])
+        expect(package.package_type).to eq('conan')
+        expect(package.conan_metadatum.package_username).to eq(params[:package_username])
+        expect(package.conan_metadatum.package_channel).to eq(params[:package_channel])
+      end
+    end
+
+    context 'invalid params' do
+      let(:params) do
+        {
+          package_name: 'my-pkg',
+          package_version: '1.0.0',
+          package_username: 'foo/bar',
+          package_channel: 'stable'
+        }
+      end
+
+      it 'fails' do
+        expect { subject.execute }.to raise_exception(ActiveRecord::RecordInvalid)
+      end
+    end
+  end
+end

spec/support/helpers/stub_object_storage.rb

@@ -56,6 +56,13 @@ module StubObjectStorage
                                  **params)
   end
 
+  def stub_package_file_object_storage(**params)
+    stub_object_storage_uploader(config: Gitlab.config.packages.object_store,
+                                 uploader: ::Packages::PackageFileUploader,
+                                 remote_directory: 'packages',
+                                 **params)
+  end
+
   def stub_uploads_object_storage(uploader = described_class, **params)
     stub_object_storage_uploader(config: Gitlab.config.uploads.object_store,
                                  uploader: uploader,