Add support for generic package file uploads

Add new API endpoints to support uploading generic packages files
(accelerated through Workhorse).

Related to https://gitlab.com/gitlab-org/gitlab/-/issues/235492.

app/services/packages/create_package_service.rb

@@ -10,6 +10,7 @@ module Packages
         .with_package_type(package_type)
         .safe_find_or_create_by!(name: name, version: version) do |pkg|
           pkg.creator = package_creator
+          yield pkg if block_given?
         end
     end
 

app/services/packages/generic/create_package_file_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module Generic
+    class CreatePackageFileService < BaseService
+      def execute
+        ::Packages::Package.transaction do
+          create_package_file(find_or_create_package)
+        end
+      end
+
+      private
+
+      def find_or_create_package
+        package_params = {
+          name: params[:package_name],
+          version: params[:package_version],
+          build: params[:build]
+        }
+
+        ::Packages::Generic::FindOrCreatePackageService
+          .new(project, current_user, package_params)
+          .execute
+      end
+
+      def create_package_file(package)
+        file_params = {
+          file: params[:file],
+          size: params[:file].size,
+          file_sha256: params[:file].sha256,
+          file_name: params[:file_name]
+        }
+
+        ::Packages::CreatePackageFileService.new(package, file_params).execute
+      end
+    end
+  end
+end

app/services/packages/generic/find_or_create_package_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module Generic
+    class FindOrCreatePackageService < ::Packages::CreatePackageService
+      def execute
+        find_or_create_package!(::Packages::Package.package_types['generic']) do |package|
+          if params[:build].present?
+            package.build_info = Packages::BuildInfo.new(pipeline: params[:build].pipeline)
+          end
+        end
+      end
+    end
+  end
+end

lib/api/generic_packages.rb

@@ -2,6 +2,11 @@
 
 module API
   class GenericPackages < Grape::API::Instance
+    GENERIC_PACKAGES_REQUIREMENTS = {
+      package_name: API::NO_SLASH_URL_PART_REGEX,
+      file_name: API::NO_SLASH_URL_PART_REGEX
+    }.freeze
+
     before do
       require_packages_enabled!
       authenticate!
@@ -17,17 +22,71 @@ module API
       route_setting :authentication, job_token_allowed: true
 
       namespace ':id/packages/generic' do
-        get 'ping' do
-          :pong
+        namespace ':package_name/*package_version/:file_name', requirements: GENERIC_PACKAGES_REQUIREMENTS do
+          desc 'Workhorse authorize generic package file' do
+            detail 'This feature was introduced in GitLab 13.5'
+          end
+
+          route_setting :authentication, job_token_allowed: true
+
+          params do
+            requires :package_name, type: String, desc: 'Package name'
+            requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex
+            requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true
+          end
+
+          put 'authorize' do
+            authorize_workhorse!(subject: project, maximum_size: project.actual_limits.generic_packages_max_file_size)
+          end
+
+          desc 'Upload package file' do
+            detail 'This feature was introduced in GitLab 13.5'
+          end
+
+          params do
+            requires :package_name, type: String, desc: 'Package name'
+            requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex
+            requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true
+            requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)'
+          end
+
+          route_setting :authentication, job_token_allowed: true
+
+          put do
+            authorize_upload!(project)
+            bad_request!('File is too large') if max_file_size_exceeded?
+
+            track_event('push_package')
+
+            create_package_file_params = declared_params.merge(build: current_authenticated_job)
+            ::Packages::Generic::CreatePackageFileService
+              .new(project, current_user, create_package_file_params)
+              .execute
+
+            created!
+          rescue ObjectStorage::RemoteStoreError => e
+            Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project.id })
+
+            forbidden!
+          end
         end
       end
     end
 
     helpers do
       include ::API::Helpers::PackagesHelpers
+      include ::API::Helpers::Packages::BasicAuthHelpers
 
       def require_generic_packages_available!
-        not_found! unless Feature.enabled?(:generic_packages, user_project)
+        not_found! unless Feature.enabled?(:generic_packages, project)
+      end
+
+      def project
+        authorized_user_project
+      end
+
+      def max_file_size_exceeded?
+        project.actual_limits.exceeded?(:generic_packages_max_file_size, params[:file].size)
       end
     end
   end

lib/gitlab/regex.rb

@@ -103,6 +103,10 @@ module Gitlab
       def generic_package_version_regex
         /\A\d+\.\d+\.\d+\z/
       end
+
+      def generic_package_file_name_regex
+        maven_file_name_regex
+      end
     end
 
     extend self

spec/lib/gitlab/regex_spec.rb

@@ -434,4 +434,18 @@ RSpec.describe Gitlab::Regex do
     it { is_expected.not_to match('%2e%2e%2f1.2.3') }
     it { is_expected.not_to match('') }
   end
+
+  describe '.generic_package_file_name_regex' do
+    subject { described_class.generic_package_file_name_regex }
+
+    it { is_expected.to match('123') }
+    it { is_expected.to match('foo') }
+    it { is_expected.to match('foo.bar.baz-2.0-20190901.47283-1.jar') }
+    it { is_expected.not_to match('../../foo') }
+    it { is_expected.not_to match('..\..\foo') }
+    it { is_expected.not_to match('%2f%2e%2e%2f%2essh%2fauthorized_keys') }
+    it { is_expected.not_to match('$foo/bar') }
+    it { is_expected.not_to match('my file name') }
+    it { is_expected.not_to match('!!()()') }
+  end
 end

spec/services/packages/generic/create_package_file_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Generic::CreatePackageFileService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+
+  describe '#execute' do
+    let(:sha256) { '440e5e148a25331bbd7991575f7d54933c0ebf6cc735a18ee5066ac1381bb590' }
+    let(:temp_file) { Tempfile.new("test") }
+    let(:file) { UploadedFile.new(temp_file.path, sha256: sha256) }
+    let(:package) { create(:generic_package, project: project) }
+    let(:params) do
+      {
+        package_name: 'mypackage',
+        package_version: '0.0.1',
+        file: file,
+        file_name: 'myfile.tar.gz.1'
+      }
+    end
+
+    before do
+      FileUtils.touch(temp_file)
+    end
+
+    after do
+      FileUtils.rm_f(temp_file)
+    end
+
+    it 'creates package file' do
+      package_service = double
+      package_params = {
+        name: params[:package_name],
+        version: params[:package_version],
+        build: params[:build]
+      }
+      expect(::Packages::Generic::FindOrCreatePackageService).to receive(:new).with(project, user, package_params).and_return(package_service)
+      expect(package_service).to receive(:execute).and_return(package)
+
+      service = described_class.new(project, user, params)
+
+      expect { service.execute }.to change { package.package_files.count }.by(1)
+
+      package_file = package.package_files.last
+      aggregate_failures do
+        expect(package_file.package).to eq(package)
+        expect(package_file.file_name).to eq('myfile.tar.gz.1')
+        expect(package_file.size).to eq(file.size)
+        expect(package_file.file_sha256).to eq(sha256)
+      end
+    end
+  end
+end

spec/services/packages/generic/find_or_create_package_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Generic::FindOrCreatePackageService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:ci_build) { create(:ci_build, :running, user: user) }
+
+  let(:params) do
+    {
+      name: 'mypackage',
+      version: '0.0.1'
+    }
+  end
+
+  describe '#execute' do
+    context 'when packages does not exist yet' do
+      it 'creates package' do
+        service = described_class.new(project, user, params)
+
+        expect { service.execute }.to change { project.packages.generic.count }.by(1)
+
+        package = project.packages.generic.last
+
+        aggregate_failures do
+          expect(package.creator).to eq(user)
+          expect(package.name).to eq('mypackage')
+          expect(package.version).to eq('0.0.1')
+          expect(package.build_info).to be_nil
+        end
+      end
+
+      it 'creates package and package build info when build is provided' do
+        service = described_class.new(project, user, params.merge(build: ci_build))
+
+        expect { service.execute }.to change { project.packages.generic.count }.by(1)
+
+        package = project.packages.generic.last
+
+        aggregate_failures do
+          expect(package.creator).to eq(user)
+          expect(package.name).to eq('mypackage')
+          expect(package.version).to eq('0.0.1')
+          expect(package.build_info.pipeline).to eq(ci_build.pipeline)
+        end
+      end
+    end
+
+    context 'when packages already exists' do
+      let!(:package) { project.packages.generic.create!(params) }
+
+      context 'when package was created manually' do
+        it 'finds the package and does not create package build info even if build is provided' do
+          service = described_class.new(project, user, params.merge(build: ci_build))
+
+          expect do
+            found_package = service.execute
+
+            expect(found_package).to eq(package)
+          end.not_to change { project.packages.generic.count }
+
+          expect(package.reload.build_info).to be_nil
+        end
+      end
+
+      context 'when package was created by pipeline' do
+        let(:pipeline) { create(:ci_pipeline, project: project) }
+
+        before do
+          package.create_build_info!(pipeline: pipeline)
+        end
+
+        it 'finds the package and does not change package build info even if build is provided' do
+          service = described_class.new(project, user, params.merge(build: ci_build))
+
+          expect do
+            found_package = service.execute
+
+            expect(found_package).to eq(package)
+          end.not_to change { project.packages.generic.count }
+
+          expect(package.reload.build_info.pipeline).to eq(pipeline)
+        end
+      end
+    end
+  end
+end