Merge branch '299263-rubygems-upload-routes' into 'master'

RubyGems create package service and upload routes

See merge request gitlab-org/gitlab!53480

app/models/packages/nuget.rb

 # frozen_string_literal: true
 module Packages
   module Nuget
+    TEMPORARY_PACKAGE_NAME = 'NuGet.Temporary.Package'
+
     def self.table_name_prefix
       'packages_nuget_'
     end

app/models/packages/package.rb

@@ -98,12 +98,12 @@ class Packages::Package < ApplicationRecord
   end
   scope :preload_composer, -> { preload(:composer_metadatum) }
 
-  scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) }
+  scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) }
 
   scope :has_version, -> { where.not(version: nil) }
   scope :processed, -> do
     where.not(package_type: :nuget).or(
-      where.not(name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME)
+      where.not(name: Packages::Nuget::TEMPORARY_PACKAGE_NAME)
     )
   end
   scope :preload_files, -> { preload(:package_files) }

app/models/packages/rubygems.rb

+# frozen_string_literal: true
+module Packages
+  module Rubygems
+    TEMPORARY_PACKAGE_NAME = 'Gem.Temporary.Package'
+
+    def self.table_name_prefix
+      'packages_rubygems_'
+    end
+  end
+end

app/models/packages/rubygems/metadatum.rb

@@ -3,7 +3,6 @@
 module Packages
   module Rubygems
     class Metadatum < ApplicationRecord
-      self.table_name = 'packages_rubygems_metadata'
       self.primary_key = :package_id
 
       belongs_to :package, -> { where(package_type: :rubygems) }, inverse_of: :rubygems_metadatum

app/services/packages/create_temporary_package_service.rb

+# frozen_string_literal: true
+
+module Packages
+  class CreateTemporaryPackageService < ::Packages::CreatePackageService
+    PACKAGE_VERSION = '0.0.0'
+
+    def execute(package_type, name: 'Temporary.Package')
+      create_package!(package_type,
+        name: name,
+        version: "#{PACKAGE_VERSION}-#{uuid}",
+        status: 'processing'
+      )
+    end
+
+    private
+
+    def uuid
+      SecureRandom.uuid
+    end
+  end
+end

app/services/packages/nuget/create_package_service.rb

-# frozen_string_literal: true
-
-module Packages
-  module Nuget
-    class CreatePackageService < ::Packages::CreatePackageService
-      TEMPORARY_PACKAGE_NAME = 'NuGet.Temporary.Package'
-      PACKAGE_VERSION = '0.0.0'
-
-      def execute
-        create_package!(:nuget,
-          name: TEMPORARY_PACKAGE_NAME,
-          version: "#{PACKAGE_VERSION}-#{uuid}"
-        )
-      end
-
-      private
-
-      def uuid
-        SecureRandom.uuid
-      end
-    end
-  end
-end

app/services/packages/nuget/update_package_from_metadata_service.rb

@@ -68,7 +68,8 @@ module Packages
       def update_linked_package
         @package_file.package.update!(
           name: package_name,
-          version: package_version
+          version: package_version,
+          status: :default
         )
 
         ::Packages::Nuget::CreateDependencyService.new(@package_file.package, package_dependencies)

db/fixtures/development/26_packages.rb

@@ -103,8 +103,10 @@ class Gitlab::Seeder::Packages
       name = "MyNugetApp.Package#{i}"
       version = "4.2.#{i}"
 
-      pkg = ::Packages::Nuget::CreatePackageService.new(project, project.creator, {}).execute
-      # when using ::Packages::Nuget::CreatePackageService, packages have a fixed name and a fixed version.
+      pkg = ::Packages::CreateTemporaryPackageService.new(
+        project, project.creator, {}
+      ).execute(:nuget, name: Packages::Nuget::TEMPORARY_PACKAGE_NAME)
+      # when using ::Packages::CreateTemporaryPackageService, packages have a fixed name and a fixed version.
       pkg.update!(name: name, version: version)
 
       filename = 'package.nupkg'

lib/api/nuget_project_packages.rb

@@ -62,8 +62,9 @@ module API
             file_name: PACKAGE_FILENAME
           )
 
-          package = ::Packages::Nuget::CreatePackageService.new(project_or_group, current_user, declared_params.merge(build: current_authenticated_job))
-                                                           .execute
+          package = ::Packages::CreateTemporaryPackageService.new(
+            project_or_group, current_user, declared_params.merge(build: current_authenticated_job)
+          ).execute(:nuget, name: ::Packages::Nuget::TEMPORARY_PACKAGE_NAME)
 
           package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job))
                                                              .execute

lib/api/rubygem_packages.rb

@@ -12,7 +12,7 @@ module API
     # The Marshal version can be found by "#{Marshal::MAJOR_VERSION}.#{Marshal::MINOR_VERSION}"
     # Updating the version should require a GitLab API version change.
     MARSHAL_VERSION = '4.8'
-
+    PACKAGE_FILENAME = 'package.gem'
     FILE_NAME_REQUIREMENTS = {
       file_name: API::NO_SLASH_URL_PART_REGEX
     }.freeze
@@ -73,16 +73,45 @@ module API
             detail 'This feature was introduced in GitLab 13.9'
           end
           post 'gems/authorize' do
-            # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299263
-            not_found!
+            authorize_workhorse!(
+              subject: user_project,
+              has_length: false,
+              maximum_size: user_project.actual_limits.rubygems_max_file_size
+            )
           end
 
           desc 'Upload a gem' do
             detail 'This feature was introduced in GitLab 13.9'
           end
+          params do
+            requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)'
+          end
           post 'gems' do
-            # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299263
-            not_found!
+            authorize_upload!(user_project)
+            bad_request!('File is too large') if user_project.actual_limits.exceeded?(:rubygems_max_file_size, params[:file].size)
+
+            track_package_event('push_package', :rubygems)
+
+            ActiveRecord::Base.transaction do
+              package = ::Packages::CreateTemporaryPackageService.new(
+                user_project, current_user, declared_params.merge(build: current_authenticated_job)
+              ).execute(:rubygems, name: ::Packages::Rubygems::TEMPORARY_PACKAGE_NAME)
+
+              file_params = {
+                file:      params[:file],
+                file_name: PACKAGE_FILENAME
+              }
+
+              ::Packages::CreatePackageFileService.new(
+                package, file_params.merge(build: current_authenticated_job)
+              ).execute
+            end
+
+            created!
+          rescue ObjectStorage::RemoteStoreError => e
+            Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: user_project.id })
+
+            forbidden!
           end
 
           desc 'Fetch a list of dependencies' do

lib/gitlab/usage_data_counters/counter_events/package_events.yml

@@ -41,6 +41,9 @@
 - i_package_pypi_delete_package
 - i_package_pypi_pull_package
 - i_package_pypi_push_package
+- i_package_rubygems_delete_package
+- i_package_rubygems_pull_package
+- i_package_rubygems_push_package
 - i_package_tag_delete_package
 - i_package_tag_pull_package
 - i_package_tag_push_package

lib/gitlab/usage_data_counters/known_events/package_events.yml

@@ -99,6 +99,16 @@
   aggregation: weekly
   redis_slot: package
   feature_flag: collect_package_events_redis
+- name: i_package_rubygems_deploy_token
+  category: deploy_token_packages
+  aggregation: weekly
+  redis_slot: package
+  feature_flag: collect_package_events_redis
+- name: i_package_rubygems_user
+  category: user_packages
+  aggregation: weekly
+  redis_slot: package
+  feature_flag: collect_package_events_redis
 - name: i_package_tag_deploy_token
   category: deploy_token_packages
   aggregation: weekly

spec/finders/packages/group_packages_finder_spec.rb

@@ -122,7 +122,7 @@ RSpec.describe Packages::GroupPackagesFinder do
       end
 
       context 'when there are processing packages' do
-        let_it_be(:package4) { create(:nuget_package, project: project, name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) }
+        let_it_be(:package4) { create(:nuget_package, project: project, name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) }
 
         it { is_expected.to match_array([package1, package2]) }
       end

spec/finders/packages/package_finder_spec.rb

@@ -14,7 +14,7 @@ RSpec.describe ::Packages::PackageFinder do
     it { is_expected.to eq(maven_package) }
 
     context 'processing packages' do
-      let_it_be(:nuget_package) { create(:nuget_package, project: project, name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) }
+      let_it_be(:nuget_package) { create(:nuget_package, project: project, name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) }
       let(:package_id) { nuget_package.id }
 
       it 'are not returned' do

spec/finders/packages/packages_finder_spec.rb

@@ -76,7 +76,7 @@ RSpec.describe ::Packages::PackagesFinder do
     end
 
     context 'with processing packages' do
-      let_it_be(:nuget_package) { create(:nuget_package, project: project, name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) }
+      let_it_be(:nuget_package) { create(:nuget_package, project: project, name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) }
 
       it { is_expected.to match_array([conan_package, maven_package]) }
     end

spec/lib/gitlab/usage_data_counters/package_event_counter_spec.rb

@@ -14,7 +14,7 @@ RSpec.describe Gitlab::UsageDataCounters::PackageEventCounter, :clean_gitlab_red
   end
 
   it 'includes the right events' do
-    expect(described_class::KNOWN_EVENTS.size).to eq 45
+    expect(described_class::KNOWN_EVENTS.size).to eq 48
   end
 
   described_class::KNOWN_EVENTS.each do |event|

spec/models/packages/package_spec.rb

@@ -513,7 +513,7 @@ RSpec.describe Packages::Package, type: :model do
 
   describe '.without_nuget_temporary_name' do
     let!(:package1) { create(:nuget_package) }
-    let!(:package2) { create(:nuget_package, name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) }
+    let!(:package2) { create(:nuget_package, name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) }
 
     subject { described_class.without_nuget_temporary_name }
 
@@ -532,7 +532,7 @@ RSpec.describe Packages::Package, type: :model do
     it { is_expected.to match_array([package1, package2, package3]) }
 
     context 'with temporary packages' do
-      let!(:package1) { create(:nuget_package, name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) }
+      let!(:package1) { create(:nuget_package, name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) }
 
       it { is_expected.to match_array([package2, package3]) }
     end

spec/services/packages/nuget/create_package_service_spec.rb → spec/services/packages/create_temporary_package_service_spec.rb

 # frozen_string_literal: true
+
 require 'spec_helper'
 
-RSpec.describe Packages::Nuget::CreatePackageService do
+RSpec.describe Packages::CreateTemporaryPackageService do
   let_it_be(:project) { create(:project) }
   let_it_be(:user) { create(:user) }
   let_it_be(:params) { {} }
+  let_it_be(:package_name) { 'my-package' }
+  let_it_be(:package_type) { 'rubygems' }
 
   describe '#execute' do
-    subject { described_class.new(project, user, params).execute }
+    subject { described_class.new(project, user, params).execute(package_type, name: package_name) }
 
     let(:package) { Packages::Package.last }
 
-    it 'creates the package' do
+    it 'creates the package', :aggregate_failures do
       expect { subject }.to change { Packages::Package.count }.by(1)
 
       expect(package).to be_valid
-      expect(package.name).to eq(Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME)
-      expect(package.version).to start_with(Packages::Nuget::CreatePackageService::PACKAGE_VERSION)
-      expect(package.package_type).to eq('nuget')
+      expect(package).to be_processing
+      expect(package.name).to eq(package_name)
+      expect(package.version).to start_with(described_class::PACKAGE_VERSION)
+      expect(package.package_type).to eq(package_type)
     end
 
-    it 'can create two packages in a row' do
+    it 'can create two packages in a row', :aggregate_failures do
       expect { subject }.to change { Packages::Package.count }.by(1)
-      expect { described_class.new(project, user, params).execute }.to change { Packages::Package.count }.by(1)
+
+      expect do
+        described_class.new(project, user, params).execute(package_type, name: package_name)
+      end.to change { Packages::Package.count }.by(1)
 
       expect(package).to be_valid
-      expect(package.name).to eq(Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME)
-      expect(package.version).to start_with(Packages::Nuget::CreatePackageService::PACKAGE_VERSION)
-      expect(package.package_type).to eq('nuget')
+      expect(package).to be_processing
+      expect(package.name).to eq(package_name)
+      expect(package.version).to start_with(described_class::PACKAGE_VERSION)
+      expect(package.package_type).to eq(package_type)
     end
 
     it_behaves_like 'assigns the package creator'
     it_behaves_like 'assigns build to package'
-    it_behaves_like 'assigns status to package'
   end
 end

spec/services/packages/nuget/update_package_from_metadata_service_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_redis_shared_state do
   include ExclusiveLeaseHelpers
 
-  let(:package) { create(:nuget_package) }
+  let(:package) { create(:nuget_package, :processing) }
   let(:package_file) { package.package_files.first }
   let(:service) { described_class.new(package_file) }
   let(:package_name) { 'DummyProject.DummyPackage' }
@@ -60,6 +60,7 @@ RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_
             .to change { ::Packages::Package.count }.by(0)
             .and change { Packages::DependencyLink.count }.by(0)
           expect(package_file.reload.file_name).not_to eq(package_file_name)
+          expect(package_file.package).to be_processing
           expect(package_file.package.reload.name).not_to eq(package_name)
           expect(package_file.package.version).not_to eq(package_version)
         end
@@ -78,6 +79,7 @@ RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_
 
         expect(package.reload.name).to eq(package_name)
         expect(package.version).to eq(package_version)
+        expect(package).to be_default
         expect(package_file.reload.file_name).to eq(package_file_name)
         # hard reset needed to properly reload package_file.file
         expect(Packages::PackageFile.find(package_file.id).file.size).not_to eq 0
@@ -184,6 +186,7 @@ RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_
 
         expect(package.reload.name).to eq(package_name)
         expect(package.version).to eq(package_version)
+        expect(package).to be_default
         expect(package_file.reload.file_name).to eq(package_file_name)
         # hard reset needed to properly reload package_file.file
         expect(Packages::PackageFile.find(package_file.id).file.size).not_to eq 0

spec/support/shared_examples/requests/api/rubygems_packages_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'rejects rubygems packages access' do |user_type, status, add_member = true|
+  context "for user type #{user_type}" do
+    before do
+      project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+    end
+
+    it_behaves_like 'returning response status', status
+  end
+end
+
+RSpec.shared_examples 'process rubygems workhorse authorization' do |user_type, status, add_member = true|
+  context "for user type #{user_type}" do
+    before do
+      project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+    end
+
+    it_behaves_like 'returning response status', status
+
+    it 'has the proper content type' do
+      subject
+
+      expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
+    end
+
+    context 'with a request that bypassed gitlab-workhorse' do
+      let(:headers) do
+        { 'HTTP_AUTHORIZATION' => personal_access_token.token }
+          .merge(workhorse_headers)
+          .tap { |h| h.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER) }
+      end
+
+      before do
+        project.add_maintainer(user)
+      end
+
+      it_behaves_like 'returning response status', :forbidden
+    end
+  end
+end
+
+RSpec.shared_examples 'process rubygems upload' do |user_type, status, add_member = true|
+  RSpec.shared_examples 'creates rubygems package files' do
+    it 'creates package files', :aggregate_failures do
+      expect { subject }
+          .to change { project.packages.count }.by(1)
+          .and change { Packages::PackageFile.count }.by(1)
+      expect(response).to have_gitlab_http_status(status)
+
+      package_file = project.packages.last.package_files.reload.last
+      expect(package_file.file_name).to eq('package.gem')
+    end
+  end
+
+  context "for user type #{user_type}" do
+    before do
+      project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+    end
+
+    context 'with object storage disabled' do
+      before do
+        stub_package_file_object_storage(enabled: false)
+      end
+
+      context 'without a file from workhorse' do
+        let(:send_rewritten_field) { false }
+
+        it_behaves_like 'returning response status', :bad_request
+      end
+
+      context 'with correct params' do
+        it_behaves_like 'package workhorse uploads'
+        it_behaves_like 'creates rubygems package files'
+        it_behaves_like 'a package tracking event', 'API::RubygemPackages', 'push_package'
+      end
+    end
+
+    context 'with object storage enabled' do
+      let(:tmp_object) do
+        fog_connection.directories.new(key: 'packages').files.create( # rubocop:disable Rails/SaveBang
+          key: "tmp/uploads/#{file_name}",
+          body: 'content'
+        )
+      end
+
+      let(:fog_file) { fog_to_uploaded_file(tmp_object) }
+      let(:params) { { file: fog_file, 'file.remote_id' => file_name } }
+
+      context 'and direct upload enabled' do
+        let(:fog_connection) do
+          stub_package_file_object_storage(direct_upload: true)
+        end
+
+        it_behaves_like 'creates rubygems package files'
+
+        ['123123', '../../123123'].each do |remote_id|
+          context "with invalid remote_id: #{remote_id}" do
+            let(:params) do
+              {
+                file: fog_file,
+                'file.remote_id' => remote_id
+              }
+            end
+
+            it_behaves_like 'returning response status', :forbidden
+          end
+        end
+      end
+
+      context 'and direct upload disabled' do
+        context 'and background upload disabled' do
+          let(:fog_connection) do
+            stub_package_file_object_storage(direct_upload: false, background_upload: false)
+          end
+
+          it_behaves_like 'creates rubygems package files'
+        end
+
+        context 'and background upload enabled' do
+          let(:fog_connection) do
+            stub_package_file_object_storage(direct_upload: false, background_upload: true)
+          end
+
+          it_behaves_like 'creates rubygems package files'
+        end
+      end
+    end
+  end
+end