Merge branch '299262-rubygems-api-skeleton' into 'master'

RubyGems API skeleton and authentication [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!52147

config/feature_flags/development/rubygem_packages.yml

+---
+name: rubygem_packages
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52147
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/299383
+milestone: '13.9'
+type: development
+group: group::package
+default_enabled: false

lib/api/api.rb

@@ -269,6 +269,7 @@ module API
       mount ::API::RemoteMirrors
       mount ::API::Repositories
       mount ::API::ResourceAccessTokens
+      mount ::API::RubygemPackages
       mount ::API::Search
       mount ::API::Services
       mount ::API::Settings

lib/api/nuget_group_packages.rb

@@ -18,7 +18,7 @@ module API
     default_format :json
 
     authenticate_with do |accept|
-      accept.token_types(:personal_access_token, :deploy_token, :job_token)
+      accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username)
             .sent_through(:http_basic_auth)
     end
 

lib/api/nuget_project_packages.rb

@@ -20,7 +20,7 @@ module API
     default_format :json
 
     authenticate_with do |accept|
-      accept.token_types(:personal_access_token, :deploy_token, :job_token)
+      accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username)
             .sent_through(:http_basic_auth)
     end
 

lib/api/rubygem_packages.rb

+# frozen_string_literal: true
+
+###
+# API endpoints for the RubyGem package registry
+module API
+  class RubygemPackages < ::API::Base
+    include ::API::Helpers::Authentication
+    helpers ::API::Helpers::PackagesHelpers
+
+    feature_category :package_registry
+
+    # The Marshal version can be found by "#{Marshal::MAJOR_VERSION}.#{Marshal::MINOR_VERSION}"
+    # Updating the version should require a GitLab API version change.
+    MARSHAL_VERSION = '4.8'
+
+    FILE_NAME_REQUIREMENTS = {
+      file_name: API::NO_SLASH_URL_PART_REGEX
+    }.freeze
+
+    content_type :binary, 'application/octet-stream'
+
+    authenticate_with do |accept|
+      accept.token_types(:personal_access_token, :deploy_token, :job_token)
+            .sent_through(:http_token)
+    end
+
+    before do
+      require_packages_enabled!
+      authenticate!
+      not_found! unless Feature.enabled?(:rubygem_packages, user_project)
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID or full path of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      namespace ':id/packages/rubygems' do
+        desc 'Download the spec index file' do
+          detail 'This feature was introduced in GitLab 13.9'
+        end
+        params do
+          requires :file_name, type: String, desc: 'Spec file name'
+        end
+        get ":file_name", requirements: FILE_NAME_REQUIREMENTS do
+          # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299267
+          not_found!
+        end
+
+        desc 'Download the gemspec file' do
+          detail 'This feature was introduced in GitLab 13.9'
+        end
+        params do
+          requires :file_name, type: String, desc: 'Gemspec file name'
+        end
+        get "quick/Marshal.#{MARSHAL_VERSION}/:file_name", requirements: FILE_NAME_REQUIREMENTS do
+          # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299284
+          not_found!
+        end
+
+        desc 'Download the .gem package' do
+          detail 'This feature was introduced in GitLab 13.9'
+        end
+        params do
+          requires :file_name, type: String, desc: 'Package file name'
+        end
+        get "gems/:file_name", requirements: FILE_NAME_REQUIREMENTS do
+          # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299283
+          not_found!
+        end
+
+        namespace 'api/v1' do
+          desc 'Authorize a gem upload from workhorse' do
+            detail 'This feature was introduced in GitLab 13.9'
+          end
+          post 'gems/authorize' do
+            # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299263
+            not_found!
+          end
+
+          desc 'Upload a gem' do
+            detail 'This feature was introduced in GitLab 13.9'
+          end
+          post 'gems' do
+            # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299263
+            not_found!
+          end
+
+          desc 'Fetch a list of dependencies' do
+            detail 'This feature was introduced in GitLab 13.9'
+          end
+          params do
+            optional :gems, type: String, desc: 'Comma delimited gem names'
+          end
+          get 'dependencies' do
+            # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299282
+            not_found!
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/api_authentication/token_locator.rb

@@ -10,7 +10,7 @@ module Gitlab
 
       attr_reader :location
 
-      validates :location, inclusion: { in: %i[http_basic_auth] }
+      validates :location, inclusion: { in: %i[http_basic_auth http_token] }
 
       def initialize(location)
         @location = location
@@ -21,6 +21,8 @@ module Gitlab
         case @location
         when :http_basic_auth
           extract_from_http_basic_auth request
+        when :http_token
+          extract_from_http_token request
         end
       end
 
@@ -32,6 +34,13 @@ module Gitlab
 
         UsernameAndPassword.new(username, password)
       end
+
+      def extract_from_http_token(request)
+        password = request.headers['Authorization']
+        return unless password.present?
+
+        UsernameAndPassword.new(nil, password)
+      end
     end
   end
 end

lib/gitlab/api_authentication/token_resolver.rb

@@ -7,7 +7,16 @@ module Gitlab
 
       attr_reader :token_type
 
-      validates :token_type, inclusion: { in: %i[personal_access_token job_token deploy_token] }
+      validates :token_type, inclusion: {
+        in: %i[
+          personal_access_token_with_username
+          job_token_with_username
+          deploy_token_with_username
+          personal_access_token
+          job_token
+          deploy_token
+        ]
+      }
 
       def initialize(token_type)
         @token_type = token_type
@@ -38,15 +47,25 @@ module Gitlab
 
         when :deploy_token
           resolve_deploy_token raw
+
+        when :personal_access_token_with_username
+          resolve_personal_access_token_with_username raw
+
+        when :job_token_with_username
+          resolve_job_token_with_username raw
+
+        when :deploy_token_with_username
+          resolve_deploy_token_with_username raw
         end
       end
 
       private
 
-      def resolve_personal_access_token(raw)
-        # Check if the password is a personal access token
-        pat = ::PersonalAccessToken.find_by_token(raw.password)
-        return unless pat
+      def resolve_personal_access_token_with_username(raw)
+        raise ::Gitlab::Auth::UnauthorizedError unless raw.username
+
+        with_personal_access_token(raw) do |pat|
+          break unless pat
 
           # Ensure that the username matches the token. This check is a subtle
           # departure from the existing behavior of #find_personal_access_token_from_http_basic_auth.
@@ -55,25 +74,20 @@ module Gitlab
 
           pat
         end
+      end
 
-      def resolve_job_token(raw)
+      def resolve_job_token_with_username(raw)
         # Only look for a job if the username is correct
         return if ::Gitlab::Auth::CI_JOB_USER != raw.username
 
-        job = ::Ci::AuthJobFinder.new(token: raw.password).execute
-
-        # Actively reject credentials with the username `gitlab-ci-token` if
-        # the password is not a valid job token. This replicates existing
-        # behavior of #find_user_from_job_token.
-        raise ::Gitlab::Auth::UnauthorizedError unless job
-
+        with_job_token(raw) do |job|
           job
         end
+      end
 
-      def resolve_deploy_token(raw)
-        # Check if the password is a deploy token
-        token = ::DeployToken.active.find_by_token(raw.password)
-        return unless token
+      def resolve_deploy_token_with_username(raw)
+        with_deploy_token(raw) do |token|
+          break unless token
 
           # Ensure that the username matches the token. This check is a subtle
           # departure from the existing behavior of #deploy_token_from_request.
@@ -83,5 +97,45 @@ module Gitlab
           token
         end
       end
+
+      def resolve_personal_access_token(raw)
+        with_personal_access_token(raw) do |pat|
+          pat
+        end
+      end
+
+      def resolve_job_token(raw)
+        with_job_token(raw) do |job|
+          job
+        end
+      end
+
+      def resolve_deploy_token(raw)
+        with_deploy_token(raw) do |token|
+          token
+        end
+      end
+
+      def with_personal_access_token(raw, &block)
+        pat = ::PersonalAccessToken.find_by_token(raw.password)
+        return unless pat
+
+        yield(pat)
+      end
+
+      def with_deploy_token(raw, &block)
+        token = ::DeployToken.active.find_by_token(raw.password)
+        return unless token
+
+        yield(token)
+      end
+
+      def with_job_token(raw, &block)
+        job = ::Ci::AuthJobFinder.new(token: raw.password).execute
+        raise ::Gitlab::Auth::UnauthorizedError unless job
+
+        yield(job)
+      end
+    end
   end
 end

spec/lib/gitlab/api_authentication/token_locator_spec.rb

@@ -51,5 +51,26 @@ RSpec.describe Gitlab::APIAuthentication::TokenLocator do
         end
       end
     end
+
+    context 'with :http_token' do
+      let(:type) { :http_token }
+
+      context 'without credentials' do
+        let(:request) { double(headers: {}) }
+
+        it 'returns nil' do
+          expect(subject).to be(nil)
+        end
+      end
+
+      context 'with credentials' do
+        let(:password) { 'bar' }
+        let(:request) { double(headers: { "Authorization" => password }) }
+
+        it 'returns the credentials' do
+          expect(subject.password).to eq(password)
+        end
+      end
+    end
   end
 end

spec/lib/gitlab/api_authentication/token_resolver_spec.rb

@@ -47,8 +47,8 @@ RSpec.describe Gitlab::APIAuthentication::TokenResolver do
 
     subject { resolver.resolve(raw) }
 
-    context 'with :personal_access_token' do
-      let(:type) { :personal_access_token }
+    context 'with :personal_access_token_with_username' do
+      let(:type) { :personal_access_token_with_username }
       let(:token) { personal_access_token }
 
       context 'with valid credentials' do
@@ -62,10 +62,16 @@ RSpec.describe Gitlab::APIAuthentication::TokenResolver do
 
         it_behaves_like 'an unauthorized request'
       end
+
+      context 'with no username' do
+        let(:raw) { username_and_password(nil, token.token) }
+
+        it_behaves_like 'an unauthorized request'
+      end
     end
 
-    context 'with :job_token' do
-      let(:type) { :job_token }
+    context 'with :job_token_with_username' do
+      let(:type) { :job_token_with_username }
       let(:token) { ci_job }
 
       context 'with valid credentials' do
@@ -93,8 +99,8 @@ RSpec.describe Gitlab::APIAuthentication::TokenResolver do
       end
     end
 
-    context 'with :deploy_token' do
-      let(:type) { :deploy_token }
+    context 'with :deploy_token_with_username' do
+      let(:type) { :deploy_token_with_username }
       let(:token) { deploy_token }
 
       context 'with a valid deploy token' do
@@ -109,6 +115,51 @@ RSpec.describe Gitlab::APIAuthentication::TokenResolver do
         it_behaves_like 'an unauthorized request'
       end
     end
+
+    context 'with :personal_access_token' do
+      let(:type) { :personal_access_token }
+      let(:token) { personal_access_token }
+
+      context 'with valid credentials' do
+        let(:raw) { username_and_password(nil, token.token) }
+
+        it_behaves_like 'an authorized request'
+      end
+    end
+
+    context 'with :job_token' do
+      let(:type) { :job_token }
+      let(:token) { ci_job }
+
+      context 'with valid credentials' do
+        let(:raw) { username_and_password(nil, token.token) }
+
+        it_behaves_like 'an authorized request'
+      end
+
+      context 'when the job is not running' do
+        let(:raw) { username_and_password(nil, ci_job_done.token) }
+
+        it_behaves_like 'an unauthorized request'
+      end
+
+      context 'with an invalid job token' do
+        let(:raw) { username_and_password(nil, "not a valid CI job token") }
+
+        it_behaves_like 'an unauthorized request'
+      end
+    end
+
+    context 'with :deploy_token' do
+      let(:type) { :deploy_token }
+      let(:token) { deploy_token }
+
+      context 'with a valid deploy token' do
+        let(:raw) { username_and_password(nil, token.token) }
+
+        it_behaves_like 'an authorized request'
+      end
+    end
   end
 
   def username_and_password(username, password)

spec/requests/api/rubygem_packages_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::RubygemPackages do
+  using RSpec::Parameterized::TableSyntax
+
+  let_it_be(:project) { create(:project) }
+  let_it_be(:personal_access_token) { create(:personal_access_token) }
+  let_it_be(:user) { personal_access_token.user }
+  let_it_be(:job) { create(:ci_build, :running, user: user) }
+  let_it_be(:deploy_token) { create(:deploy_token, read_package_registry: true, write_package_registry: true) }
+  let_it_be(:project_deploy_token) { create(:project_deploy_token, deploy_token: deploy_token, project: project) }
+  let_it_be(:headers) { {} }
+
+  shared_examples 'when feature flag is disabled' do
+    let(:headers) do
+      { 'HTTP_AUTHORIZATION' => personal_access_token.token }
+    end
+
+    before do
+      stub_feature_flags(rubygem_packages: false)
+    end
+
+    it_behaves_like 'returning response status', :not_found
+  end
+
+  shared_examples 'when package feature is disabled' do
+    before do
+      stub_config(packages: { enabled: false })
+    end
+
+    it_behaves_like 'returning response status', :not_found
+  end
+
+  shared_examples 'without authentication' do
+    it_behaves_like 'returning response status', :unauthorized
+  end
+
+  shared_examples 'with authentication' do
+    let(:headers) do
+      { 'HTTP_AUTHORIZATION' => token }
+    end
+
+    let(:tokens) do
+      {
+        personal_access_token: personal_access_token.token,
+        deploy_token: deploy_token.token,
+        job_token: job.token
+      }
+    end
+
+    where(:user_role, :token_type, :valid_token, :status) do
+      :guest     | :personal_access_token   | true  | :not_found
+      :guest     | :personal_access_token   | false | :unauthorized
+      :guest     | :deploy_token            | true  | :not_found
+      :guest     | :deploy_token            | false | :unauthorized
+      :guest     | :job_token               | true  | :not_found
+      :guest     | :job_token               | false | :unauthorized
+      :reporter  | :personal_access_token   | true  | :not_found
+      :reporter  | :personal_access_token   | false | :unauthorized
+      :reporter  | :deploy_token            | true  | :not_found
+      :reporter  | :deploy_token            | false | :unauthorized
+      :reporter  | :job_token               | true  | :not_found
+      :reporter  | :job_token               | false | :unauthorized
+      :developer | :personal_access_token   | true  | :not_found
+      :developer | :personal_access_token   | false | :unauthorized
+      :developer | :deploy_token            | true  | :not_found
+      :developer | :deploy_token            | false | :unauthorized
+      :developer | :job_token               | true  | :not_found
+      :developer | :job_token               | false | :unauthorized
+    end
+
+    with_them do
+      before do
+        project.send("add_#{user_role}", user) unless user_role == :anonymous
+      end
+
+      let(:token) { valid_token ? tokens[token_type] : 'invalid-token123' }
+
+      it_behaves_like 'returning response status', params[:status]
+    end
+  end
+
+  shared_examples 'an unimplemented route' do
+    it_behaves_like 'without authentication'
+    it_behaves_like 'with authentication'
+    it_behaves_like 'when feature flag is disabled'
+    it_behaves_like 'when package feature is disabled'
+  end
+
+  describe 'GET /api/v4/projects/:project_id/packages/rubygems/:filename' do
+    let(:url) { api("/projects/#{project.id}/packages/rubygems/specs.4.8.gz") }
+
+    subject { get(url, headers: headers) }
+
+    it_behaves_like 'an unimplemented route'
+  end
+
+  describe 'GET /api/v4/projects/:project_id/packages/rubygems/quick/Marshal.4.8/:file_name' do
+    let(:url) { api("/projects/#{project.id}/packages/rubygems/quick/Marshal.4.8/my_gem-1.0.0.gemspec.rz") }
+
+    subject { get(url, headers: headers) }
+
+    it_behaves_like 'an unimplemented route'
+  end
+
+  describe 'GET /api/v4/projects/:project_id/packages/rubygems/gems/:file_name' do
+    let(:url) { api("/projects/#{project.id}/packages/rubygems/gems/my_gem-1.0.0.gem") }
+
+    subject { get(url, headers: headers) }
+
+    it_behaves_like 'an unimplemented route'
+  end
+
+  describe 'POST /api/v4/projects/:project_id/packages/rubygems/api/v1/gems/authorize' do
+    let(:url) { api("/projects/#{project.id}/packages/rubygems/api/v1/gems/authorize") }
+
+    subject { post(url, headers: headers) }
+
+    it_behaves_like 'an unimplemented route'
+  end
+
+  describe 'POST /api/v4/projects/:project_id/packages/rubygems/api/v1/gems' do
+    let(:url) { api("/projects/#{project.id}/packages/rubygems/api/v1/gems") }
+
+    subject { post(url, headers: headers) }
+
+    it_behaves_like 'an unimplemented route'
+  end
+
+  describe 'GET /api/v4/projects/:project_id/packages/rubygems/api/v1/dependencies' do
+    let(:url) { api("/projects/#{project.id}/packages/rubygems/api/v1/dependencies") }
+
+    subject { get(url, headers: headers) }
+
+    it_behaves_like 'an unimplemented route'
+  end
+end