Group level PyPI packages

Adds endpoints to the PyPI package manager API
to allow consumption of packages at the group
level.

Changlog: added

app/finders/packages/group_or_project_package_finder.rb

@@ -26,9 +26,9 @@ module Packages
 
     def base
       if project?
-        packages_for_project(@project_or_group)
+        project_packages
       elsif group?
-        packages_visible_to_user(@current_user, within_group: @project_or_group)
+        group_packages
       else
         ::Packages::Package.none
       end
@@ -41,5 +41,13 @@ module Packages
     def group?
       @project_or_group.is_a?(::Group)
     end
+
+    def project_packages
+      packages_for_project(@project_or_group)
+    end
+
+    def group_packages
+      packages_visible_to_user(@current_user, within_group: @project_or_group)
+    end
   end
 end

app/finders/packages/pypi/package_finder.rb

@@ -12,6 +12,16 @@ module Packages
       def packages
         base.pypi.has_version
       end
+
+      def group_packages
+        # PyPI finds packages without checking permissions.
+        # The package download endpoint uses obfuscation to secure the file
+        # instead of authentication. This is behavior the PyPI package
+        # manager defines and is not something GitLab controls.
+        ::Packages::Package.for_projects(
+          @project_or_group.all_projects.select(:id)
+        ).installable
+      end
     end
   end
 end

app/presenters/packages/pypi/package_presenter.rb

@@ -7,9 +7,9 @@ module Packages
     class PackagePresenter
       include API::Helpers::RelatedResourcesHelpers
 
-      def initialize(packages, project)
+      def initialize(packages, project_or_group)
         @packages = packages
-        @project = project
+        @project_or_group = project_or_group
       end
 
       # Returns the HTML body for PyPI simple API.
@@ -51,16 +51,27 @@ module Packages
       end
 
       def build_pypi_package_path(file)
-        expose_url(
-          api_v4_projects_packages_pypi_files_file_identifier_path(
-            {
-              id: @project.id,
+        params = {
+          id: @project_or_group.id,
           sha256: file.file_sha256,
           file_identifier: file.file_name
-            },
-            true
+        }
+
+        if project?
+          expose_url(
+            api_v4_projects_packages_pypi_files_file_identifier_path(
+              params, true
             )
           ) + "#sha256=#{file.file_sha256}"
+        elsif group?
+          expose_url(
+            api_v4_groups___packages_pypi_files_file_identifier_path(
+              params, true
+            )
+          ) + "#sha256=#{file.file_sha256}"
+        else
+          ''
+        end
       end
 
       def name
@@ -70,6 +81,14 @@ module Packages
       def escape(str)
         ERB::Util.html_escape(str)
       end
+
+      def project?
+        @project_or_group.is_a?(::Project)
+      end
+
+      def group?
+        @project_or_group.is_a?(::Group)
+      end
     end
   end
 end

doc/api/packages/pypi.md

@@ -20,11 +20,82 @@ These endpoints do not adhere to the standard API authentication methods.
 See the [PyPI package registry documentation](../../user/packages/pypi_repository/index.md)
 for details on which headers and token types are supported.
 
-## Download a package file
+## Download a package file from a group
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12.
+
+Download a PyPI package file. The [simple API](#group-level-simple-api-entry-point)
+normally supplies this URL.
+
+```plaintext
+GET groups/:id/packages/pypi/files/:sha256/:file_identifier
+```
+
+| Attribute         | Type   | Required | Description |
+| ----------------- | ------ | -------- | ----------- |
+| `id`              | string | yes      | The ID or full path of the group. |
+| `sha256`          | string | yes      | The PyPI package file's sha256 checksum. |
+| `file_identifier` | string | yes      | The PyPI package file's name. |
+
+```shell
+curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz"
+```
+
+To write the output to a file:
+
+```shell
+curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz" >> my.pypi.package-0.0.1.tar.gz
+```
+
+This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory.
+
+## Group level simple API entry point
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12.
+
+Returns the package descriptor as an HTML file:
+
+```plaintext
+GET groups/:id/packages/pypi/simple/:package_name
+```
+
+| Attribute      | Type   | Required | Description |
+| -------------- | ------ | -------- | ----------- |
+| `id`           | string | yes      | The ID or full path of the group. |
+| `package_name` | string | yes      | The name of the package. |
+
+```shell
+curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package"
+```
+
+Example response:
+
+```html
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Links for my.pypi.package</title>
+  </head>
+  <body>
+    <h1>Links for my.pypi.package</h1>
+    <a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1-py3-none-any.whl#sha256=5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff" data-requires-python="&gt;=3.6">my.pypi.package-0.0.1-py3-none-any.whl</a><br><a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/9s9w01b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2/my.pypi.package-0.0.1.tar.gz#sha256=9s9w011b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2" data-requires-python="&gt;=3.6">my.pypi.package-0.0.1.tar.gz</a><br>
+  </body>
+</html>
+```
+
+To write the output to a file:
+
+```shell
+curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package" >> simple.html
+```
+
+This writes the downloaded file to `simple.html` in the current directory.
+
+## Download a package file from a project
 
 > Introduced in GitLab 12.10.
 
-Download a PyPI package file. The [simple API](#simple-api-entry-point)
+Download a PyPI package file. The [simple API](#project-level-simple-api-entry-point)
 normally supplies this URL.
 
 ```plaintext
@@ -49,7 +120,7 @@ curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v
 
 This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory.
 
-## Simple API entry point
+## Project-level simple API entry point
 
 > Introduced in GitLab 12.10.
 

doc/user/packages/pypi_repository/index.md

@@ -316,6 +316,8 @@ more than once, a `404 Bad Request` error occurs.
 
 ## Install a PyPI package
 
+### Install from the project level
+
 To install the latest version of a package, use the following command:
 
 ```shell
@@ -350,6 +352,33 @@ Installing collected packages: mypypipackage
 Successfully installed mypypipackage-0.0.1
 ```
 
+### Install from the group level
+
+To install the latest version of a package from a group, use the following command:
+
+```shell
+pip install --index-url https://<personal_access_token_name>:<personal_access_token>@gitlab.example.com/api/v4/groups/<group_id>/packages/pypi/simple --no-deps <package_name>
+```
+
+In this command:
+
+- `<package_name>` is the package name.
+- `<personal_access_token_name>` is a personal access token name with the `read_api` scope.
+- `<personal_access_token>` is a personal access token with the `read_api` scope.
+- `<group_id>` is the group ID.
+
+In these commands, you can use `--extra-index-url` instead of `--index-url`. However, using
+`--extra-index-url` makes you vulnerable to dependency confusion attacks because it checks the PyPi
+repository for the package before it checks the custom repository. `--extra-index-url` adds the
+provided URL as an additional registry which the client checks if the package is present.
+`--index-url` tells the client to check for the package at the provided URL only.
+
+If you're following the guide and want to install the `MyPyPiPackage` package, you can run:
+
+```shell
+pip install mypypipackage --no-deps --index-url https://<personal_access_token_name>:<personal_access_token>@gitlab.example.com/api/v4/groups/<your_group_id>/packages/pypi/simple
+```
+
 ### Package names
 
 GitLab looks for packages that use

lib/api/helpers/packages/basic_auth_helpers.rb

@@ -22,6 +22,14 @@ module API
           unauthorized_user_project || not_found!
         end
 
+        def unauthorized_user_group
+          @unauthorized_user_group ||= find_group(params[:id])
+        end
+
+        def unauthorized_user_group!
+          unauthorized_user_group || not_found!
+        end
+
         def authorized_user_project
           @authorized_user_project ||= authorized_project_find!
         end

lib/api/pypi_packages.rb

@@ -28,6 +28,73 @@ module API
       require_packages_enabled!
     end
 
+    helpers do
+      params :package_download do
+        requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true
+        requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
+      end
+
+      params :package_name do
+        requires :package_name, type: String, file_path: true, desc: 'The PyPi package name'
+      end
+    end
+
+    params do
+      requires :id, type: Integer, desc: 'The ID of a group'
+    end
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      after_validation do
+        unauthorized_user_group!
+      end
+
+      namespace ':id/-/packages/pypi' do
+        params do
+          use :package_download
+        end
+
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+        get 'files/:sha256/*file_identifier' do
+          group = unauthorized_user_group!
+
+          filename = "#{params[:file_identifier]}.#{params[:format]}"
+          package = Packages::Pypi::PackageFinder.new(current_user, group, { filename: filename, sha256: params[:sha256] }).execute
+          package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute
+
+          track_package_event('pull_package', :pypi)
+
+          present_carrierwave_file!(package_file.file, supports_direct_download: true)
+        end
+
+        desc 'The PyPi Simple Endpoint' do
+          detail 'This feature was introduced in GitLab 12.10'
+        end
+
+        params do
+          use :package_name
+        end
+
+        # An Api entry point but returns an HTML file instead of JSON.
+        # PyPi simple API returns the package descriptor as a simple HTML file.
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+        get 'simple/*package_name', format: :txt do
+          group = find_authorized_group!
+          authorize_read_package!(group)
+
+          track_package_event('list_package', :pypi)
+
+          packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute!
+          presenter = ::Packages::Pypi::PackagePresenter.new(packages, group)
+
+          # Adjusts grape output format
+          # to be HTML
+          content_type "text/html; charset=utf-8"
+          env['api.format'] = :binary
+
+          body presenter.body
+        end
+      end
+    end
+
     params do
       requires :id, type: Integer, desc: 'The ID of a project'
     end
@@ -43,8 +110,7 @@ module API
         end
 
         params do
-          requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true
-          requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
+          use :package_download
         end
 
         route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
@@ -65,7 +131,7 @@ module API
         end
 
         params do
-          requires :package_name, type: String, file_path: true, desc: 'The PyPi package name'
+          use :package_name
         end
 
         # An Api entry point but returns an HTML file instead of JSON.

spec/finders/packages/pypi/package_finder_spec.rb

@@ -31,15 +31,7 @@ RSpec.describe Packages::Pypi::PackageFinder do
     context 'within a group' do
       let(:scope) { group }
 
-      it { expect { subject }.to raise_error(ActiveRecord::RecordNotFound) }
-
-      context 'user with access' do
-        before do
-          project.add_developer(user)
-        end
-
       it { is_expected.to eq(package2) }
     end
   end
-  end
 end

spec/presenters/packages/pypi/package_presenter_spec.rb

@@ -5,21 +5,31 @@ require 'spec_helper'
 RSpec.describe ::Packages::Pypi::PackagePresenter do
   using RSpec::Parameterized::TableSyntax
 
-  let_it_be(:project) { create(:project) }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, group: group) }
   let_it_be(:package_name) { 'sample-project' }
   let_it_be(:package1) { create(:pypi_package, project: project, name: package_name, version: '1.0.0') }
   let_it_be(:package2) { create(:pypi_package, project: project, name: package_name, version: '2.0.0') }
 
   let(:packages) { [package1, package2] }
-  let(:presenter) { described_class.new(packages, project) }
 
-  describe '#body' do
-    subject { presenter.body}
-
-    shared_examples_for "pypi package presenter" do
   let(:file) { package.package_files.first }
   let(:filename) { file.file_name }
-      let(:expected_file) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" }
+
+  subject(:presenter) { described_class.new(packages, project_or_group).body}
+
+  describe '#body' do
+    shared_examples_for "pypi package presenter" do
+      where(:version, :expected_version, :with_package1) do
+        '>=2.7'                        | '&gt;=2.7'                                        | true
+        '"><script>alert(1)</script>'  | '&quot;&gt;&lt;script&gt;alert(1)&lt;/script&gt;' | true
+        '>=2.7, !=3.0'                 | '&gt;=2.7, !=3.0'                                 | false
+      end
+
+      with_them do
+        let(:python_version) { version }
+        let(:expected_python_version) { expected_version }
+        let(:package) { with_package1 ? package1 : package2 }
 
         before do
           package.pypi_metadatum.required_python = python_version
@@ -27,23 +37,20 @@ RSpec.describe ::Packages::Pypi::PackagePresenter do
 
         it { is_expected.to include expected_file }
       end
-
-    it_behaves_like "pypi package presenter" do
-      let(:python_version) { '>=2.7' }
-      let(:expected_python_version) { '&gt;=2.7' }
-      let(:package) { package1 }
     end
 
-    it_behaves_like "pypi package presenter" do
-      let(:python_version) { '"><script>alert(1)</script>' }
-      let(:expected_python_version) { '&quot;&gt;&lt;script&gt;alert(1)&lt;/script&gt;' }
-      let(:package) { package1 }
+    context 'for project' do
+      let(:project_or_group) { project }
+      let(:expected_file) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" }
+
+      it_behaves_like 'pypi package presenter'
     end
 
-    it_behaves_like "pypi package presenter" do
-      let(:python_version) { '>=2.7, !=3.0' }
-      let(:expected_python_version) { '&gt;=2.7, !=3.0' }
-      let(:package) { package2 }
+    context 'for group' do
+      let(:project_or_group) { group }
+      let(:expected_file) { "<a href=\"http://localhost/api/v4/groups/#{group.id}/-/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" }
+
+      it_behaves_like 'pypi package presenter'
     end
   end
 end

spec/support/shared_examples/requests/api/pypi_packages_shared_examples.rb

@@ -110,6 +110,7 @@ RSpec.shared_examples 'PyPI package versions' do |user_type, status, add_member
   context "for user type #{user_type}" do
     before do
       project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+      group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
     end
 
     it 'returns the package listing' do
@@ -127,6 +128,7 @@ RSpec.shared_examples 'PyPI package download' do |user_type, status, add_member
   context "for user type #{user_type}" do
     before do
       project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+      group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
     end
 
     it 'returns the package listing' do
@@ -144,16 +146,14 @@ RSpec.shared_examples 'process PyPI api request' do |user_type, status, add_memb
   context "for user type #{user_type}" do
     before do
       project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+      group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
     end
 
     it_behaves_like 'returning response status', status
   end
 end
 
-RSpec.shared_examples 'rejects PyPI access with unknown project id' do
-  context 'with an unknown project' do
-    let(:project) { OpenStruct.new(id: 1234567890) }
-
+RSpec.shared_examples 'unknown PyPI scope id' do
   context 'as anonymous' do
     it_behaves_like 'process PyPI api request', :anonymous, :not_found
   end
@@ -163,5 +163,167 @@ RSpec.shared_examples 'rejects PyPI access with unknown project id' do
 
     it_behaves_like 'process PyPI api request', :anonymous, :not_found
   end
+end
+
+RSpec.shared_examples 'rejects PyPI access with unknown project id' do
+  context 'with an unknown project' do
+    let(:project) { OpenStruct.new(id: 1234567890) }
+
+    it_behaves_like 'unknown PyPI scope id'
+  end
+end
+
+RSpec.shared_examples 'rejects PyPI access with unknown group id' do
+  context 'with an unknown project' do
+    let(:group) { OpenStruct.new(id: 1234567890) }
+
+    it_behaves_like 'unknown PyPI scope id'
+  end
+end
+
+RSpec.shared_examples 'pypi simple API endpoint' do
+  using RSpec::Parameterized::TableSyntax
+
+  context 'with valid project' do
+    where(:visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
+      :public  | :developer  | true  | true  | 'PyPI package versions' | :success
+      :public  | :guest      | true  | true  | 'PyPI package versions' | :success
+      :public  | :developer  | true  | false | 'PyPI package versions' | :success
+      :public  | :guest      | true  | false | 'PyPI package versions' | :success
+      :public  | :developer  | false | true  | 'PyPI package versions' | :success
+      :public  | :guest      | false | true  | 'PyPI package versions' | :success
+      :public  | :developer  | false | false | 'PyPI package versions' | :success
+      :public  | :guest      | false | false | 'PyPI package versions' | :success
+      :public  | :anonymous  | false | true  | 'PyPI package versions' | :success
+      :private | :developer  | true  | true  | 'PyPI package versions' | :success
+      :private | :guest      | true  | true  | 'process PyPI api request' | :forbidden
+      :private | :developer  | true  | false | 'process PyPI api request' | :unauthorized
+      :private | :guest      | true  | false | 'process PyPI api request' | :unauthorized
+      :private | :developer  | false | true  | 'process PyPI api request' | :not_found
+      :private | :guest      | false | true  | 'process PyPI api request' | :not_found
+      :private | :developer  | false | false | 'process PyPI api request' | :unauthorized
+      :private | :guest      | false | false | 'process PyPI api request' | :unauthorized
+      :private | :anonymous  | false | true  | 'process PyPI api request' | :unauthorized
+    end
+
+    with_them do
+      let(:token) { user_token ? personal_access_token.token : 'wrong' }
+      let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
+
+      before do
+        project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
+        group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
+      end
+
+      it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
+    end
+  end
+
+  context 'with a normalized package name' do
+    let_it_be(:package) { create(:pypi_package, project: project, name: 'my.package') }
+
+    let(:url) { "/projects/#{project.id}/packages/pypi/simple/my-package" }
+    let(:headers) { basic_auth_header(user.username, personal_access_token.token) }
+
+    it_behaves_like 'PyPI package versions', :developer, :success
+  end
+end
+
+RSpec.shared_examples 'pypi file download endpoint' do
+  using RSpec::Parameterized::TableSyntax
+
+  context 'with valid project' do
+    where(:visibility_level, :user_role, :member, :user_token) do
+      :public  | :developer  | true  | true
+      :public  | :guest      | true  | true
+      :public  | :developer  | true  | false
+      :public  | :guest      | true  | false
+      :public  | :developer  | false | true
+      :public  | :guest      | false | true
+      :public  | :developer  | false | false
+      :public  | :guest      | false | false
+      :public  | :anonymous  | false | true
+      :private | :developer  | true  | true
+      :private | :guest      | true  | true
+      :private | :developer  | true  | false
+      :private | :guest      | true  | false
+      :private | :developer  | false | true
+      :private | :guest      | false | true
+      :private | :developer  | false | false
+      :private | :guest      | false | false
+      :private | :anonymous  | false | true
+    end
+
+    with_them do
+      let(:token) { user_token ? personal_access_token.token : 'wrong' }
+      let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
+
+      before do
+        project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
+        group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
+      end
+
+      it_behaves_like 'PyPI package download', params[:user_role], :success, params[:member]
+    end
+  end
+
+  context 'with deploy token headers' do
+    let(:headers) { basic_auth_header(deploy_token.username, deploy_token.token) }
+
+    context 'valid token' do
+      it_behaves_like 'returning response status', :success
+    end
+
+    context 'invalid token' do
+      let(:headers) { basic_auth_header('foo', 'bar') }
+
+      it_behaves_like 'returning response status', :success
+    end
+  end
+
+  context 'with job token headers' do
+    let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, job.token) }
+
+    context 'valid token' do
+      it_behaves_like 'returning response status', :success
+    end
+
+    context 'invalid token' do
+      let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, 'bar') }
+
+      it_behaves_like 'returning response status', :unauthorized
+    end
+
+    context 'invalid user' do
+      let(:headers) { basic_auth_header('foo', job.token) }
+
+      it_behaves_like 'returning response status', :success
+    end
+  end
+end
+
+RSpec.shared_examples 'a pypi user namespace endpoint' do
+  using RSpec::Parameterized::TableSyntax
+
+  # only group namespaces are supported at this time
+  where(:visibility_level, :user_role, :expected_status) do
+    :public  | :owner     | :not_found
+    :private | :owner     | :not_found
+    :public  | :external  | :not_found
+    :private | :external  | :not_found
+    :public  | :anonymous | :not_found
+    :private | :anonymous | :not_found
+  end
+
+  with_them do
+    let_it_be_with_reload(:group) { create(:namespace) }
+    let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
+
+    before do
+      group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
+      group.update_column(:owner_id, user.id) if user_role == :owner
+    end
+
+    it_behaves_like 'returning response status', params[:expected_status]
   end
 end