Commit 70f1921d authored by Steve Abrams's avatar Steve Abrams

Group level PyPI packages

Adds endpoints to the PyPI package manager API
to allow consumption of packages at the group
level.

Changlog: added
parent 50022004
......@@ -26,9 +26,9 @@ module Packages
def base
if project?
packages_for_project(@project_or_group)
project_packages
elsif group?
packages_visible_to_user(@current_user, within_group: @project_or_group)
group_packages
else
::Packages::Package.none
end
......@@ -41,5 +41,13 @@ module Packages
def group?
@project_or_group.is_a?(::Group)
end
def project_packages
packages_for_project(@project_or_group)
end
def group_packages
packages_visible_to_user(@current_user, within_group: @project_or_group)
end
end
end
......@@ -12,6 +12,16 @@ module Packages
def packages
base.pypi.has_version
end
def group_packages
# PyPI finds packages without checking permissions.
# The package download endpoint uses obfuscation to secure the file
# instead of authentication. This is behavior the PyPI package
# manager defines and is not something GitLab controls.
::Packages::Package.for_projects(
@project_or_group.all_projects.select(:id)
).installable
end
end
end
end
......@@ -7,9 +7,9 @@ module Packages
class PackagePresenter
include API::Helpers::RelatedResourcesHelpers
def initialize(packages, project)
def initialize(packages, project_or_group)
@packages = packages
@project = project
@project_or_group = project_or_group
end
# Returns the HTML body for PyPI simple API.
......@@ -51,16 +51,27 @@ module Packages
end
def build_pypi_package_path(file)
expose_url(
api_v4_projects_packages_pypi_files_file_identifier_path(
{
id: @project.id,
sha256: file.file_sha256,
file_identifier: file.file_name
},
true
)
) + "#sha256=#{file.file_sha256}"
params = {
id: @project_or_group.id,
sha256: file.file_sha256,
file_identifier: file.file_name
}
if project?
expose_url(
api_v4_projects_packages_pypi_files_file_identifier_path(
params, true
)
) + "#sha256=#{file.file_sha256}"
elsif group?
expose_url(
api_v4_groups___packages_pypi_files_file_identifier_path(
params, true
)
) + "#sha256=#{file.file_sha256}"
else
''
end
end
def name
......@@ -70,6 +81,14 @@ module Packages
def escape(str)
ERB::Util.html_escape(str)
end
def project?
@project_or_group.is_a?(::Project)
end
def group?
@project_or_group.is_a?(::Group)
end
end
end
end
......@@ -20,11 +20,82 @@ These endpoints do not adhere to the standard API authentication methods.
See the [PyPI package registry documentation](../../user/packages/pypi_repository/index.md)
for details on which headers and token types are supported.
## Download a package file
## Download a package file from a group
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12.
Download a PyPI package file. The [simple API](#group-level-simple-api-entry-point)
normally supplies this URL.
```plaintext
GET groups/:id/packages/pypi/files/:sha256/:file_identifier
```
| Attribute | Type | Required | Description |
| ----------------- | ------ | -------- | ----------- |
| `id` | string | yes | The ID or full path of the group. |
| `sha256` | string | yes | The PyPI package file's sha256 checksum. |
| `file_identifier` | string | yes | The PyPI package file's name. |
```shell
curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz"
```
To write the output to a file:
```shell
curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz" >> my.pypi.package-0.0.1.tar.gz
```
This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory.
## Group level simple API entry point
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12.
Returns the package descriptor as an HTML file:
```plaintext
GET groups/:id/packages/pypi/simple/:package_name
```
| Attribute | Type | Required | Description |
| -------------- | ------ | -------- | ----------- |
| `id` | string | yes | The ID or full path of the group. |
| `package_name` | string | yes | The name of the package. |
```shell
curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package"
```
Example response:
```html
<!DOCTYPE html>
<html>
<head>
<title>Links for my.pypi.package</title>
</head>
<body>
<h1>Links for my.pypi.package</h1>
<a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1-py3-none-any.whl#sha256=5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff" data-requires-python="&gt;=3.6">my.pypi.package-0.0.1-py3-none-any.whl</a><br><a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/9s9w01b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2/my.pypi.package-0.0.1.tar.gz#sha256=9s9w011b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2" data-requires-python="&gt;=3.6">my.pypi.package-0.0.1.tar.gz</a><br>
</body>
</html>
```
To write the output to a file:
```shell
curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package" >> simple.html
```
This writes the downloaded file to `simple.html` in the current directory.
## Download a package file from a project
> Introduced in GitLab 12.10.
Download a PyPI package file. The [simple API](#simple-api-entry-point)
Download a PyPI package file. The [simple API](#project-level-simple-api-entry-point)
normally supplies this URL.
```plaintext
......@@ -49,7 +120,7 @@ curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v
This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory.
## Simple API entry point
## Project-level simple API entry point
> Introduced in GitLab 12.10.
......
......@@ -316,6 +316,8 @@ more than once, a `404 Bad Request` error occurs.
## Install a PyPI package
### Install from the project level
To install the latest version of a package, use the following command:
```shell
......@@ -350,6 +352,33 @@ Installing collected packages: mypypipackage
Successfully installed mypypipackage-0.0.1
```
### Install from the group level
To install the latest version of a package from a group, use the following command:
```shell
pip install --index-url https://<personal_access_token_name>:<personal_access_token>@gitlab.example.com/api/v4/groups/<group_id>/packages/pypi/simple --no-deps <package_name>
```
In this command:
- `<package_name>` is the package name.
- `<personal_access_token_name>` is a personal access token name with the `read_api` scope.
- `<personal_access_token>` is a personal access token with the `read_api` scope.
- `<group_id>` is the group ID.
In these commands, you can use `--extra-index-url` instead of `--index-url`. However, using
`--extra-index-url` makes you vulnerable to dependency confusion attacks because it checks the PyPi
repository for the package before it checks the custom repository. `--extra-index-url` adds the
provided URL as an additional registry which the client checks if the package is present.
`--index-url` tells the client to check for the package at the provided URL only.
If you're following the guide and want to install the `MyPyPiPackage` package, you can run:
```shell
pip install mypypipackage --no-deps --index-url https://<personal_access_token_name>:<personal_access_token>@gitlab.example.com/api/v4/groups/<your_group_id>/packages/pypi/simple
```
### Package names
GitLab looks for packages that use
......
......@@ -22,6 +22,14 @@ module API
unauthorized_user_project || not_found!
end
def unauthorized_user_group
@unauthorized_user_group ||= find_group(params[:id])
end
def unauthorized_user_group!
unauthorized_user_group || not_found!
end
def authorized_user_project
@authorized_user_project ||= authorized_project_find!
end
......
......@@ -28,6 +28,73 @@ module API
require_packages_enabled!
end
helpers do
params :package_download do
requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true
requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
end
params :package_name do
requires :package_name, type: String, file_path: true, desc: 'The PyPi package name'
end
end
params do
requires :id, type: Integer, desc: 'The ID of a group'
end
resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
after_validation do
unauthorized_user_group!
end
namespace ':id/-/packages/pypi' do
params do
use :package_download
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get 'files/:sha256/*file_identifier' do
group = unauthorized_user_group!
filename = "#{params[:file_identifier]}.#{params[:format]}"
package = Packages::Pypi::PackageFinder.new(current_user, group, { filename: filename, sha256: params[:sha256] }).execute
package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute
track_package_event('pull_package', :pypi)
present_carrierwave_file!(package_file.file, supports_direct_download: true)
end
desc 'The PyPi Simple Endpoint' do
detail 'This feature was introduced in GitLab 12.10'
end
params do
use :package_name
end
# An Api entry point but returns an HTML file instead of JSON.
# PyPi simple API returns the package descriptor as a simple HTML file.
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get 'simple/*package_name', format: :txt do
group = find_authorized_group!
authorize_read_package!(group)
track_package_event('list_package', :pypi)
packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute!
presenter = ::Packages::Pypi::PackagePresenter.new(packages, group)
# Adjusts grape output format
# to be HTML
content_type "text/html; charset=utf-8"
env['api.format'] = :binary
body presenter.body
end
end
end
params do
requires :id, type: Integer, desc: 'The ID of a project'
end
......@@ -43,8 +110,7 @@ module API
end
params do
requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true
requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
use :package_download
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
......@@ -65,7 +131,7 @@ module API
end
params do
requires :package_name, type: String, file_path: true, desc: 'The PyPi package name'
use :package_name
end
# An Api entry point but returns an HTML file instead of JSON.
......
......@@ -31,15 +31,7 @@ RSpec.describe Packages::Pypi::PackageFinder do
context 'within a group' do
let(:scope) { group }
it { expect { subject }.to raise_error(ActiveRecord::RecordNotFound) }
context 'user with access' do
before do
project.add_developer(user)
end
it { is_expected.to eq(package2) }
end
it { is_expected.to eq(package2) }
end
end
end
......@@ -5,45 +5,52 @@ require 'spec_helper'
RSpec.describe ::Packages::Pypi::PackagePresenter do
using RSpec::Parameterized::TableSyntax
let_it_be(:project) { create(:project) }
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, group: group) }
let_it_be(:package_name) { 'sample-project' }
let_it_be(:package1) { create(:pypi_package, project: project, name: package_name, version: '1.0.0') }
let_it_be(:package2) { create(:pypi_package, project: project, name: package_name, version: '2.0.0') }
let(:packages) { [package1, package2] }
let(:presenter) { described_class.new(packages, project) }
describe '#body' do
subject { presenter.body}
let(:file) { package.package_files.first }
let(:filename) { file.file_name }
shared_examples_for "pypi package presenter" do
let(:file) { package.package_files.first }
let(:filename) { file.file_name }
let(:expected_file) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" }
subject(:presenter) { described_class.new(packages, project_or_group).body}
before do
package.pypi_metadatum.required_python = python_version
describe '#body' do
shared_examples_for "pypi package presenter" do
where(:version, :expected_version, :with_package1) do
'>=2.7' | '&gt;=2.7' | true
'"><script>alert(1)</script>' | '&quot;&gt;&lt;script&gt;alert(1)&lt;/script&gt;' | true
'>=2.7, !=3.0' | '&gt;=2.7, !=3.0' | false
end
it { is_expected.to include expected_file }
end
with_them do
let(:python_version) { version }
let(:expected_python_version) { expected_version }
let(:package) { with_package1 ? package1 : package2 }
it_behaves_like "pypi package presenter" do
let(:python_version) { '>=2.7' }
let(:expected_python_version) { '&gt;=2.7' }
let(:package) { package1 }
before do
package.pypi_metadatum.required_python = python_version
end
it { is_expected.to include expected_file }
end
end
it_behaves_like "pypi package presenter" do
let(:python_version) { '"><script>alert(1)</script>' }
let(:expected_python_version) { '&quot;&gt;&lt;script&gt;alert(1)&lt;/script&gt;' }
let(:package) { package1 }
context 'for project' do
let(:project_or_group) { project }
let(:expected_file) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" }
it_behaves_like 'pypi package presenter'
end
it_behaves_like "pypi package presenter" do
let(:python_version) { '>=2.7, !=3.0' }
let(:expected_python_version) { '&gt;=2.7, !=3.0' }
let(:package) { package2 }
context 'for group' do
let(:project_or_group) { group }
let(:expected_file) { "<a href=\"http://localhost/api/v4/groups/#{group.id}/-/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" }
it_behaves_like 'pypi package presenter'
end
end
end
This diff is collapsed.
......@@ -110,6 +110,7 @@ RSpec.shared_examples 'PyPI package versions' do |user_type, status, add_member
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it 'returns the package listing' do
......@@ -127,6 +128,7 @@ RSpec.shared_examples 'PyPI package download' do |user_type, status, add_member
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it 'returns the package listing' do
......@@ -144,24 +146,184 @@ RSpec.shared_examples 'process PyPI api request' do |user_type, status, add_memb
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it_behaves_like 'returning response status', status
end
end
RSpec.shared_examples 'unknown PyPI scope id' do
context 'as anonymous' do
it_behaves_like 'process PyPI api request', :anonymous, :not_found
end
context 'as authenticated user' do
subject { get api(url), headers: basic_auth_header(user.username, personal_access_token.token) }
it_behaves_like 'process PyPI api request', :anonymous, :not_found
end
end
RSpec.shared_examples 'rejects PyPI access with unknown project id' do
context 'with an unknown project' do
let(:project) { OpenStruct.new(id: 1234567890) }
context 'as anonymous' do
it_behaves_like 'process PyPI api request', :anonymous, :not_found
it_behaves_like 'unknown PyPI scope id'
end
end
RSpec.shared_examples 'rejects PyPI access with unknown group id' do
context 'with an unknown project' do
let(:group) { OpenStruct.new(id: 1234567890) }
it_behaves_like 'unknown PyPI scope id'
end
end
RSpec.shared_examples 'pypi simple API endpoint' do
using RSpec::Parameterized::TableSyntax
context 'with valid project' do
where(:visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
:public | :developer | true | true | 'PyPI package versions' | :success
:public | :guest | true | true | 'PyPI package versions' | :success
:public | :developer | true | false | 'PyPI package versions' | :success
:public | :guest | true | false | 'PyPI package versions' | :success
:public | :developer | false | true | 'PyPI package versions' | :success
:public | :guest | false | true | 'PyPI package versions' | :success
:public | :developer | false | false | 'PyPI package versions' | :success
:public | :guest | false | false | 'PyPI package versions' | :success
:public | :anonymous | false | true | 'PyPI package versions' | :success
:private | :developer | true | true | 'PyPI package versions' | :success
:private | :guest | true | true | 'process PyPI api request' | :forbidden
:private | :developer | true | false | 'process PyPI api request' | :unauthorized
:private | :guest | true | false | 'process PyPI api request' | :unauthorized
:private | :developer | false | true | 'process PyPI api request' | :not_found
:private | :guest | false | true | 'process PyPI api request' | :not_found
:private | :developer | false | false | 'process PyPI api request' | :unauthorized
:private | :guest | false | false | 'process PyPI api request' | :unauthorized
:private | :anonymous | false | true | 'process PyPI api request' | :unauthorized
end
context 'as authenticated user' do
subject { get api(url), headers: basic_auth_header(user.username, personal_access_token.token) }
with_them do
let(:token) { user_token ? personal_access_token.token : 'wrong' }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
it_behaves_like 'process PyPI api request', :anonymous, :not_found
before do
project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
end
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
end
end
context 'with a normalized package name' do
let_it_be(:package) { create(:pypi_package, project: project, name: 'my.package') }
let(:url) { "/projects/#{project.id}/packages/pypi/simple/my-package" }
let(:headers) { basic_auth_header(user.username, personal_access_token.token) }
it_behaves_like 'PyPI package versions', :developer, :success
end
end
RSpec.shared_examples 'pypi file download endpoint' do
using RSpec::Parameterized::TableSyntax
context 'with valid project' do
where(:visibility_level, :user_role, :member, :user_token) do
:public | :developer | true | true
:public | :guest | true | true
:public | :developer | true | false
:public | :guest | true | false
:public | :developer | false | true
:public | :guest | false | true
:public | :developer | false | false
:public | :guest | false | false
:public | :anonymous | false | true
:private | :developer | true | true
:private | :guest | true | true
:private | :developer | true | false
:private | :guest | true | false
:private | :developer | false | true
:private | :guest | false | true
:private | :developer | false | false
:private | :guest | false | false
:private | :anonymous | false | true
end
with_them do
let(:token) { user_token ? personal_access_token.token : 'wrong' }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
before do
project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
end
it_behaves_like 'PyPI package download', params[:user_role], :success, params[:member]
end
end
context 'with deploy token headers' do
let(:headers) { basic_auth_header(deploy_token.username, deploy_token.token) }
context 'valid token' do
it_behaves_like 'returning response status', :success
end
context 'invalid token' do
let(:headers) { basic_auth_header('foo', 'bar') }
it_behaves_like 'returning response status', :success
end
end
context 'with job token headers' do
let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, job.token) }
context 'valid token' do
it_behaves_like 'returning response status', :success
end
context 'invalid token' do
let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, 'bar') }
it_behaves_like 'returning response status', :unauthorized
end
context 'invalid user' do
let(:headers) { basic_auth_header('foo', job.token) }
it_behaves_like 'returning response status', :success
end
end
end
RSpec.shared_examples 'a pypi user namespace endpoint' do
using RSpec::Parameterized::TableSyntax
# only group namespaces are supported at this time
where(:visibility_level, :user_role, :expected_status) do
:public | :owner | :not_found
:private | :owner | :not_found
:public | :external | :not_found
:private | :external | :not_found
:public | :anonymous | :not_found
:private | :anonymous | :not_found
end
with_them do
let_it_be_with_reload(:group) { create(:namespace) }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
before do
group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
group.update_column(:owner_id, user.id) if user_role == :owner
end
it_behaves_like 'returning response status', params[:expected_status]
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment