Commit 0c7ef335 authored by David Fernandez's avatar David Fernandez

Merge branch '225545-pypi-group-api' into 'master'

PyPi group-level package API

See merge request gitlab-org/gitlab!61189
parents ab2d8ec0 70f1921d
......@@ -26,9 +26,9 @@ module Packages
def base
if project?
packages_for_project(@project_or_group)
project_packages
elsif group?
packages_visible_to_user(@current_user, within_group: @project_or_group)
group_packages
else
::Packages::Package.none
end
......@@ -41,5 +41,13 @@ module Packages
def group?
@project_or_group.is_a?(::Group)
end
def project_packages
packages_for_project(@project_or_group)
end
def group_packages
packages_visible_to_user(@current_user, within_group: @project_or_group)
end
end
end
......@@ -12,6 +12,16 @@ module Packages
def packages
base.pypi.has_version
end
def group_packages
# PyPI finds packages without checking permissions.
# The package download endpoint uses obfuscation to secure the file
# instead of authentication. This is behavior the PyPI package
# manager defines and is not something GitLab controls.
::Packages::Package.for_projects(
@project_or_group.all_projects.select(:id)
).installable
end
end
end
end
......@@ -7,9 +7,9 @@ module Packages
class PackagePresenter
include API::Helpers::RelatedResourcesHelpers
def initialize(packages, project)
def initialize(packages, project_or_group)
@packages = packages
@project = project
@project_or_group = project_or_group
end
# Returns the HTML body for PyPI simple API.
......@@ -51,16 +51,27 @@ module Packages
end
def build_pypi_package_path(file)
expose_url(
api_v4_projects_packages_pypi_files_file_identifier_path(
{
id: @project.id,
sha256: file.file_sha256,
file_identifier: file.file_name
},
true
)
) + "#sha256=#{file.file_sha256}"
params = {
id: @project_or_group.id,
sha256: file.file_sha256,
file_identifier: file.file_name
}
if project?
expose_url(
api_v4_projects_packages_pypi_files_file_identifier_path(
params, true
)
) + "#sha256=#{file.file_sha256}"
elsif group?
expose_url(
api_v4_groups___packages_pypi_files_file_identifier_path(
params, true
)
) + "#sha256=#{file.file_sha256}"
else
''
end
end
def name
......@@ -70,6 +81,14 @@ module Packages
def escape(str)
ERB::Util.html_escape(str)
end
def project?
@project_or_group.is_a?(::Project)
end
def group?
@project_or_group.is_a?(::Group)
end
end
end
end
......@@ -20,11 +20,82 @@ These endpoints do not adhere to the standard API authentication methods.
See the [PyPI package registry documentation](../../user/packages/pypi_repository/index.md)
for details on which headers and token types are supported.
## Download a package file
## Download a package file from a group
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12.
Download a PyPI package file. The [simple API](#group-level-simple-api-entry-point)
normally supplies this URL.
```plaintext
GET groups/:id/packages/pypi/files/:sha256/:file_identifier
```
| Attribute | Type | Required | Description |
| ----------------- | ------ | -------- | ----------- |
| `id` | string | yes | The ID or full path of the group. |
| `sha256` | string | yes | The PyPI package file's sha256 checksum. |
| `file_identifier` | string | yes | The PyPI package file's name. |
```shell
curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz"
```
To write the output to a file:
```shell
curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz" >> my.pypi.package-0.0.1.tar.gz
```
This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory.
## Group level simple API entry point
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12.
Returns the package descriptor as an HTML file:
```plaintext
GET groups/:id/packages/pypi/simple/:package_name
```
| Attribute | Type | Required | Description |
| -------------- | ------ | -------- | ----------- |
| `id` | string | yes | The ID or full path of the group. |
| `package_name` | string | yes | The name of the package. |
```shell
curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package"
```
Example response:
```html
<!DOCTYPE html>
<html>
<head>
<title>Links for my.pypi.package</title>
</head>
<body>
<h1>Links for my.pypi.package</h1>
<a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1-py3-none-any.whl#sha256=5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff" data-requires-python="&gt;=3.6">my.pypi.package-0.0.1-py3-none-any.whl</a><br><a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/9s9w01b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2/my.pypi.package-0.0.1.tar.gz#sha256=9s9w011b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2" data-requires-python="&gt;=3.6">my.pypi.package-0.0.1.tar.gz</a><br>
</body>
</html>
```
To write the output to a file:
```shell
curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package" >> simple.html
```
This writes the downloaded file to `simple.html` in the current directory.
## Download a package file from a project
> Introduced in GitLab 12.10.
Download a PyPI package file. The [simple API](#simple-api-entry-point)
Download a PyPI package file. The [simple API](#project-level-simple-api-entry-point)
normally supplies this URL.
```plaintext
......@@ -49,7 +120,7 @@ curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v
This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory.
## Simple API entry point
## Project-level simple API entry point
> Introduced in GitLab 12.10.
......
......@@ -316,6 +316,8 @@ more than once, a `404 Bad Request` error occurs.
## Install a PyPI package
### Install from the project level
To install the latest version of a package, use the following command:
```shell
......@@ -350,6 +352,33 @@ Installing collected packages: mypypipackage
Successfully installed mypypipackage-0.0.1
```
### Install from the group level
To install the latest version of a package from a group, use the following command:
```shell
pip install --index-url https://<personal_access_token_name>:<personal_access_token>@gitlab.example.com/api/v4/groups/<group_id>/packages/pypi/simple --no-deps <package_name>
```
In this command:
- `<package_name>` is the package name.
- `<personal_access_token_name>` is a personal access token name with the `read_api` scope.
- `<personal_access_token>` is a personal access token with the `read_api` scope.
- `<group_id>` is the group ID.
In these commands, you can use `--extra-index-url` instead of `--index-url`. However, using
`--extra-index-url` makes you vulnerable to dependency confusion attacks because it checks the PyPi
repository for the package before it checks the custom repository. `--extra-index-url` adds the
provided URL as an additional registry which the client checks if the package is present.
`--index-url` tells the client to check for the package at the provided URL only.
If you're following the guide and want to install the `MyPyPiPackage` package, you can run:
```shell
pip install mypypipackage --no-deps --index-url https://<personal_access_token_name>:<personal_access_token>@gitlab.example.com/api/v4/groups/<your_group_id>/packages/pypi/simple
```
### Package names
GitLab looks for packages that use
......
......@@ -22,6 +22,14 @@ module API
unauthorized_user_project || not_found!
end
def unauthorized_user_group
@unauthorized_user_group ||= find_group(params[:id])
end
def unauthorized_user_group!
unauthorized_user_group || not_found!
end
def authorized_user_project
@authorized_user_project ||= authorized_project_find!
end
......
......@@ -28,6 +28,73 @@ module API
require_packages_enabled!
end
helpers do
params :package_download do
requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true
requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
end
params :package_name do
requires :package_name, type: String, file_path: true, desc: 'The PyPi package name'
end
end
params do
requires :id, type: Integer, desc: 'The ID of a group'
end
resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
after_validation do
unauthorized_user_group!
end
namespace ':id/-/packages/pypi' do
params do
use :package_download
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get 'files/:sha256/*file_identifier' do
group = unauthorized_user_group!
filename = "#{params[:file_identifier]}.#{params[:format]}"
package = Packages::Pypi::PackageFinder.new(current_user, group, { filename: filename, sha256: params[:sha256] }).execute
package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute
track_package_event('pull_package', :pypi)
present_carrierwave_file!(package_file.file, supports_direct_download: true)
end
desc 'The PyPi Simple Endpoint' do
detail 'This feature was introduced in GitLab 12.10'
end
params do
use :package_name
end
# An Api entry point but returns an HTML file instead of JSON.
# PyPi simple API returns the package descriptor as a simple HTML file.
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get 'simple/*package_name', format: :txt do
group = find_authorized_group!
authorize_read_package!(group)
track_package_event('list_package', :pypi)
packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute!
presenter = ::Packages::Pypi::PackagePresenter.new(packages, group)
# Adjusts grape output format
# to be HTML
content_type "text/html; charset=utf-8"
env['api.format'] = :binary
body presenter.body
end
end
end
params do
requires :id, type: Integer, desc: 'The ID of a project'
end
......@@ -43,8 +110,7 @@ module API
end
params do
requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true
requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
use :package_download
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
......@@ -65,7 +131,7 @@ module API
end
params do
requires :package_name, type: String, file_path: true, desc: 'The PyPi package name'
use :package_name
end
# An Api entry point but returns an HTML file instead of JSON.
......
......@@ -31,15 +31,7 @@ RSpec.describe Packages::Pypi::PackageFinder do
context 'within a group' do
let(:scope) { group }
it { expect { subject }.to raise_error(ActiveRecord::RecordNotFound) }
context 'user with access' do
before do
project.add_developer(user)
end
it { is_expected.to eq(package2) }
end
it { is_expected.to eq(package2) }
end
end
end
......@@ -5,45 +5,52 @@ require 'spec_helper'
RSpec.describe ::Packages::Pypi::PackagePresenter do
using RSpec::Parameterized::TableSyntax
let_it_be(:project) { create(:project) }
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, group: group) }
let_it_be(:package_name) { 'sample-project' }
let_it_be(:package1) { create(:pypi_package, project: project, name: package_name, version: '1.0.0') }
let_it_be(:package2) { create(:pypi_package, project: project, name: package_name, version: '2.0.0') }
let(:packages) { [package1, package2] }
let(:presenter) { described_class.new(packages, project) }
describe '#body' do
subject { presenter.body}
let(:file) { package.package_files.first }
let(:filename) { file.file_name }
shared_examples_for "pypi package presenter" do
let(:file) { package.package_files.first }
let(:filename) { file.file_name }
let(:expected_file) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" }
subject(:presenter) { described_class.new(packages, project_or_group).body}
before do
package.pypi_metadatum.required_python = python_version
describe '#body' do
shared_examples_for "pypi package presenter" do
where(:version, :expected_version, :with_package1) do
'>=2.7' | '&gt;=2.7' | true
'"><script>alert(1)</script>' | '&quot;&gt;&lt;script&gt;alert(1)&lt;/script&gt;' | true
'>=2.7, !=3.0' | '&gt;=2.7, !=3.0' | false
end
it { is_expected.to include expected_file }
end
with_them do
let(:python_version) { version }
let(:expected_python_version) { expected_version }
let(:package) { with_package1 ? package1 : package2 }
it_behaves_like "pypi package presenter" do
let(:python_version) { '>=2.7' }
let(:expected_python_version) { '&gt;=2.7' }
let(:package) { package1 }
before do
package.pypi_metadatum.required_python = python_version
end
it { is_expected.to include expected_file }
end
end
it_behaves_like "pypi package presenter" do
let(:python_version) { '"><script>alert(1)</script>' }
let(:expected_python_version) { '&quot;&gt;&lt;script&gt;alert(1)&lt;/script&gt;' }
let(:package) { package1 }
context 'for project' do
let(:project_or_group) { project }
let(:expected_file) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" }
it_behaves_like 'pypi package presenter'
end
it_behaves_like "pypi package presenter" do
let(:python_version) { '>=2.7, !=3.0' }
let(:expected_python_version) { '&gt;=2.7, !=3.0' }
let(:package) { package2 }
context 'for group' do
let(:project_or_group) { group }
let(:expected_file) { "<a href=\"http://localhost/api/v4/groups/#{group.id}/-/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" }
it_behaves_like 'pypi package presenter'
end
end
end
This diff is collapsed.
......@@ -110,6 +110,7 @@ RSpec.shared_examples 'PyPI package versions' do |user_type, status, add_member
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it 'returns the package listing' do
......@@ -127,6 +128,7 @@ RSpec.shared_examples 'PyPI package download' do |user_type, status, add_member
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it 'returns the package listing' do
......@@ -144,24 +146,184 @@ RSpec.shared_examples 'process PyPI api request' do |user_type, status, add_memb
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it_behaves_like 'returning response status', status
end
end
RSpec.shared_examples 'unknown PyPI scope id' do
context 'as anonymous' do
it_behaves_like 'process PyPI api request', :anonymous, :not_found
end
context 'as authenticated user' do
subject { get api(url), headers: basic_auth_header(user.username, personal_access_token.token) }
it_behaves_like 'process PyPI api request', :anonymous, :not_found
end
end
RSpec.shared_examples 'rejects PyPI access with unknown project id' do
context 'with an unknown project' do
let(:project) { OpenStruct.new(id: 1234567890) }
context 'as anonymous' do
it_behaves_like 'process PyPI api request', :anonymous, :not_found
it_behaves_like 'unknown PyPI scope id'
end
end
RSpec.shared_examples 'rejects PyPI access with unknown group id' do
context 'with an unknown project' do
let(:group) { OpenStruct.new(id: 1234567890) }
it_behaves_like 'unknown PyPI scope id'
end
end
RSpec.shared_examples 'pypi simple API endpoint' do
using RSpec::Parameterized::TableSyntax
context 'with valid project' do
where(:visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
:public | :developer | true | true | 'PyPI package versions' | :success
:public | :guest | true | true | 'PyPI package versions' | :success
:public | :developer | true | false | 'PyPI package versions' | :success
:public | :guest | true | false | 'PyPI package versions' | :success
:public | :developer | false | true | 'PyPI package versions' | :success
:public | :guest | false | true | 'PyPI package versions' | :success
:public | :developer | false | false | 'PyPI package versions' | :success
:public | :guest | false | false | 'PyPI package versions' | :success
:public | :anonymous | false | true | 'PyPI package versions' | :success
:private | :developer | true | true | 'PyPI package versions' | :success
:private | :guest | true | true | 'process PyPI api request' | :forbidden
:private | :developer | true | false | 'process PyPI api request' | :unauthorized
:private | :guest | true | false | 'process PyPI api request' | :unauthorized
:private | :developer | false | true | 'process PyPI api request' | :not_found
:private | :guest | false | true | 'process PyPI api request' | :not_found
:private | :developer | false | false | 'process PyPI api request' | :unauthorized
:private | :guest | false | false | 'process PyPI api request' | :unauthorized
:private | :anonymous | false | true | 'process PyPI api request' | :unauthorized
end
context 'as authenticated user' do
subject { get api(url), headers: basic_auth_header(user.username, personal_access_token.token) }
with_them do
let(:token) { user_token ? personal_access_token.token : 'wrong' }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
it_behaves_like 'process PyPI api request', :anonymous, :not_found
before do
project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
end
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
end
end
context 'with a normalized package name' do
let_it_be(:package) { create(:pypi_package, project: project, name: 'my.package') }
let(:url) { "/projects/#{project.id}/packages/pypi/simple/my-package" }
let(:headers) { basic_auth_header(user.username, personal_access_token.token) }
it_behaves_like 'PyPI package versions', :developer, :success
end
end
RSpec.shared_examples 'pypi file download endpoint' do
using RSpec::Parameterized::TableSyntax
context 'with valid project' do
where(:visibility_level, :user_role, :member, :user_token) do
:public | :developer | true | true
:public | :guest | true | true
:public | :developer | true | false
:public | :guest | true | false
:public | :developer | false | true
:public | :guest | false | true
:public | :developer | false | false
:public | :guest | false | false
:public | :anonymous | false | true
:private | :developer | true | true
:private | :guest | true | true
:private | :developer | true | false
:private | :guest | true | false
:private | :developer | false | true
:private | :guest | false | true
:private | :developer | false | false
:private | :guest | false | false
:private | :anonymous | false | true
end
with_them do
let(:token) { user_token ? personal_access_token.token : 'wrong' }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
before do
project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
end
it_behaves_like 'PyPI package download', params[:user_role], :success, params[:member]
end
end
context 'with deploy token headers' do
let(:headers) { basic_auth_header(deploy_token.username, deploy_token.token) }
context 'valid token' do
it_behaves_like 'returning response status', :success
end
context 'invalid token' do
let(:headers) { basic_auth_header('foo', 'bar') }
it_behaves_like 'returning response status', :success
end
end
context 'with job token headers' do
let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, job.token) }
context 'valid token' do
it_behaves_like 'returning response status', :success
end
context 'invalid token' do
let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, 'bar') }
it_behaves_like 'returning response status', :unauthorized
end
context 'invalid user' do
let(:headers) { basic_auth_header('foo', job.token) }
it_behaves_like 'returning response status', :success
end
end
end
RSpec.shared_examples 'a pypi user namespace endpoint' do
using RSpec::Parameterized::TableSyntax
# only group namespaces are supported at this time
where(:visibility_level, :user_role, :expected_status) do
:public | :owner | :not_found
:private | :owner | :not_found
:public | :external | :not_found
:private | :external | :not_found
:public | :anonymous | :not_found
:private | :anonymous | :not_found
end
with_them do
let_it_be_with_reload(:group) { create(:namespace) }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
before do
group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
group.update_column(:owner_id, user.id) if user_role == :owner
end
it_behaves_like 'returning response status', params[:expected_status]
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment