Commit 86abff01 authored by Mayra Cabrera's avatar Mayra Cabrera

Merge branch '233413-pypi-forwarding' into 'master'

Add PyPI package requests forward

See merge request gitlab-org/gitlab!67962
parents ed0290e8 377ab2f5
......@@ -3,11 +3,8 @@
module Packages
module Pypi
class PackagesFinder < ::Packages::GroupOrProjectPackageFinder
def execute!
results = packages.with_normalized_pypi_name(@params[:package_name])
raise ActiveRecord::RecordNotFound if results.empty?
results
def execute
packages.with_normalized_pypi_name(@params[:package_name])
end
private
......
# frozen_string_literal: true
class AddPypiPackageRequestsForwardingToApplicationSettings < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
def up
with_lock_retries do
add_column(:application_settings, :pypi_package_requests_forwarding, :boolean, default: true, null: false)
end
end
def down
with_lock_retries do
remove_column(:application_settings, :pypi_package_requests_forwarding)
end
end
end
1bdbcc6ef5ccf7a2bfb1f9571885e218e230a81b632a2d993302bd87432963f3
\ No newline at end of file
......@@ -9624,6 +9624,7 @@ CREATE TABLE application_settings (
usage_ping_features_enabled boolean DEFAULT false NOT NULL,
encrypted_customers_dot_jwt_signing_key bytea,
encrypted_customers_dot_jwt_signing_key_iv bytea,
pypi_package_requests_forwarding boolean DEFAULT true NOT NULL,
CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
CONSTRAINT app_settings_ext_pipeline_validation_service_url_text_limit CHECK ((char_length(external_pipeline_validation_service_url) <= 255)),
CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)),
......@@ -79,6 +79,7 @@ Example response:
"asset_proxy_whitelist": ["example.com", "*.example.com", "your-instance.com"],
"asset_proxy_allowlist": ["example.com", "*.example.com", "your-instance.com"],
"npm_package_requests_forwarding": true,
"pypi_package_requests_forwarding": true,
"snippet_size_limit": 52428800,
"issues_create_limit": 300,
"raw_blob_request_limit": 300,
......@@ -180,6 +181,7 @@ Example response:
"allow_local_requests_from_web_hooks_and_services": true,
"allow_local_requests_from_system_hooks": false,
"npm_package_requests_forwarding": true,
"pypi_package_requests_forwarding": true,
"snippet_size_limit": 52428800,
"issues_create_limit": 300,
"raw_blob_request_limit": 300,
......@@ -347,6 +349,7 @@ listed in the descriptions of the relevant settings.
| `mirror_max_capacity` | integer | no | **(PREMIUM)** Maximum number of mirrors that can be synchronizing at the same time. |
| `mirror_max_delay` | integer | no | **(PREMIUM)** Maximum time (in minutes) between updates that a mirror can have when scheduled to synchronize. |
| `npm_package_requests_forwarding` | boolean | no | **(PREMIUM)** Use npmjs.org as a default remote repository when the package is not found in the GitLab Package Registry for npm. |
| `pypi_package_requests_forwarding` | boolean | no | **(PREMIUM)** Use pypi.org as a default remote repository when the package is not found in the GitLab Package Registry for PyPI. |
| `outbound_local_requests_whitelist` | array of strings | no | Define a list of trusted domains or IP addresses to which local requests are allowed when local requests for hooks and services are disabled.
| `pages_domain_verification_enabled` | boolean | no | Require users to prove ownership of custom domains. Domain verification is an essential security measure for public GitLab sites. Users are required to demonstrate they control a domain before it is enabled. |
| `password_authentication_enabled_for_git` | boolean | no | Enable authentication for Git over HTTP(S) via a GitLab account password. Default is `true`. |
......
......@@ -262,10 +262,20 @@ To disable it:
1. On the top bar, select **Menu >** **{admin}** **Admin**.
1. On the left sidebar, select **Settings > CI/CD**.
1. Expand the **Package Registry** section.
1. Uncheck **Enable forwarding of npm package requests to npmjs.org**.
1. Click **Save changes**.
1. Clear the checkbox **Forward npm package requests to the npm Registry if the packages are not found in the GitLab Package Registry**.
1. Select **Save changes**.
### PyPI Forwarding **(PREMIUM SELF)**
GitLab administrators can disable the forwarding of PyPI requests to [pypi.org](https://pypi.org/).
To disable it:
![npm package requests forwarding](img/admin_package_registry_npm_package_requests_forward.png)
1. On the top bar, select **Menu >** **{admin}** **Admin**.
1. On the left sidebar, select **Settings > CI/CD**.
1. Expand the **Package Registry** section.
1. Clear the checkbox **Forward PyPI package requests to the PyPI Registry if the packages are not found in the GitLab Package Registry**.
1. Select **Save changes**.
### Package file size limits
......
......@@ -328,6 +328,11 @@ more than once, a `400 Bad Request` error occurs.
## Install a PyPI package
In [GitLab 14.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/233413),
when a PyPI package is not found in the Package Registry, the request is forwarded to [pypi.org](https://pypi.org/).
Administrators can disable this behavior in the [Continuous Integration settings](../../admin_area/settings/continuous_integration.md).
### Install from the project level
To install the latest version of a package, use the following command:
......
......@@ -47,7 +47,7 @@ module EE
adjourned_deletion_for_projects_and_groups: [:delayed_project_deletion, :deletion_adjourned_period],
required_ci_templates: :required_instance_ci_template,
disable_name_update_for_users: :updating_name_disabled_for_users,
package_forwarding: :npm_package_requests_forwarding,
package_forwarding: [:npm_package_requests_forwarding, :pypi_package_requests_forwarding],
default_branch_protection_restriction_in_groups: :group_owners_can_manage_default_branch_protection
}.each do |license_feature, attribute_names|
if License.feature_available?(license_feature)
......
......@@ -119,6 +119,7 @@ module EE
deletion_adjourned_period
updating_name_disabled_for_users
npm_package_requests_forwarding
pypi_package_requests_forwarding
maintenance_mode
maintenance_mode_message
]
......
......@@ -7,6 +7,12 @@
.form-check
= f.check_box :npm_package_requests_forwarding, class: 'form-check-input'
= f.label :npm_package_requests_forwarding, class: 'form-check-label' do
Forward npm package requests to the npm Registry if the packages are not found in the GitLab Package Registry
= _('Forward %{package_type} package requests to the %{registry_type} Registry if the packages are not found in the GitLab Package Registry') % { package_type: 'npm', registry_type: 'npm' }
.form-group
.form-check
= f.check_box :pypi_package_requests_forwarding, class: 'form-check-input'
= f.label :pypi_package_requests_forwarding, class: 'form-check-label' do
= _('Forward %{package_type} package requests to the %{registry_type} Registry if the packages are not found in the GitLab Package Registry') % { package_type: 'PyPI', registry_type: 'PyPI' }
= f.submit _('Save changes'), class: 'btn gl-button btn-confirm'
......@@ -20,6 +20,7 @@ module EE
expose :deletion_adjourned_period, if: ->(_instance, _opts) { ::License.feature_available?(:adjourned_deletion_for_projects_and_groups) }
expose :updating_name_disabled_for_users, if: ->(_instance, _opts) { ::License.feature_available?(:disable_name_update_for_users) }
expose :npm_package_requests_forwarding, if: ->(_instance, _opts) { ::License.feature_available?(:package_forwarding) }
expose :pypi_package_requests_forwarding, if: ->(_instance, _opts) { ::License.feature_available?(:package_forwarding) }
expose :group_owners_can_manage_default_branch_protection, if: ->(_instance, _opts) { ::License.feature_available?(:default_branch_protection_restriction_in_groups) }
expose :maintenance_mode, if: ->(_instance, _opts) { ::Gitlab::Geo.license_allows? }
expose :maintenance_mode_message, if: ->(_instance, _opts) { ::Gitlab::Geo.license_allows? }
......
......@@ -50,6 +50,7 @@ module EE
optional :prevent_merge_requests_author_approval, type: Grape::API::Boolean, desc: 'Disable Merge request author ability to approve request.'
optional :prevent_merge_requests_committers_approval, type: Grape::API::Boolean, desc: 'Disable Merge request committer ability to approve request.'
optional :npm_package_requests_forwarding, type: Grape::API::Boolean, desc: 'NPM package requests are forwarded to npmjs.org if not found on GitLab.'
optional :pypi_package_requests_forwarding, type: Grape::API::Boolean, desc: 'PyPI package requests are forwarded to pypi.org if not found on GitLab.'
optional :group_owners_can_manage_default_branch_protection, type: Grape::API::Boolean, desc: 'Allow owners to manage default branch protection in groups'
optional :maintenance_mode, type: Grape::API::Boolean, desc: 'When instance is in maintenance mode, non-admin users can sign in with read-only access and make read-only API requests'
optional :maintenance_mode_message, type: String, desc: 'Message displayed when instance is in maintenance mode'
......
......@@ -41,7 +41,7 @@ module EE
end
unless License.feature_available?(:package_forwarding)
attrs = attrs.except(:npm_package_requests_forwarding)
attrs = attrs.except(:npm_package_requests_forwarding, :pypi_package_requests_forwarding)
end
unless License.feature_available?(:default_branch_protection_restriction_in_groups)
......
......@@ -111,6 +111,13 @@ RSpec.describe Admin::ApplicationSettingsController do
it_behaves_like 'settings for licensed features'
end
context 'updating pypi packages request forwarding setting' do
let(:settings) { { pypi_package_requests_forwarding: true } }
let(:feature) { :package_forwarding }
it_behaves_like 'settings for licensed features'
end
context 'updating `git_two_factor_session_expiry` setting' do
before do
stub_feature_flags(two_factor_for_cli: true)
......
......@@ -266,7 +266,7 @@ RSpec.describe 'Admin updates EE-only settings' do
visit ci_cd_admin_application_settings_path
end
it 'allows you to change the npm_forwaring setting' do
it 'allows you to change the npm_forwarding setting' do
page.within('#js-package-settings') do
check 'Forward npm package requests to the npm Registry if the packages are not found in the GitLab Package Registry'
click_button 'Save'
......@@ -274,6 +274,15 @@ RSpec.describe 'Admin updates EE-only settings' do
expect(current_settings.npm_package_requests_forwarding).to be true
end
it 'allows you to change the pypi_forwarding setting' do
page.within('#js-package-settings') do
check 'Forward PyPI package requests to the PyPI Registry if the packages are not found in the GitLab Package Registry'
click_button 'Save'
end
expect(current_settings.pypi_package_requests_forwarding).to be true
end
end
context 'sign up settings', :js do
......
......@@ -5,11 +5,17 @@ module API
module Packages
module DependencyProxyHelpers
REGISTRY_BASE_URLS = {
npm: 'https://registry.npmjs.org/'
npm: 'https://registry.npmjs.org/',
pypi: 'https://pypi.org/simple/'
}.freeze
APPLICATION_SETTING_NAMES = {
npm: 'npm_package_requests_forwarding',
pypi: 'pypi_package_requests_forwarding'
}.freeze
def redirect_registry_request(forward_to_registry, package_type, options)
if forward_to_registry && redirect_registry_request_available?
if forward_to_registry && redirect_registry_request_available?(package_type)
::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward")
redirect(registry_url(package_type, options))
else
......@@ -25,11 +31,20 @@ module API
case package_type
when :npm
"#{base_url}#{options[:package_name]}"
when :pypi
"#{base_url}#{options[:package_name]}/"
end
end
def redirect_registry_request_available?
::Gitlab::CurrentSettings.current_application_settings.npm_package_requests_forwarding
def redirect_registry_request_available?(package_type)
application_setting_name = APPLICATION_SETTING_NAMES[package_type]
raise ArgumentError, "Can't find application setting for package_type #{package_type}" unless application_setting_name
::Gitlab::CurrentSettings
.current_application_settings
.attributes
.fetch(application_setting_name, false)
end
end
end
......
......@@ -10,6 +10,7 @@ module API
helpers ::API::Helpers::PackagesManagerClientsHelpers
helpers ::API::Helpers::RelatedResourcesHelpers
helpers ::API::Helpers::Packages::BasicAuthHelpers
helpers ::API::Helpers::Packages::DependencyProxyHelpers
include ::API::Helpers::Packages::BasicAuthHelpers::Constants
feature_category :package_registry
......@@ -82,15 +83,20 @@ module API
track_package_event('list_package', :pypi)
packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute!
presenter = ::Packages::Pypi::PackagePresenter.new(packages, group)
packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute
empty_packages = packages.empty?
# Adjusts grape output format
# to be HTML
content_type "text/html; charset=utf-8"
env['api.format'] = :binary
redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do
not_found!('Package') if empty_packages
presenter = ::Packages::Pypi::PackagePresenter.new(packages, group)
body presenter.body
# Adjusts grape output format
# to be HTML
content_type "text/html; charset=utf-8"
env['api.format'] = :binary
body presenter.body
end
end
end
end
......@@ -142,15 +148,20 @@ module API
track_package_event('list_package', :pypi, project: authorized_user_project, namespace: authorized_user_project.namespace)
packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute!
presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project)
packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute
empty_packages = packages.empty?
redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do
not_found!('Package') if empty_packages
presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project)
# Adjusts grape output format
# to be HTML
content_type "text/html; charset=utf-8"
env['api.format'] = :binary
# Adjusts grape output format
# to be HTML
content_type "text/html; charset=utf-8"
env['api.format'] = :binary
body presenter.body
body presenter.body
end
end
desc 'The PyPi Package upload endpoint' do
......
......@@ -14578,6 +14578,9 @@ msgstr ""
msgid "Format: %{dateFormat}"
msgstr ""
msgid "Forward %{package_type} package requests to the %{registry_type} Registry if the packages are not found in the GitLab Package Registry"
msgstr ""
msgid "Found errors in your %{gitlab_ci_yml}:"
msgstr ""
......
......@@ -14,14 +14,14 @@ RSpec.describe Packages::Pypi::PackagesFinder do
let(:package_name) { package2.name }
describe 'execute!' do
subject { described_class.new(user, scope, package_name: package_name).execute! }
describe 'execute' do
subject { described_class.new(user, scope, package_name: package_name).execute }
shared_examples 'when no package is found' do
context 'non-existing package' do
let(:package_name) { 'none' }
it { expect { subject }.to raise_error(ActiveRecord::RecordNotFound) }
it { expect(subject).to be_empty }
end
end
......@@ -29,7 +29,7 @@ RSpec.describe Packages::Pypi::PackagesFinder do
context 'non-existing package' do
let(:package_name) { package2.name.upcase.tr('-', '.') }
it { expect { subject }.to raise_error(ActiveRecord::RecordNotFound) }
it { expect(subject).to be_empty }
end
end
......@@ -45,7 +45,7 @@ RSpec.describe Packages::Pypi::PackagesFinder do
context 'within a group' do
let(:scope) { group }
it { expect { subject }.to raise_error(ActiveRecord::RecordNotFound) }
it { expect(subject).to be_empty }
context 'user with access to only one project' do
before do
......
......@@ -5,7 +5,7 @@ require 'spec_helper'
RSpec.describe API::Helpers::Packages::DependencyProxyHelpers do
let_it_be(:helper) { Class.new.include(described_class).new }
describe 'redirect_registry_request' do
describe '#redirect_registry_request' do
using RSpec::Parameterized::TableSyntax
let(:options) { {} }
......@@ -13,7 +13,7 @@ RSpec.describe API::Helpers::Packages::DependencyProxyHelpers do
subject { helper.redirect_registry_request(forward_to_registry, package_type, options) { helper.fallback } }
before do
allow(helper).to receive(:options).and_return(for: API::NpmInstancePackages)
allow(helper).to receive(:options).and_return(for: described_class)
end
shared_examples 'executing fallback' do
......@@ -34,38 +34,66 @@ RSpec.describe API::Helpers::Packages::DependencyProxyHelpers do
subject
expect_snowplow_event(category: 'API::NpmInstancePackages', action: 'npm_request_forward')
expect_snowplow_event(category: described_class.to_s, action: "#{package_type}_request_forward")
end
end
context 'with npm packages' do
let(:package_type) { :npm }
%i[npm pypi].each do |forwardable_package_type|
context "with #{forwardable_package_type} packages" do
include_context 'dependency proxy helpers context'
where(:application_setting, :forward_to_registry, :example_name) do
true | true | 'executing redirect'
true | false | 'executing fallback'
false | true | 'executing fallback'
false | false | 'executing fallback'
end
let(:package_type) { forwardable_package_type }
with_them do
before do
stub_application_setting(npm_package_requests_forwarding: application_setting)
where(:application_setting, :forward_to_registry, :example_name) do
true | true | 'executing redirect'
true | false | 'executing fallback'
false | true | 'executing fallback'
false | false | 'executing fallback'
end
it_behaves_like params[:example_name]
with_them do
before do
allow_fetch_application_setting(attribute: "#{forwardable_package_type}_package_requests_forwarding", return_value: application_setting)
end
it_behaves_like params[:example_name]
end
end
end
context 'with non-forwardable packages' do
context 'with non-forwardable package type' do
let(:forward_to_registry) { true }
before do
stub_application_setting(npm_package_requests_forwarding: true)
stub_application_setting(pypi_package_requests_forwarding: true)
end
Packages::Package.package_types.keys.without('npm').each do |pkg_type|
Packages::Package.package_types.keys.without('npm', 'pypi').each do |pkg_type|
context "#{pkg_type}" do
let(:package_type) { pkg_type.to_sym }
it 'raises an error' do
expect { subject }.to raise_error(ArgumentError, "Can't find application setting for package_type #{package_type}")
end
end
end
end
describe '#registry_url' do
subject { helper.registry_url(package_type, package_name: 'test') }
where(:package_type, :expected_result) do
:npm | 'https://registry.npmjs.org/test'
:pypi | 'https://pypi.org/simple/test/'
end
with_them do
it { is_expected.to eq(expected_result) }
end
Packages::Package.package_types.keys.without('npm', 'pypi').each do |pkg_type|
context "with non-forwardable package type #{pkg_type}" do
let(:package_type) { pkg_type }
it 'raises an error' do
......
......@@ -23,7 +23,8 @@ RSpec.describe API::PypiPackages do
subject { get api(url), headers: headers }
describe 'GET /api/v4/groups/:id/-/packages/pypi/simple/:package_name' do
let(:url) { "/groups/#{group.id}/-/packages/pypi/simple/#{package.name}" }
let(:package_name) { package.name }
let(:url) { "/groups/#{group.id}/-/packages/pypi/simple/#{package_name}" }
let(:snowplow_gitlab_standard_context) { {} }
it_behaves_like 'pypi simple API endpoint'
......@@ -40,7 +41,7 @@ RSpec.describe API::PypiPackages do
it_behaves_like 'deploy token for package GET requests'
context 'with group path as id' do
let(:url) { "/groups/#{CGI.escape(group.full_path)}/-/packages/pypi/simple/#{package.name}" }
let(:url) { "/groups/#{CGI.escape(group.full_path)}/-/packages/pypi/simple/#{package_name}"}
it_behaves_like 'deploy token for package GET requests'
end
......@@ -60,7 +61,8 @@ RSpec.describe API::PypiPackages do
end
describe 'GET /api/v4/projects/:id/packages/pypi/simple/:package_name' do
let(:url) { "/projects/#{project.id}/packages/pypi/simple/#{package.name}" }
let(:package_name) { package.name }
let(:url) { "/projects/#{project.id}/packages/pypi/simple/#{package_name}" }
let(:snowplow_gitlab_standard_context) { { project: project, namespace: project.namespace } }
it_behaves_like 'pypi simple API endpoint'
......
# frozen_string_literal: true
RSpec.shared_context 'dependency proxy helpers context' do
def allow_fetch_application_setting(attribute:, return_value:)
attributes = double
allow(::Gitlab::CurrentSettings.current_application_settings).to receive(:attributes).and_return(attributes)
allow(attributes).to receive(:fetch).with(attribute, false).and_return(return_value)
end
end
......@@ -46,6 +46,8 @@ RSpec.shared_examples 'handling get metadata requests' do |scope: :project|
end
shared_examples 'handling all conditions' do
include_context 'dependency proxy helpers context'
where(:auth, :package_name_type, :request_forward, :visibility, :user_role, :expected_result, :expected_status) do
nil | :scoped_naming_convention | true | :public | nil | :accept | :ok
nil | :scoped_naming_convention | false | :public | nil | :accept | :ok
......@@ -243,7 +245,7 @@ RSpec.shared_examples 'handling get metadata requests' do |scope: :project|
project.send("add_#{user_role}", user) if user_role
project.update!(visibility: visibility.to_s)
package.update!(name: package_name) unless package_name == 'non-existing-package'
stub_application_setting(npm_package_requests_forwarding: request_forward)
allow_fetch_application_setting(attribute: "npm_package_requests_forwarding", return_value: request_forward)
end
example_name = "#{params[:expected_result]} metadata request"
......
......@@ -10,9 +10,10 @@ end
RSpec.shared_examples 'accept package tags request' do |status:|
using RSpec::Parameterized::TableSyntax
include_context 'dependency proxy helpers context'
before do
stub_application_setting(npm_package_requests_forwarding: false)
allow_fetch_application_setting(attribute: "npm_package_requests_forwarding", return_value: false)
end
context 'with valid package name' do
......
......@@ -228,6 +228,35 @@ RSpec.shared_examples 'pypi simple API endpoint' do
it_behaves_like 'PyPI package versions', :developer, :success
end
context 'package request forward' do
include_context 'dependency proxy helpers context'
where(:forward, :package_in_project, :shared_examples_name, :expected_status) do
true | true | 'PyPI package versions' | :success
true | false | 'process PyPI api request' | :redirect
false | true | 'PyPI package versions' | :success
false | false | 'process PyPI api request' | :not_found
end
with_them do
let_it_be(:package) { create(:pypi_package, project: project, name: 'foobar') }
let(:package_name) do
if package_in_project
'foobar'
else
'barfoo'
end
end
before do
allow_fetch_application_setting(attribute: "pypi_package_requests_forwarding", return_value: forward)
end
it_behaves_like params[:shared_examples_name], :reporter, params[:expected_status]
end
end
end
RSpec.shared_examples 'pypi file download endpoint' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment