Merge branch '202012-pypi-job-tokens' into 'master'

PyPI authentication with CI job tokens

See merge request gitlab-org/gitlab!40888

changelogs/unreleased/202012-pypi-job-tokens.yml

+---
+title: Add job token authentication for the GitLab PyPI package repository
+merge_request: 40888
+author:
+type: added

doc/user/packages/package_registry/index.md

@@ -26,19 +26,19 @@ For information on how to create and upload a package, view the GitLab documenta
 ## Use GitLab CI/CD to build packages
 
 You can use [GitLab CI/CD](../../../ci/README.md) to build packages.
-For Maven, NuGet and NPM packages, and Composer dependencies, you can
+For Maven, NuGet, NPM, Conan, and PyPI packages, and Composer dependencies, you can
 authenticate with GitLab by using the `CI_JOB_TOKEN`.
 
 CI/CD templates, which you can use to get started, are in [this repo](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates).
 
-Learn more about [using CI/CD to build Maven packages](../maven_repository/index.md#creating-maven-packages-with-gitlab-cicd), [NPM packages](../npm_registry/index.md#publishing-a-package-with-cicd), [Composer packages](../composer_repository/index.md#publishing-the-package-with-cicd), and [NuGet packages](../nuget_repository/index.md#publishing-a-nuget-package-with-cicd).
+Learn more about [using CI/CD to build Maven packages](../maven_repository/index.md#creating-maven-packages-with-gitlab-cicd), [NPM packages](../npm_registry/index.md#publishing-a-package-with-cicd), [Composer packages](../composer_repository/index.md#publishing-the-package-with-cicd), [NuGet Packages](../nuget_repository/index.md#publishing-a-nuget-package-with-cicd), [Conan Packages](../conan_repository/index.md#using-gitlab-ci-with-conan-packages), and [PyPI packages](../pypi_repository/index.md#using-gitlab-ci-with-pypi-packages).
 
 If you use CI/CD to build a package, extended activity
 information is displayed when you view the package details:
 
 ![Package CI/CD activity](img/package_activity_v12_10.png)
 
-You can view which pipeline published the package, as well as the commit and
+When using Maven and NPM, you can view which pipeline published the package, as well as the commit and
 user who triggered it.
 
 ## Download a package

doc/user/packages/pypi_repository/index.md

@@ -302,20 +302,10 @@ Successfully installed mypypipackage-0.0.1
 
 ## Using GitLab CI with PyPI packages
 
-NOTE: **Note:**
-`CI_JOB_TOKEN`s are not yet supported for use with PyPI.
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11678) in GitLab 13.4.
 
 To work with PyPI commands within [GitLab CI/CD](./../../../ci/README.md), you can use
-[environment variables](./../../../ci/variables/README.md#custom-environment-variables)
-to access your authentication tokens in your commands.
-
-Set up environment variables for `TWINE_PASSWORD` and `TWINE_USERNAME` using either:
-
-- A [personal access token](../../../user/profile/personal_access_tokens.md) and your GitLab username.
-- A [deploy token](./../../project/deploy_tokens/index.md) and its associated deploy token username.
-
-You can now access your `TWINE_USERNAME` and `TWINE_PASSWORD` using any `twine` command in your
-`.gitlab-ci.yml` file.
+`CI_JOB_TOKEN` in place of the personal access token or deploy token in your commands.
 
 For example:
 
@@ -326,5 +316,18 @@ run:
   script:
     - pip install twine
     - python setup.py sdist bdist_wheel
-    - TWINE_PASSWORD=${TWINE_PASSWORD} TWINE_USERNAME=${TWINE_USERNAME} python -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/*
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/*
+```
+
+You can also use `CI_JOB_TOKEN` in a `~/.pypirc` file that you check into GitLab:
+
+```ini
+[distutils]
+index-servers =
+    gitlab
+
+[gitlab]
+repository = https://gitlab.com/api/v4/projects/${env.CI_PROJECT_ID}/packages/pypi
+username = gitlab-ci-token
+password = ${env.CI_JOB_TOKEN}
 ```

lib/api/pypi_packages.rb

@@ -64,7 +64,7 @@ module API
           requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
         end
 
-        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
         get 'files/:sha256/*file_identifier' do
           project = unauthorized_user_project!
 
@@ -87,7 +87,7 @@ module API
 
         # An Api entry point but returns an HTML file instead of JSON.
         # PyPi simple API returns the package descriptor as a simple HTML file.
-        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
         get 'simple/*package_name', format: :txt do
           authorize_read_package!(authorized_user_project)
 
@@ -117,7 +117,7 @@ module API
           optional :sha256_digest, type: String
         end
 
-        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
         post do
           authorize_upload!(authorized_user_project)
           bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size)
@@ -135,7 +135,7 @@ module API
           forbidden!
         end
 
-        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
         post 'authorize' do
           authorize_workhorse!(
             subject: authorized_user_project,

spec/requests/api/pypi_packages_spec.rb

@@ -11,6 +11,7 @@ RSpec.describe API::PypiPackages do
   let_it_be(:personal_access_token) { create(:personal_access_token, user: user) }
   let_it_be(:deploy_token) { create(:deploy_token, read_package_registry: true, write_package_registry: true) }
   let_it_be(:project_deploy_token) { create(:project_deploy_token, deploy_token: deploy_token, project: project) }
+  let_it_be(:job) { create(:ci_build, :running, user: user) }
 
   describe 'GET /api/v4/projects/:id/packages/pypi/simple/:package_name' do
     let_it_be(:package) { create(:pypi_package, project: project) }
@@ -58,6 +59,8 @@ RSpec.describe API::PypiPackages do
 
     it_behaves_like 'deploy token for package GET requests'
 
+    it_behaves_like 'job token for package GET requests'
+
     it_behaves_like 'rejects PyPI access with unknown project id'
   end
 
@@ -108,6 +111,8 @@ RSpec.describe API::PypiPackages do
 
     it_behaves_like 'deploy token for package uploads'
 
+    it_behaves_like 'job token for package uploads'
+
     it_behaves_like 'rejects PyPI access with unknown project id'
   end
 
@@ -198,6 +203,8 @@ RSpec.describe API::PypiPackages do
 
     it_behaves_like 'deploy token for package uploads'
 
+    it_behaves_like 'job token for package uploads'
+
     it_behaves_like 'rejects PyPI access with unknown project id'
 
     context 'file size above maximum limit' do
@@ -273,6 +280,26 @@ RSpec.describe API::PypiPackages do
       end
     end
 
+    context 'with job token headers' do
+      let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, job.token) }
+
+      context 'valid token' do
+        it_behaves_like 'returning response status', :success
+      end
+
+      context 'invalid token' do
+        let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, 'bar') }
+
+        it_behaves_like 'returning response status', :success
+      end
+
+      context 'invalid user' do
+        let(:headers) { basic_auth_header('foo', job.token) }
+
+        it_behaves_like 'returning response status', :success
+      end
+    end
+
     it_behaves_like 'rejects PyPI access with unknown project id'
   end
 end

spec/support/shared_examples/requests/api/packages_shared_examples.rb

@@ -70,3 +70,59 @@ RSpec.shared_examples 'does not cause n^2 queries' do
     end.not_to exceed_query_limit(control)
   end
 end
+
+RSpec.shared_examples 'job token for package GET requests' do
+  context 'with job token headers' do
+    let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, job.token) }
+
+    subject { get api(url), headers: headers }
+
+    before do
+      project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
+      project.add_developer(user)
+    end
+
+    context 'valid token' do
+      it_behaves_like 'returning response status', :success
+    end
+
+    context 'invalid token' do
+      let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, 'bar') }
+
+      it_behaves_like 'returning response status', :unauthorized
+    end
+
+    context 'invalid user' do
+      let(:headers) { basic_auth_header('foo', job.token) }
+
+      it_behaves_like 'returning response status', :unauthorized
+    end
+  end
+end
+
+RSpec.shared_examples 'job token for package uploads' do
+  context 'with job token headers' do
+    let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, job.token).merge(workhorse_header) }
+
+    before do
+      project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
+      project.add_developer(user)
+    end
+
+    context 'valid token' do
+      it_behaves_like 'returning response status', :success
+    end
+
+    context 'invalid token' do
+      let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, 'bar').merge(workhorse_header) }
+
+      it_behaves_like 'returning response status', :unauthorized
+    end
+
+    context 'invalid user' do
+      let(:headers) { basic_auth_header('foo', job.token).merge(workhorse_header) }
+
+      it_behaves_like 'returning response status', :unauthorized
+    end
+  end
+end