Merge branch 'zj-job-user-ci-token-auth' into 'master'

Allow API access with CI_JOB_TOKEN

Closes #2770

See merge request !2346

app/models/license.rb

@@ -6,6 +6,7 @@ class License < ActiveRecord::Base
   AUDITOR_USER_FEATURE = 'GitLab_Auditor_User'.freeze
   BURNDOWN_CHARTS_FEATURE = 'GitLab_BurndownCharts'.freeze
   CONTRIBUTION_ANALYTICS_FEATURE = 'GitLab_ContributionAnalytics'.freeze
+  CROSS_PROJECT_PIPELINES_FEATURE = 'GitLab_CrossProjectPipelines'.freeze
   DB_LOAD_BALANCING_FEATURE = 'GitLab_DbLoadBalancing'.freeze
   DEPLOY_BOARD_FEATURE = 'GitLab_DeployBoard'.freeze
   ELASTIC_SEARCH_FEATURE = 'GitLab_ElasticSearch'.freeze
@@ -51,6 +52,7 @@ class License < ActiveRecord::Base
     audit_events: AUDIT_EVENTS_FEATURE,
     burndown_charts: BURNDOWN_CHARTS_FEATURE,
     contribution_analytics: CONTRIBUTION_ANALYTICS_FEATURE,
+    cross_project_pipelines: CROSS_PROJECT_PIPELINES_FEATURE,
     deploy_board: DEPLOY_BOARD_FEATURE,
     export_issues: EXPORT_ISSUES_FEATURE,
     fast_forward_merge: FAST_FORWARD_MERGE_FEATURE,
@@ -106,6 +108,7 @@ class License < ActiveRecord::Base
     *EES_FEATURES,
     { ADMIN_AUDIT_LOG_FEATURE => 1 },
     { AUDITOR_USER_FEATURE => 1 },
+    { CROSS_PROJECT_PIPELINES_FEATURE => 1 },
     { DB_LOAD_BALANCING_FEATURE => 1 },
     { DEPLOY_BOARD_FEATURE => 1 },
     { FILE_LOCKS_FEATURE => 1 },
@@ -131,6 +134,7 @@ class License < ActiveRecord::Base
     { AUDITOR_USER_FEATURE => 1 },
     { BURNDOWN_CHARTS_FEATURE => 1 },
     { CONTRIBUTION_ANALYTICS_FEATURE => 1 },
+    { CROSS_PROJECT_PIPELINES_FEATURE => 1 },
     { DEPLOY_BOARD_FEATURE => 1 },
     { EXPORT_ISSUES_FEATURE => 1 },
     { FAST_FORWARD_MERGE_FEATURE => 1 },

app/policies/project_policy.rb

@@ -101,6 +101,7 @@ class ProjectPolicy < BasePolicy
   end
 
   rule { owner | reporter }.policy do
+    enable :build_read_project
     enable :build_download_code
     enable :build_read_container_image
   end

changelogs/unreleased-ee/zj-job-user-ci-token-auth.yml

+---
+title: Allow artifacts access with job_token parameter or CI_JOB_TOKEN header
+merge_request:
+author:

doc/api/jobs.md

@@ -294,9 +294,12 @@ Example of response
 
 ## Get job artifacts
 
-> [Introduced][ce-2893] in GitLab 8.5
+> **Notes**:
+- [Introduced][ce-2893] in GitLab 8.5.
+- The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced][ee-2346]
+  in [GitLab Enterprise Edition Premium][ee] 9.5.
 
-Get job artifacts of a project
+Get job artifacts of a project.
 
 ```
 GET /projects/:id/jobs/:job_id/artifacts
@@ -306,10 +309,27 @@ GET /projects/:id/jobs/:job_id/artifacts
 |------------|---------|----------|---------------------|
 | `id`       | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
 | `job_id` | integer | yes      | The ID of a job   |
+| `job_token` | string  | no       | To be used with [triggers] for multi-project pipelines. Is should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. |
 
-```
-curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/jobs/8/artifacts"
-```
+Example requests:
+
+- Using the `PRIVATE-TOKEN` header:
+
+    ```
+    curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test"
+    ```
+
+- Using the `JOB-TOKEN` header (only inside `.gitlab-ci.yml`):
+
+    ```
+    curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test"
+    ```
+
+- Using the `job_token` parameter (only inside `.gitlab-ci.yml`):
+
+    ```
+    curl --header --form "job-token=$CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test"
+    ```
 
 Response:
 
@@ -322,7 +342,10 @@ Response:
 
 ## Download the artifacts file
 
-> [Introduced][ce-5347] in GitLab 8.10.
+> **Notes**:
+- [Introduced][ce-5347] in GitLab 8.10.
+- The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced][ee-2346]
+  in [GitLab Enterprise Edition Premium][ee] 9.5.
 
 Download the artifacts file from the given reference name and job provided the
 job finished successfully.
@@ -338,12 +361,27 @@ Parameters
 | `id`        | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user       |
 | `ref_name`  | string  | yes      | The ref from a repository |
 | `job`       | string  | yes      | The name of the job       |
+| `job_token` | string  | no       | To be used with [triggers] for multi-project pipelines. Is should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. |
 
-Example request:
+Example requests:
 
-```
-curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test"
-```
+- Using the `PRIVATE-TOKEN` header:
+
+    ```
+    curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test"
+    ```
+
+- Using the `JOB-TOKEN` header (only inside `.gitlab-ci.yml`):
+
+    ```
+    curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test"
+    ```
+
+- Using the `job_token` parameter (only inside `.gitlab-ci.yml`):
+
+    ```
+    curl --header --form "job-token=$CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test"
+    ```
 
 Example response:
 
@@ -615,3 +653,7 @@ Example of response
   "user": null
 }
 ```
+
+[ee]: https://about.gitlab.com/gitlab-ee/
+[ee-2346]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2346
+[triggers]: ../ci/triggers/README.md#when-a-pipeline-depends-on-the-artifacts-of-another-pipeline

doc/ci/triggers/README.md

@@ -18,11 +18,14 @@ A unique trigger token can be obtained when [adding a new trigger](#adding-a-new
 
 ### CI job token
 
-> **Note**:
-[Introduced][ee-2017] in [GitLab Enterprise Edition Premium][ee] 9.3
+You can use the `CI_JOB_TOKEN` [variable][predef] (used to authenticate
+with the [GitLab Container Registry][registry]) in the following cases.
+
+#### When used with multi-project pipelines
 
-You can trigger a new pipeline using the `CI_JOB_TOKEN` [variable][predef]
-which is used to authenticate with the [GitLab Container Registry][registry].
+> **Note**:
+The use of `CI_JOB_TOKEN` for multi-project pipelines was [introduced][ee-2017]
+in [GitLab Enterprise Edition Premium][ee] 9.3.
 
 This way of triggering can only be used when invoked inside `.gitlab-ci.yml`,
 and it creates a dependent pipeline relation visible on the
@@ -40,7 +43,34 @@ build_docs:
 Pipelines triggered that way also expose a special variable:
 `CI_PIPELINE_SOURCE=pipeline`.
 
-For more information, read about [triggering a pipeline](#triggering-a-pipeline).
+Read more about the [pipelines trigger API][trigapi].
+
+#### When a pipeline depends on the artifacts of another pipeline
+
+> **Note**:
+The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced][ee-2346]
+in [GitLab Enterprise Edition Premium][ee] 9.5.
+
+With the introduction of dependencies between different projects, one of
+them may need to access artifacts created by a previous one. This process
+must be granted for authorized accesses, and it can be done using the
+`CI_JOB_TOKEN` variable that identifies a specific job. For example:
+
+```yaml
+build_submodule:
+  stage: test
+  script:
+  - curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test"
+  - unzip artifacts.zip
+  only:
+  - tags
+```
+
+This allows you to use that for multi-project pipelines and download artifacts
+from any project to which you have access as this follows the same principles
+with the [permission model][permissions].
+
+Read more about the [jobs API].
 
 ## Adding a new trigger
 
@@ -244,8 +274,12 @@ removed with one of the future versions of GitLab. You are advised to
 [take ownership](#taking-ownership) of any legacy triggers.
 
 [ee-2017]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2017
+[ee-2346]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2346
 [ci-229]: https://gitlab.com/gitlab-org/gitlab-ci/merge_requests/229
 [ee]: https://about.gitlab.com/gitlab-ee/
 [variables]: ../variables/README.md
 [predef]: ../variables/README.md#predefined-variables-environment-variables
 [registry]: ../../user/project/container_registry.md
+[permissions]: ../../user/permissions.md#jobs-permissions
+[trigapi]: ../../api/pipeline_triggers.md
+[jobs api]: ../../api/jobs.md

doc/ci/variables/README.md

@@ -50,7 +50,7 @@ future GitLab releases.**
 | **CI_JOB_MANUAL**               | 8.12   | all    | The flag to indicate that job was manually started |
 | **CI_JOB_NAME**                 | 9.0    | 0.5    | The name of the job as defined in `.gitlab-ci.yml` |
 | **CI_JOB_STAGE**                | 9.0    | 0.5    | The name of the stage as defined in `.gitlab-ci.yml` |
-| **CI_JOB_TOKEN**                | 9.0    | 1.2    | Token used for authenticating with the GitLab Container Registry |
+| **CI_JOB_TOKEN**                | 9.0    | 1.2    | Token used for authenticating with the GitLab Container Registry. Also used to authenticate with multi-project pipelines when [triggers][trigger-job-token] are involved. |
 | **CI_REPOSITORY_URL**           | 9.0    | all    | The URL to clone the Git repository |
 | **CI_RUNNER_DESCRIPTION**       | 8.10   | 0.5    | The description of the runner as saved in GitLab |
 | **CI_RUNNER_ID**                | 8.10   | 0.5    | The unique id of runner being used |
@@ -461,3 +461,4 @@ export CI_REGISTRY_PASSWORD="longalfanumstring"
 [triggered]: ../triggers/README.md
 [triggers]: ../triggers/README.md#pass-job-variables-to-a-trigger
 [subgroups]: ../../user/group/subgroups/index.md
+[trigger-job-token]: ../triggers/README.md#ci-job-token

lib/api/api_guard.rb

@@ -8,6 +8,8 @@ module API
 
     PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN".freeze
     PRIVATE_TOKEN_PARAM = :private_token
+    JOB_TOKEN_HEADER = "HTTP_JOB_TOKEN".freeze
+    JOB_TOKEN_PARAM = :job_token
 
     included do |base|
       # OAuth2 Resource Server Authentication
@@ -87,12 +89,26 @@ module API
         find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes)
       end
 
+      def find_user_by_job_token
+        return @user_by_job_token if defined?(@user_by_job_token)
+
+        @user_by_job_token =
+          if route_authentication_setting[:job_token_allowed]
+            token_string = params[JOB_TOKEN_PARAM].presence || env[JOB_TOKEN_HEADER].presence
+            Ci::Build.find_by_token(token_string)&.user if token_string
+          end
+      end
+
       def current_user
         @current_user
       end
 
       private
 
+      def route_authentication_setting
+        route_setting(:authentication) || {}
+      end
+
       def find_user_by_authentication_token(token_string)
         User.find_by_authentication_token(token_string)
       end

lib/api/helpers.rb

@@ -60,7 +60,12 @@ module API
     def find_project!(id)
       project = find_project(id)
 
-      if can?(current_user, :read_project, project)
+      # CI job token authentication:
+      # this method grants limited privileged for admin users
+      # admin users can only access project if they are direct member
+      ability = job_token_authentication? ? :build_read_project : :read_project
+
+      if can?(current_user, ability, project)
         project
       else
         not_found!('Project')
@@ -84,6 +89,10 @@ module API
     end
 
     def find_group!(id)
+      # CI job token authentication:
+      # currently we do not allow any group access for CI job token
+      not_found!('Group') if job_token_authentication?
+
       group = find_group(id)
 
       if can?(current_user, :read_group, group)
@@ -352,6 +361,10 @@ module API
       params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER]
     end
 
+    def job_token_authentication?
+      initial_current_user && initial_current_user == find_user_by_job_token
+    end
+
     def warden
       env['warden']
     end
@@ -368,10 +381,12 @@ module API
 
     def initial_current_user
       return @initial_current_user if defined?(@initial_current_user)
+
       Gitlab::Auth::UniqueIpsLimiter.limit_user! do
         @initial_current_user ||= find_user_by_private_token(scopes: scopes_registered_for_endpoint)
         @initial_current_user ||= doorkeeper_guard(scopes: scopes_registered_for_endpoint)
         @initial_current_user ||= find_user_from_warden
+        @initial_current_user ||= find_user_by_job_token
 
         unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed?
           @initial_current_user = nil

lib/api/jobs.rb

@@ -77,8 +77,10 @@ module API
       params do
         requires :job_id, type: Integer, desc: 'The ID of a job'
       end
+      route_setting :authentication, job_token_allowed: true
       get ':id/jobs/:job_id/artifacts' do
         authorize_read_builds!
+        check_cross_project_pipelines_feature!
 
         build = get_build!(params[:job_id])
 
@@ -92,9 +94,11 @@ module API
         requires :ref_name, type: String, desc: 'The ref from repository'
         requires :job,      type: String, desc: 'The name for the job'
       end
+      route_setting :authentication, job_token_allowed: true
       get ':id/jobs/artifacts/:ref_name/download',
         requirements: { ref_name: /.+/ } do
         authorize_read_builds!
+        check_cross_project_pipelines_feature!
 
         builds = user_project.latest_successful_builds_for(params[:ref_name])
         latest_build = builds.find_by!(name: params[:job])
@@ -242,6 +246,10 @@ module API
       def authorize_update_builds!
         authorize! :update_build, user_project
       end
+
+      def check_cross_project_pipelines_feature!
+        not_found!('Project') if job_token_authentication? && !@project.feature_available?(:cross_project_pipelines)
+      end
     end
   end
 end

spec/requests/api/helpers_spec.rb

@@ -21,9 +21,12 @@ describe API::Helpers do
     }
   end
   let(:header) { }
+  let(:route_authentication_setting) { {} }
 
   before do
     allow_any_instance_of(self.class).to receive(:options).and_return({})
+    allow_any_instance_of(self.class).to receive(:route_authentication_setting)
+      .and_return(route_authentication_setting)
   end
 
   def set_env(user_or_token, identifier)
@@ -42,11 +45,13 @@ describe API::Helpers do
 
   def clear_env
     env.delete(API::APIGuard::PRIVATE_TOKEN_HEADER)
+    env.delete(API::APIGuard::JOB_TOKEN_HEADER)
     env.delete(API::Helpers::SUDO_HEADER)
   end
 
   def clear_param
     params.delete(API::APIGuard::PRIVATE_TOKEN_PARAM)
+    params.delete(API::APIGuard::JOB_TOKEN_PARAM)
     params.delete(API::Helpers::SUDO_PARAM)
   end
 
@@ -237,6 +242,35 @@ describe API::Helpers do
       end
     end
 
+    describe "when authenticating using a job token" do
+      let(:job) { create(:ci_build, user: current_user) }
+      let(:route_authentication_setting) { { job_token_allowed: true } }
+
+      before do
+        allow_any_instance_of(described_class).to receive(:doorkeeper_guard).and_return(nil)
+      end
+
+      it "returns nil for an invalid token" do
+        env[API::APIGuard::JOB_TOKEN_HEADER] = 'invalid token'
+
+        expect(current_user).to be_nil
+      end
+
+      it "returns nil for a user without access" do
+        env[API::APIGuard::JOB_TOKEN_HEADER] = job.token
+        allow_any_instance_of(Gitlab::UserAccess).to receive(:allowed?).and_return(false)
+
+        expect(current_user).to be_nil
+      end
+
+      it "returns nil for a user with access, but route not allowed to be authenticated" do
+        env[API::APIGuard::JOB_TOKEN_HEADER] = job.token
+        allow_any_instance_of(Gitlab::UserAccess).to receive(:allowed?).and_return(true)
+
+        expect(current_user).to be_nil
+      end
+    end
+
     context 'sudo usage' do
       context 'with admin' do
         context 'with header' do

spec/requests/api/jobs_spec.rb

@@ -17,8 +17,10 @@ describe API::Jobs do
   let(:api_user) { user }
   let(:reporter) { create(:project_member, :reporter, project: project).user }
   let(:guest) { create(:project_member, :guest, project: project).user }
+  let(:cross_project_pipeline_enabled) { true }
 
   before do
+    stub_licensed_features(cross_project_pipelines: cross_project_pipeline_enabled)
     project.add_developer(user)
   end
 
@@ -191,55 +193,91 @@ describe API::Jobs do
   end
 
   describe 'GET /projects/:id/jobs/:job_id/artifacts' do
-    before do
-      stub_artifacts_object_storage
-      job
-      get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
+    shared_examples 'downloads artifact' do
+      let(:download_headers) do
+        { 'Content-Transfer-Encoding' => 'binary',
+          'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' }
+      end
+
+      it 'returns specific job artifacts' do
+        expect(response).to have_http_status(200)
+        expect(response.headers).to include(download_headers)
+        expect(response.body).to match_file(job.artifacts_file.file.file)
+      end
     end
 
-    context 'job with artifacts' do
-      context 'when artifacts are stored locally' do
-        let(:job) { create(:ci_build, :artifacts, pipeline: pipeline) }
+    context 'normal authentication' do
+      before do
+        stub_artifacts_object_storage
+        get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
+      end
 
-        context 'authorized user' do
-          let(:download_headers) do
-            { 'Content-Transfer-Encoding' => 'binary',
-              'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' }
+      context 'job with artifacts' do
+        context 'when artifacts are stored locally' do
+          let(:job) { create(:ci_build, :artifacts, pipeline: pipeline) }
+
+          context 'authorized user' do
+            it_behaves_like 'downloads artifact'
           end
 
-          it 'returns specific job artifacts' do
-            expect(response).to have_http_status(200)
-            expect(response.headers).to include(download_headers)
-            expect(response.body).to match_file(job.artifacts_file.file.file)
+          context 'unauthorized user' do
+            let(:api_user) { nil }
+
+            it 'does not return specific job artifacts' do
+              expect(response).to have_http_status(401)
+            end
           end
         end
 
-        context 'unauthorized user' do
-          let(:api_user) { nil }
+        context 'when artifacts are stored remotely' do
+          let(:job) { create(:ci_build, :artifacts, :remote_store, pipeline: pipeline) }
 
-          it 'does not return specific job artifacts' do
-            expect(response).to have_http_status(401)
+          it 'returns location redirect' do
+            expect(response).to have_http_status(302)
           end
         end
+
+        it 'does not return job artifacts if not uploaded' do
+          expect(response).to have_http_status(404)
+        end
+      end
+    end
+
+    context 'authorized by job_token' do
+      let(:job) { create(:ci_build, :artifacts, pipeline: pipeline, user: api_user) }
+
+      before do
+        get api("/projects/#{project.id}/jobs/#{job.id}/artifacts"), job_token: job.token
+      end
+
+      context 'user is developer' do
+        let(:api_user) { user }
+
+        it_behaves_like 'downloads artifact'
       end
 
-      context 'when artifacts are stored remotely' do
-        let(:job) { create(:ci_build, :artifacts, :remote_store, pipeline: pipeline) }
+      context 'user is admin, but not member' do
+        let(:api_user) { create(:admin) }
 
-        it 'returns location redirect' do
-          expect(response).to have_http_status(302)
+        it 'does not allow to see that artfiact is present' do
+          expect(response).to have_http_status(404)
         end
       end
-    end
 
-    it 'does not return job artifacts if not uploaded' do
-      expect(response).to have_http_status(404)
+      context 'feature is disabled for EES' do
+        let(:api_user) { user }
+        let(:cross_project_pipeline_enabled) { false }
+
+        it 'disallows access to the artifacts' do
+          expect(response).to have_http_status(404)
+        end
+      end
     end
   end
 
   describe 'GET /projects/:id/artifacts/:ref_name/download?job=name' do
     let(:api_user) { reporter }
-    let(:job) { create(:ci_build, :artifacts, pipeline: pipeline) }
+    let(:job) { create(:ci_build, :artifacts, pipeline: pipeline, user: api_user) }
 
     before do
       stub_artifacts_object_storage
@@ -310,7 +348,7 @@ describe API::Jobs do
         end
 
         context 'when artifacts are stored remotely' do
-          let(:job) { create(:ci_build, :artifacts, :remote_store, pipeline: pipeline) }
+          let(:job) { create(:ci_build, :artifacts, :remote_store, pipeline: pipeline, user: api_user) }
 
           it 'returns location redirect' do
             expect(response).to have_http_status(302)
@@ -343,6 +381,29 @@ describe API::Jobs do
 
         it_behaves_like 'a valid file'
       end
+
+      context 'when using job_token to authenticate' do
+        before do
+          pipeline.reload
+          pipeline.update(ref: 'master',
+                          sha: project.commit('master').sha)
+
+          get api("/projects/#{project.id}/jobs/artifacts/master/download"), job: job.name, job_token: job.token
+        end
+
+        context 'when user is reporter' do
+          it_behaves_like 'a valid file'
+        end
+
+        context 'when user is admin, but not member' do
+          let(:api_user) { create(:admin) }
+          let(:job) { create(:ci_build, :artifacts, pipeline: pipeline, user: api_user) }
+
+          it 'does not allow to see that artfiact is present' do
+            expect(response).to have_http_status(404)
+          end
+        end
+      end
     end
   end