Merge branch 'feature/gb/download-single-job-artifact-using-api' into 'master'

Add API endpoint for downloading a single job artifact

Closes #37196

See merge request !14027

app/controllers/projects/artifacts_controller.rb

@@ -7,7 +7,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
   before_action :authorize_update_build!, only: [:keep]
   before_action :extract_ref_name_and_path
   before_action :validate_artifacts!
-  before_action :set_path_and_entry, only: [:file, :raw]
+  before_action :entry, only: [:file]
 
   def download
     if artifacts_file.file_storage?
@@ -41,7 +41,10 @@ class Projects::ArtifactsController < Projects::ApplicationController
   end
 
   def raw
-    send_artifacts_entry(build, @entry)
+    path = Gitlab::Ci::Build::Artifacts::Path
+      .new(params[:path])
+
+    send_artifacts_entry(build, path)
   end
 
   def keep
@@ -93,9 +96,8 @@ class Projects::ArtifactsController < Projects::ApplicationController
     @artifacts_file ||= build.artifacts_file
   end
 
-  def set_path_and_entry
-    @path = params[:path]
-    @entry = build.artifacts_metadata_entry(@path)
+  def entry
+    @entry = build.artifacts_metadata_entry(params[:path])
 
     render_404 unless @entry.exists?
   end

changelogs/unreleased/feature-gb-download-single-job-artifact-using-api.yml

+---
+title: Make it possible to download a single job artifact file using the API
+merge_request: 14027
+author:
+type: added

doc/api/jobs.md

@@ -320,11 +320,11 @@ Response:
 
 [ce-2893]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/2893
 
-## Download the artifacts file
+## Download the artifacts archive
 
 > [Introduced][ce-5347] in GitLab 8.10.
 
-Download the artifacts file from the given reference name and job provided the
+Download the artifacts archive from the given reference name and job provided the
 job finished successfully.
 
 ```
@@ -354,6 +354,40 @@ Example response:
 
 [ce-5347]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5347
 
+## Download a single artifact file
+
+> Introduced in GitLab 10.0
+
+Download a single artifact file from within the job's artifacts archive.
+
+Only a single file is going to be extracted from the archive and streamed to a client.
+
+```
+GET /projects/:id/jobs/:job_id/artifacts/*artifact_path
+```
+
+Parameters
+
+| Attribute       | Type    | Required | Description               |
+|-----------------|---------|----------|-------------------------- |
+| `id`            | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user       |
+| `job_id  `      | integer | yes      | The unique job identifier |
+| `artifact_path` | string  | yes      | Path to a file inside the artifacts archive |
+
+Example request:
+
+```
+curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/jobs/5/artifacts/some/release/file.pdf"
+```
+
+Example response:
+
+| Status    | Description                          |
+|-----------|--------------------------------------|
+| 200       | Sends a single artifact file         |
+| 400       | Invalid path provided                |
+| 404       | Build not found or no file/artifacts |
+
 ## Get a trace file
 
 Get a trace of a specific job of a project

lib/api/api.rb

@@ -108,6 +108,7 @@ module API
     mount ::API::Internal
     mount ::API::Issues
     mount ::API::Jobs
+    mount ::API::JobArtifacts
     mount ::API::Keys
     mount ::API::Labels
     mount ::API::Lint

lib/api/helpers.rb

@@ -128,6 +128,10 @@ module API
       merge_request
     end
 
+    def find_build!(id)
+      user_project.builds.find(id.to_i)
+    end
+
     def authenticate!
       unauthorized! unless current_user && can?(initial_current_user, :access_api)
     end
@@ -160,6 +164,14 @@ module API
       authorize! :admin_project, user_project
     end
 
+    def authorize_read_builds!
+      authorize! :read_build, user_project
+    end
+
+    def authorize_update_builds!
+      authorize! :update_build, user_project
+    end
+
     def require_gitlab_workhorse!
       unless env['HTTP_GITLAB_WORKHORSE'].present?
         forbidden!('Request should be executed via GitLab Workhorse')
@@ -210,7 +222,7 @@ module API
 
     def bad_request!(attribute)
       message = ["400 (Bad request)"]
-      message << "\"" + attribute.to_s + "\" not given"
+      message << "\"" + attribute.to_s + "\" not given" if attribute
       render_api_error!(message.join(' '), 400)
     end
 
@@ -432,6 +444,10 @@ module API
       header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format))
     end
 
+    def send_artifacts_entry(build, entry)
+      header(*Gitlab::Workhorse.send_artifacts_entry(build, entry))
+    end
+
     # The Grape Error Middleware only has access to env but no params. We workaround this by
     # defining a method that returns the right value.
     def define_params_for_grape_middleware

lib/api/job_artifacts.rb

+module API
+  class JobArtifacts < Grape::API
+    before { authenticate_non_get! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      desc 'Download the artifacts file from a job' do
+        detail 'This feature was introduced in GitLab 8.10'
+      end
+      params do
+        requires :ref_name, type: String, desc: 'The ref from repository'
+        requires :job,      type: String, desc: 'The name for the job'
+      end
+      get ':id/jobs/artifacts/:ref_name/download',
+        requirements: { ref_name: /.+/ } do
+        authorize_read_builds!
+
+        builds = user_project.latest_successful_builds_for(params[:ref_name])
+        latest_build = builds.find_by!(name: params[:job])
+
+        present_artifacts!(latest_build.artifacts_file)
+      end
+
+      desc 'Download the artifacts file from a job' do
+        detail 'This feature was introduced in GitLab 8.5'
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+      end
+      get ':id/jobs/:job_id/artifacts' do
+        authorize_read_builds!
+
+        build = find_build!(params[:job_id])
+
+        present_artifacts!(build.artifacts_file)
+      end
+
+      desc 'Download a specific file from artifacts archive' do
+        detail 'This feature was introduced in GitLab 10.0'
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+        requires :artifact_path, type: String, desc: 'Artifact path'
+      end
+      get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do
+        authorize_read_builds!
+
+        build = find_build!(params[:job_id])
+        not_found! unless build.artifacts?
+
+        path = Gitlab::Ci::Build::Artifacts::Path
+          .new(params[:artifact_path])
+        bad_request! unless path.valid?
+
+        send_artifacts_entry(build, path)
+      end
+
+      desc 'Keep the artifacts to prevent them from being deleted' do
+        success Entities::Job
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+      end
+      post ':id/jobs/:job_id/artifacts/keep' do
+        authorize_update_builds!
+
+        build = find_build!(params[:job_id])
+        authorize!(:update_build, build)
+        return not_found!(build) unless build.artifacts?
+
+        build.keep_artifacts!
+
+        status 200
+        present build, with: Entities::Job
+      end
+    end
+  end
+end

lib/api/jobs.rb

@@ -66,42 +66,11 @@ module API
       get ':id/jobs/:job_id' do
         authorize_read_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
 
         present build, with: Entities::Job
       end
 
-      desc 'Download the artifacts file from a job' do
-        detail 'This feature was introduced in GitLab 8.5'
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      get ':id/jobs/:job_id/artifacts' do
-        authorize_read_builds!
-
-        build = get_build!(params[:job_id])
-
-        present_artifacts!(build.artifacts_file)
-      end
-
-      desc 'Download the artifacts file from a job' do
-        detail 'This feature was introduced in GitLab 8.10'
-      end
-      params do
-        requires :ref_name, type: String, desc: 'The ref from repository'
-        requires :job,      type: String, desc: 'The name for the job'
-      end
-      get ':id/jobs/artifacts/:ref_name/download',
-        requirements: { ref_name: /.+/ } do
-        authorize_read_builds!
-
-        builds = user_project.latest_successful_builds_for(params[:ref_name])
-        latest_build = builds.find_by!(name: params[:job])
-
-        present_artifacts!(latest_build.artifacts_file)
-      end
-
       # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace
       #       is saved in the DB instead of file). But before that, we need to consider how to replace the value of
       #       `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
@@ -112,7 +81,7 @@ module API
       get ':id/jobs/:job_id/trace' do
         authorize_read_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
 
         header 'Content-Disposition', "infile; filename=\"#{build.id}.log\""
         content_type 'text/plain'
@@ -131,7 +100,7 @@ module API
       post ':id/jobs/:job_id/cancel' do
         authorize_update_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
         authorize!(:update_build, build)
 
         build.cancel
@@ -148,7 +117,7 @@ module API
       post ':id/jobs/:job_id/retry' do
         authorize_update_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
         authorize!(:update_build, build)
         return forbidden!('Job is not retryable') unless build.retryable?
 
@@ -166,7 +135,7 @@ module API
       post ':id/jobs/:job_id/erase' do
         authorize_update_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
         authorize!(:update_build, build)
         return forbidden!('Job is not erasable!') unless build.erasable?
 
@@ -174,25 +143,6 @@ module API
         present build, with: Entities::Job
       end
 
-      desc 'Keep the artifacts to prevent them from being deleted' do
-        success Entities::Job
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      post ':id/jobs/:job_id/artifacts/keep' do
-        authorize_update_builds!
-
-        build = get_build!(params[:job_id])
-        authorize!(:update_build, build)
-        return not_found!(build) unless build.artifacts?
-
-        build.keep_artifacts!
-
-        status 200
-        present build, with: Entities::Job
-      end
-
       desc 'Trigger a manual job' do
         success Entities::Job
         detail 'This feature was added in GitLab 8.11'
@@ -203,7 +153,7 @@ module API
       post ":id/jobs/:job_id/play" do
         authorize_read_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
 
         authorize!(:update_build, build)
         bad_request!("Unplayable Job") unless build.playable?
@@ -216,14 +166,6 @@ module API
     end
 
     helpers do
-      def find_build(id)
-        user_project.builds.find_by(id: id.to_i)
-      end
-
-      def get_build!(id)
-        find_build(id) || not_found!
-      end
-
       def filter_builds(builds, scope)
         return builds if scope.nil? || scope.empty?
 
@@ -234,14 +176,6 @@ module API
 
         builds.where(status: available_statuses && scope)
       end
-
-      def authorize_read_builds!
-        authorize! :read_build, user_project
-      end
-
-      def authorize_update_builds!
-        authorize! :update_build, user_project
-      end
     end
   end
 end

lib/gitlab/ci/build/artifacts/metadata/entry.rb

 module Gitlab
-  module Ci::Build::Artifacts
+  module Ci
+    module Build
+      module Artifacts
         class Metadata
           ##
           # Class that represents an entry (path and metadata) to a file or
@@ -11,26 +13,18 @@ module Gitlab
           # This class is working only with UTF-8 encoded paths.
           #
           class Entry
-        attr_reader :path, :entries
+            attr_reader :entries
             attr_accessor :name
 
             def initialize(path, entries)
-          @path = path.dup.force_encoding('UTF-8')
               @entries = entries
-
-          if path.include?("\0")
-            raise ArgumentError, 'Path contains zero byte character!'
-          end
-
-          unless path.valid_encoding?
-            raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
-          end
+              @path = Artifacts::Path.new(path)
             end
 
             delegate :empty?, to: :children
 
             def directory?
-          blank_node? || @path.end_with?('/')
+              blank_node? || @path.directory?
             end
 
             def file?
@@ -49,7 +43,7 @@ module Gitlab
 
             def parent
               return nil unless has_parent?
-          self.class.new(@path.chomp(basename), @entries)
+              self.class.new(@path.to_s.chomp(basename), @entries)
             end
 
             def basename
@@ -57,14 +51,14 @@ module Gitlab
             end
 
             def name
-          @name || @path.split('/').last.to_s
+              @name || @path.name
             end
 
             def children
               return [] unless directory?
               return @children if @children
 
-          child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$}
+              child_pattern = %r{^#{Regexp.escape(@path.to_s)}[^/]+/?$}
               @children = select_entries { |path| path =~ child_pattern }
             end
 
@@ -84,38 +78,42 @@ module Gitlab
             end
 
             def metadata
-          @entries[@path] || {}
+              @entries[@path.to_s] || {}
             end
 
             def nodes
-          @path.count('/') + (file? ? 1 : 0)
+              @path.nodes + (file? ? 1 : 0)
             end
 
             def blank_node?
-          @path.empty? # "" is considered to be './'
+              @path.to_s.empty? # "" is considered to be './'
             end
 
             def exists?
-          blank_node? || @entries.include?(@path)
+              blank_node? || @entries.include?(@path.to_s)
             end
 
             def total_size
-          descendant_pattern = %r{^#{Regexp.escape(@path)}}
+              descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}}
               entries.sum do |path, entry|
                 (entry[:size] if path =~ descendant_pattern).to_i
               end
             end
 
+            def path
+              @path.to_s
+            end
+
             def to_s
-          @path
+              @path.to_s
             end
 
             def ==(other)
-          @path == other.path && @entries == other.entries
+              path == other.path && @entries == other.entries
             end
 
             def inspect
-          "#{self.class.name}: #{@path}"
+              "#{self.class.name}: #{self}"
             end
 
             private
@@ -127,4 +125,6 @@ module Gitlab
           end
         end
       end
+    end
+  end
 end

lib/gitlab/ci/build/artifacts/path.rb

+module Gitlab
+  module Ci
+    module Build
+      module Artifacts
+        class Path
+          def initialize(path)
+            @path = path.dup.force_encoding('UTF-8')
+          end
+
+          def valid?
+            nonzero? && utf8?
+          end
+
+          def directory?
+            @path.end_with?('/')
+          end
+
+          def name
+            @path.split('/').last.to_s
+          end
+
+          def nodes
+            @path.count('/')
+          end
+
+          def to_s
+            @path.tap do |path|
+              unless nonzero?
+                raise ArgumentError, 'Path contains zero byte character!'
+              end
+
+              unless utf8?
+                raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
+              end
+            end
+          end
+
+          private
+
+          def nonzero?
+            @path.exclude?("\0")
+          end
+
+          def utf8?
+            @path.valid_encoding?
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/workhorse.rb

@@ -121,10 +121,10 @@ module Gitlab
         ]
       end
 
-      def send_artifacts_entry(build, entry)
+      def send_artifacts_entry(build, path)
         params = {
           'Archive' => build.artifacts_file.path,
-          'Entry' => Base64.encode64(entry.path)
+          'Entry' => Base64.encode64(path.to_s)
         }
 
         [

spec/controllers/projects/artifacts_controller_spec.rb

@@ -81,14 +81,6 @@ describe Projects::ArtifactsController do
         expect(params['Entry']).to eq(Base64.encode64('ci_artifacts.txt'))
       end
     end
-
-    context 'when the file does not exist' do
-      it 'responds Not Found' do
-        get :raw, namespace_id: project.namespace, project_id: project, job_id: job, path: 'unknown'
-
-        expect(response).to be_not_found
-      end
-    end
   end
 
   describe 'GET latest_succeeded' do

spec/lib/gitlab/ci/build/artifacts/path_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Artifacts::Path do
+  describe '#valid?' do
+    context 'when path contains a zero character' do
+      it 'is not valid' do
+        expect(described_class.new("something/\255")).not_to be_valid
+      end
+    end
+
+    context 'when path is not utf8 string' do
+      it 'is not valid' do
+        expect(described_class.new("something/\0")).not_to be_valid
+      end
+    end
+
+    context 'when path is valid' do
+      it 'is valid' do
+        expect(described_class.new("some/file/path")).to be_valid
+      end
+    end
+  end
+
+  describe '#directory?' do
+    context 'when path ends with a directory indicator' do
+      it 'is a directory' do
+        expect(described_class.new("some/file/dir/")).to be_directory
+      end
+    end
+
+    context 'when path does not end with a directory indicator' do
+      it 'is not a directory' do
+        expect(described_class.new("some/file")).not_to be_directory
+      end
+    end
+  end
+
+  describe '#name' do
+    it 'returns a base name' do
+      expect(described_class.new("some/file").name).to eq 'file'
+    end
+  end
+
+  describe '#nodes' do
+    it 'returns number of path nodes' do
+      expect(described_class.new("some/dir/file").nodes).to eq 2
+    end
+  end
+
+  describe '#to_s' do
+    context 'when path is valid' do
+      it 'returns a string representation of a path' do
+        expect(described_class.new('some/path').to_s).to eq 'some/path'
+      end
+    end
+
+    context 'when path is invalid' do
+      it 'raises an error' do
+        expect { described_class.new("invalid/\0").to_s }
+          .to raise_error ArgumentError
+      end
+    end
+  end
+end

spec/requests/api/jobs_spec.rb

 require 'spec_helper'
 
 describe API::Jobs do
-  let!(:project) do
+  set(:project) do
     create(:project, :repository, public_builds: false)
   end
 
-  let!(:pipeline) do
+  set(:pipeline) do
     create(:ci_empty_pipeline, project: project,
                                sha: project.commit.id,
                                ref: project.default_branch)
@@ -188,6 +188,84 @@ describe API::Jobs do
     end
   end
 
+  describe 'GET /projects/:id/jobs/:job_id/artifacts/:artifact_path' do
+    context 'when job has artifacts' do
+      let(:job) { create(:ci_build, :artifacts, pipeline: pipeline) }
+
+      let(:artifact) do
+        'other_artifacts_0.1.2/another-subdirectory/banana_sample.gif'
+      end
+
+      context 'when user is anonymous' do
+        let(:api_user) { nil }
+
+        context 'when project is public' do
+          it 'allows to access artifacts' do
+            project.update_column(:visibility_level,
+                                  Gitlab::VisibilityLevel::PUBLIC)
+            project.update_column(:public_builds, true)
+
+            get_artifact_file(artifact)
+
+            expect(response).to have_http_status(200)
+          end
+        end
+
+        context 'when project is public with builds access disabled' do
+          it 'rejects access to artifacts' do
+            project.update_column(:visibility_level,
+                                  Gitlab::VisibilityLevel::PUBLIC)
+            project.update_column(:public_builds, false)
+
+            get_artifact_file(artifact)
+
+            expect(response).to have_http_status(403)
+          end
+        end
+
+        context 'when project is private' do
+          it 'rejects access and hides existence of artifacts' do
+            project.update_column(:visibility_level,
+                                  Gitlab::VisibilityLevel::PRIVATE)
+            project.update_column(:public_builds, true)
+
+            get_artifact_file(artifact)
+
+            expect(response).to have_http_status(404)
+          end
+        end
+      end
+
+      context 'when user is authorized' do
+        it 'returns a specific artifact file for a valid path' do
+          expect(Gitlab::Workhorse)
+            .to receive(:send_artifacts_entry)
+            .and_call_original
+
+          get_artifact_file(artifact)
+
+          expect(response).to have_http_status(200)
+          expect(response.headers)
+            .to include('Content-Type' => 'application/json',
+                        'Gitlab-Workhorse-Send-Data' => /artifacts-entry/)
+        end
+      end
+    end
+
+    context 'when job does not have artifacts' do
+      it 'does not return job artifact file' do
+        get_artifact_file('some/artifact')
+
+        expect(response).to have_http_status(404)
+      end
+    end
+
+    def get_artifact_file(artifact_path)
+      get api("/projects/#{project.id}/jobs/#{job.id}/" \
+              "artifacts/#{artifact_path}", api_user)
+    end
+  end
+
   describe 'GET /projects/:id/jobs/:job_id/artifacts' do
     before do
       get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
@@ -209,11 +287,12 @@ describe API::Jobs do
         end
       end
 
-      context 'unauthorized user' do
+      context 'when anonymous user is accessing private artifacts' do
         let(:api_user) { nil }
 
-        it 'does not return specific job artifacts' do
-          expect(response).to have_http_status(401)
+        it 'hides artifacts and rejects request' do
+          expect(project).to be_private
+          expect(response).to have_http_status(404)
         end
       end
     end
@@ -242,8 +321,9 @@ describe API::Jobs do
         get_for_ref
       end
 
-      it 'gives 401' do
-        expect(response).to have_http_status(401)
+      it 'does not find a resource in a private project' do
+        expect(project).to be_private
+        expect(response).to have_http_status(404)
       end
     end