Merge branch 'feature/runner-jobs-v4-api' into 'master'

Feature/runner jobs v4 api

Closes #28513

See merge request !9273

app/models/ci/build.rb

@@ -517,6 +517,27 @@ module Ci
       ]
     end
 
+    def steps
+      [Gitlab::Ci::Build::Step.from_commands(self),
+       Gitlab::Ci::Build::Step.from_after_script(self)].compact
+    end
+
+    def image
+      Gitlab::Ci::Build::Image.from_image(self)
+    end
+
+    def services
+      Gitlab::Ci::Build::Image.from_services(self)
+    end
+
+    def artifacts
+      [options[:artifacts]]
+    end
+
+    def cache
+      [options[:cache]]
+    end
+
     def credentials
       Gitlab::Ci::Build::Credentials::Factory.new(self).create!
     end

app/services/ci/register_build_service.rb → app/services/ci/register_job_service.rb

 module Ci
   # This class responsible for assigning
   # proper pending build to runner on runner API request
-  class RegisterBuildService
+  class RegisterJobService
     include Gitlab::CurrentSettings
 
     attr_reader :runner

changelogs/unreleased/feature-runner-jobs-v4-api.yml

+---
+title: Add Runner's jobs v4 API
+merge_request: 9273
+author:

lib/api/entities.rb

@@ -705,5 +705,83 @@ module API
       expose :id, :message, :starts_at, :ends_at, :color, :font
       expose :active?, as: :active
     end
+
+    module JobRequest
+      class JobInfo < Grape::Entity
+        expose :name, :stage
+        expose :project_id, :project_name
+      end
+
+      class GitInfo < Grape::Entity
+        expose :repo_url, :ref, :sha, :before_sha
+        expose :ref_type do |model|
+          if model.tag
+            'tag'
+          else
+            'branch'
+          end
+        end
+      end
+
+      class RunnerInfo < Grape::Entity
+        expose :timeout
+      end
+
+      class Step < Grape::Entity
+        expose :name, :script, :timeout, :when, :allow_failure
+      end
+
+      class Image < Grape::Entity
+        expose :name
+      end
+
+      class Artifacts < Grape::Entity
+        expose :name, :untracked, :paths, :when, :expire_in
+      end
+
+      class Cache < Grape::Entity
+        expose :key, :untracked, :paths
+      end
+
+      class Credentials < Grape::Entity
+        expose :type, :url, :username, :password
+      end
+
+      class ArtifactFile < Grape::Entity
+        expose :filename, :size
+      end
+
+      class Dependency < Grape::Entity
+        expose :id, :name
+        expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? }
+      end
+
+      class Response < Grape::Entity
+        expose :id
+        expose :token
+        expose :allow_git_fetch
+
+        expose :job_info, using: JobInfo do |model|
+          model
+        end
+
+        expose :git_info, using: GitInfo do |model|
+          model
+        end
+
+        expose :runner_info, using: RunnerInfo do |model|
+          model
+        end
+
+        expose :variables
+        expose :steps, using: Step
+        expose :image, using: Image
+        expose :services, using: Image
+        expose :artifacts, using: Artifacts
+        expose :cache, using: Cache
+        expose :credentials, using: Credentials
+        expose :depends_on_builds, as: :dependencies, using: Dependency
+      end
+    end
   end
 end

lib/api/helpers/runner.rb

 module API
   module Helpers
     module Runner
+      JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze
+      JOB_TOKEN_PARAM = :token
+      UPDATE_RUNNER_EVERY = 10 * 60
+
       def runner_registration_token_valid?
         ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token],
                                                                   current_application_settings.runners_registration_token)
@@ -18,6 +22,56 @@ module API
       def current_runner
         @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s)
       end
+
+      def update_runner_info
+        return unless update_runner?
+
+        current_runner.contacted_at = Time.now
+        current_runner.assign_attributes(get_runner_version_from_params)
+        current_runner.save if current_runner.changed?
+      end
+
+      def update_runner?
+        # Use a random threshold to prevent beating DB updates.
+        # It generates a distribution between [40m, 80m].
+        #
+        contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
+
+        current_runner.contacted_at.nil? ||
+          (Time.now - current_runner.contacted_at) >= contacted_at_max_age
+      end
+
+      def job_not_found!
+        if headers['User-Agent'].to_s =~ /gitlab(-ci-multi)?-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /
+          no_content!
+        else
+          not_found!
+        end
+      end
+
+      def validate_job!(job)
+        not_found! unless job
+
+        yield if block_given?
+
+        forbidden!('Project has been deleted!') unless job.project
+        forbidden!('Job has been erased!') if job.erased?
+      end
+
+      def authenticate_job!(job)
+        validate_job!(job) do
+          forbidden! unless job_token_valid?(job)
+        end
+      end
+
+      def job_token_valid?(job)
+        token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s
+        token && job.valid_token?(token)
+      end
+
+      def max_artifacts_size
+        current_application_settings.max_artifacts_size.megabytes.to_i
+      end
     end
   end
 end

lib/api/runner.rb

@@ -48,5 +48,203 @@ module API
         Ci::Runner.find_by_token(params[:token]).destroy
       end
     end
+
+    resource :jobs do
+      desc 'Request a job' do
+        success Entities::JobRequest::Response
+      end
+      params do
+        requires :token, type: String, desc: %q(Runner's authentication token)
+        optional :last_update, type: String, desc: %q(Runner's queue last_update token)
+        optional :info, type: Hash, desc: %q(Runner's metadata)
+      end
+      post '/request' do
+        authenticate_runner!
+        not_found! unless current_runner.active?
+        update_runner_info
+
+        if current_runner.is_runner_queue_value_latest?(params[:last_update])
+          header 'X-GitLab-Last-Update', params[:last_update]
+          Gitlab::Metrics.add_event(:build_not_found_cached)
+          return job_not_found!
+        end
+
+        new_update = current_runner.ensure_runner_queue_value
+        result = ::Ci::RegisterJobService.new(current_runner).execute
+
+        if result.valid?
+          if result.build
+            Gitlab::Metrics.add_event(:build_found,
+                                      project: result.build.project.path_with_namespace)
+            present result.build, with: Entities::JobRequest::Response
+          else
+            Gitlab::Metrics.add_event(:build_not_found)
+            header 'X-GitLab-Last-Update', new_update
+            job_not_found!
+          end
+        else
+          # We received build that is invalid due to concurrency conflict
+          Gitlab::Metrics.add_event(:build_invalid)
+          conflict!
+        end
+      end
+
+      desc 'Updates a job' do
+        http_codes [[200, 'Job was updated'], [403, 'Forbidden']]
+      end
+      params do
+        requires :token, type: String, desc: %q(Runners's authentication token)
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :trace, type: String, desc: %q(Job's full trace)
+        optional :state, type: String, desc: %q(Job's status: success, failed)
+      end
+      put '/:id' do
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+
+        job.update_attributes(trace: params[:trace]) if params[:trace]
+
+        Gitlab::Metrics.add_event(:update_build,
+                                  project: job.project.path_with_namespace)
+
+        case params[:state].to_s
+        when 'success'
+          job.success
+        when 'failed'
+          job.drop
+        end
+      end
+
+      desc 'Appends a patch to the job trace' do
+        http_codes [[202, 'Trace was patched'],
+                    [400, 'Missing Content-Range header'],
+                    [403, 'Forbidden'],
+                    [416, 'Range not satisfiable']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+      end
+      patch '/:id/trace' do
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+
+        error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range')
+        content_range = request.headers['Content-Range']
+        content_range = content_range.split('-')
+
+        current_length = job.trace_length
+        unless current_length == content_range[0].to_i
+          return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
+        end
+
+        job.append_trace(request.body.read, content_range[0].to_i)
+
+        status 202
+        header 'Job-Status', job.status
+        header 'Range', "0-#{job.trace_length}"
+      end
+
+      desc 'Authorize artifacts uploading for job' do
+        http_codes [[200, 'Upload allowed'],
+                    [403, 'Forbidden'],
+                    [405, 'Artifacts support not enabled'],
+                    [413, 'File too large']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+        optional :filesize, type: Integer, desc: %q(Artifacts filesize)
+      end
+      post '/:id/artifacts/authorize' do
+        not_allowed! unless Gitlab.config.artifacts.enabled
+        require_gitlab_workhorse!
+        Gitlab::Workhorse.verify_api_request!(headers)
+
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+        forbidden!('Job is not running') unless job.running?
+
+        if params[:filesize]
+          file_size = params[:filesize].to_i
+          file_to_large! unless file_size < max_artifacts_size
+        end
+
+        status 200
+        content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
+        Gitlab::Workhorse.artifact_upload_ok
+      end
+
+      desc 'Upload artifacts for job' do
+        success Entities::JobRequest::Response
+        http_codes [[201, 'Artifact uploaded'],
+                    [400, 'Bad request'],
+                    [403, 'Forbidden'],
+                    [405, 'Artifacts support not enabled'],
+                    [413, 'File too large']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+        optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
+        optional :file, type: File, desc: %q(Artifact's file)
+        optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
+        optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
+        optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
+        optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
+        optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
+      end
+      post '/:id/artifacts' do
+        not_allowed! unless Gitlab.config.artifacts.enabled
+        require_gitlab_workhorse!
+
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+        forbidden!('Job is not running!') unless job.running?
+
+        artifacts_upload_path = ArtifactUploader.artifacts_upload_path
+        artifacts = uploaded_file(:file, artifacts_upload_path)
+        metadata = uploaded_file(:metadata, artifacts_upload_path)
+
+        bad_request!('Missing artifacts file!') unless artifacts
+        file_to_large! unless artifacts.size < max_artifacts_size
+
+        job.artifacts_file = artifacts
+        job.artifacts_metadata = metadata
+        job.artifacts_expire_in = params['expire_in'] ||
+          Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
+
+        if job.save
+          present job, with: Entities::JobRequest::Response
+        else
+          render_validation_error!(job)
+        end
+      end
+
+      desc 'Download the artifacts file for job' do
+        http_codes [[200, 'Upload allowed'],
+                    [403, 'Forbidden'],
+                    [404, 'Artifact not found']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+      end
+      get '/:id/artifacts' do
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+
+        artifacts_file = job.artifacts_file
+        unless artifacts_file.file_storage?
+          return redirect_to job.artifacts_file.url
+        end
+
+        unless artifacts_file.exists?
+          not_found!
+        end
+
+        present_file!(artifacts_file.path, artifacts_file.filename)
+      end
+    end
   end
 end

lib/ci/api/builds.rb

@@ -24,7 +24,7 @@ module Ci
 
           new_update = current_runner.ensure_runner_queue_value
 
-          result = Ci::RegisterBuildService.new(current_runner).execute
+          result = Ci::RegisterJobService.new(current_runner).execute
 
           if result.valid?
             if result.build

lib/gitlab/ci/build/image.rb

+module Gitlab
+  module Ci
+    module Build
+      class Image
+        attr_reader :name
+
+        class << self
+          def from_image(job)
+            image = Gitlab::Ci::Build::Image.new(job.options[:image])
+            return unless image.valid?
+            image
+          end
+
+          def from_services(job)
+            services = job.options[:services].to_a.map do |service|
+              Gitlab::Ci::Build::Image.new(service)
+            end
+
+            services.select(&:valid?).compact
+          end
+        end
+
+        def initialize(image)
+          @name = image
+        end
+
+        def valid?
+          @name.present?
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/build/step.rb

+module Gitlab
+  module Ci
+    module Build
+      class Step
+        WHEN_ON_FAILURE = 'on_failure'.freeze
+        WHEN_ON_SUCCESS = 'on_success'.freeze
+        WHEN_ALWAYS = 'always'.freeze
+
+        attr_reader :name
+        attr_writer :script
+        attr_accessor :timeout, :when, :allow_failure
+
+        class << self
+          def from_commands(job)
+            self.new(:script).tap do |step|
+              step.script = job.commands
+              step.timeout = job.timeout
+              step.when = WHEN_ON_SUCCESS
+            end
+          end
+
+          def from_after_script(job)
+            after_script = job.options[:after_script]
+            return unless after_script
+
+            self.new(:after_script).tap do |step|
+              step.script = after_script
+              step.timeout = job.timeout
+              step.when = WHEN_ALWAYS
+              step.allow_failure = true
+            end
+          end
+        end
+
+        def initialize(name)
+          @name = name
+          @allow_failure = false
+        end
+
+        def script
+          @script.split("\n")
+        end
+      end
+    end
+  end
+end

spec/factories/ci/builds.rb

@@ -15,8 +15,8 @@ FactoryGirl.define do
 
     options do
       {
-        image: "ruby:2.1",
-        services: ["postgres"]
+        image: 'ruby:2.1',
+        services: ['postgres']
       }
     end
 
@@ -166,5 +166,31 @@ FactoryGirl.define do
         allow(build).to receive(:commit).and_return build(:commit)
       end
     end
+
+    trait :extended_options do
+      options do
+        {
+            image: 'ruby:2.1',
+            services: ['postgres'],
+            after_script: "ls\ndate",
+            artifacts: {
+                name: 'artifacts_file',
+                untracked: false,
+                paths: ['out/'],
+                when: 'always',
+                expire_in: '7d'
+            },
+            cache: {
+                key: 'cache_key',
+                untracked: false,
+                paths: ['vendor/*']
+            }
+        }
+      end
+    end
+
+    trait :no_options do
+      options { {} }
+    end
   end
 end

spec/lib/gitlab/ci/build/image_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Image do
+  let(:job) { create(:ci_build, :no_options) }
+
+  describe '#from_image' do
+    subject { described_class.from_image(job) }
+
+    context 'when image is defined in job' do
+      let(:image_name) { 'ruby:2.1' }
+      let(:job) { create(:ci_build, options: { image: image_name } ) }
+
+      it 'fabricates an object of the proper class' do
+        is_expected.to be_kind_of(described_class)
+      end
+
+      it 'populates fabricated object with the proper name attribute' do
+        expect(subject.name).to eq(image_name)
+      end
+
+      context 'when image name is empty' do
+        let(:image_name) { '' }
+
+        it 'does not fabricate an object' do
+          is_expected.to be_nil
+        end
+      end
+    end
+
+    context 'when image is not defined in job' do
+      it 'does not fabricate an object' do
+        is_expected.to be_nil
+      end
+    end
+  end
+
+  describe '#from_services' do
+    subject { described_class.from_services(job) }
+
+    context 'when services are defined in job' do
+      let(:service_image_name) { 'postgres' }
+      let(:job) { create(:ci_build, options: { services: [service_image_name] }) }
+
+      it 'fabricates an non-empty array of objects' do
+        is_expected.to be_kind_of(Array)
+        is_expected.not_to be_empty
+        expect(subject.first.name).to eq(service_image_name)
+      end
+
+      context 'when service image name is empty' do
+        let(:service_image_name) { '' }
+
+        it 'fabricates an empty array' do
+          is_expected.to be_kind_of(Array)
+          is_expected.to be_empty
+        end
+      end
+    end
+
+    context 'when services are not defined in job' do
+      it 'fabricates an empty array' do
+        is_expected.to be_kind_of(Array)
+        is_expected.to be_empty
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/build/step_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Step do
+  let(:job) { create(:ci_build, :no_options, commands: "ls -la\ndate") }
+
+  describe '#from_commands' do
+    subject { described_class.from_commands(job) }
+
+    it 'fabricates an object' do
+      expect(subject.name).to eq(:script)
+      expect(subject.script).to eq(['ls -la', 'date'])
+      expect(subject.timeout).to eq(job.timeout)
+      expect(subject.when).to eq('on_success')
+      expect(subject.allow_failure).to be_falsey
+    end
+  end
+
+  describe '#from_after_script' do
+    subject { described_class.from_after_script(job) }
+
+    context 'when after_script is empty' do
+      it 'doesn not fabricate an object' do
+        is_expected.to be_nil
+      end
+    end
+
+    context 'when after_script is not empty' do
+      let(:job) { create(:ci_build, options: { after_script: "ls -la\ndate" }) }
+
+      it 'fabricates an object' do
+        expect(subject.name).to eq(:after_script)
+        expect(subject.script).to eq(['ls -la', 'date'])
+        expect(subject.timeout).to eq(job.timeout)
+        expect(subject.when).to eq('always')
+        expect(subject.allow_failure).to be_truthy
+      end
+    end
+  end
+end

spec/requests/api/runner_spec.rb

@@ -16,6 +16,7 @@ describe API::Runner do
       context 'when no token is provided' do
         it 'returns 400 error' do
           post api('/runners')
+
           expect(response).to have_http_status 400
         end
       end
@@ -23,6 +24,7 @@ describe API::Runner do
       context 'when invalid token is provided' do
         it 'returns 403 error' do
           post api('/runners'), token: 'invalid'
+
           expect(response).to have_http_status 403
         end
       end
@@ -108,7 +110,7 @@ describe API::Runner do
         context "when info parameter '#{param}' info is present" do
           let(:value) { "#{param}_value" }
 
-          it %q(updates provided Runner's parameter) do
+          it "updates provided Runner's parameter" do
             post api('/runners'), token: registration_token,
                                   info: { param => value }
 
@@ -148,4 +150,874 @@ describe API::Runner do
       end
     end
   end
+
+  describe '/api/v4/jobs' do
+    let(:project) { create(:empty_project, shared_runners_enabled: false) }
+    let(:pipeline) { create(:ci_pipeline_without_jobs, project: project, ref: 'master') }
+    let(:runner) { create(:ci_runner) }
+    let!(:job) do
+      create(:ci_build, :artifacts, :extended_options,
+             pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0, commands: "ls\ndate")
+    end
+
+    before { project.runners << runner }
+
+    describe 'POST /api/v4/jobs/request' do
+      let!(:last_update) {}
+      let!(:new_update) { }
+      let(:user_agent) { 'gitlab-runner 9.0.0 (9-0-stable; go1.7.4; linux/amd64)' }
+
+      before { stub_container_registry_config(enabled: false) }
+
+      shared_examples 'no jobs available' do
+        before { request_job }
+
+        context 'when runner sends version in User-Agent' do
+          context 'for stable version' do
+            it 'gives 204 and set X-GitLab-Last-Update' do
+              expect(response).to have_http_status(204)
+              expect(response.header).to have_key('X-GitLab-Last-Update')
+            end
+          end
+
+          context 'when last_update is up-to-date' do
+            let(:last_update) { runner.ensure_runner_queue_value }
+
+            it 'gives 204 and set the same X-GitLab-Last-Update' do
+              expect(response).to have_http_status(204)
+              expect(response.header['X-GitLab-Last-Update']).to eq(last_update)
+            end
+          end
+
+          context 'when last_update is outdated' do
+            let(:last_update) { runner.ensure_runner_queue_value }
+            let(:new_update) { runner.tick_runner_queue }
+
+            it 'gives 204 and set a new X-GitLab-Last-Update' do
+              expect(response).to have_http_status(204)
+              expect(response.header['X-GitLab-Last-Update']).to eq(new_update)
+            end
+          end
+
+          context 'when beta version is sent' do
+            let(:user_agent) { 'gitlab-runner 9.0.0~beta.167.g2b2bacc (master; go1.7.4; linux/amd64)' }
+
+            it { expect(response).to have_http_status(204) }
+          end
+
+          context 'when pre-9-0 version is sent' do
+            let(:user_agent) { 'gitlab-ci-multi-runner 1.6.0 (1-6-stable; go1.6.3; linux/amd64)' }
+
+            it { expect(response).to have_http_status(204) }
+          end
+
+          context 'when pre-9-0 beta version is sent' do
+            let(:user_agent) { 'gitlab-ci-multi-runner 1.6.0~beta.167.g2b2bacc (master; go1.6.3; linux/amd64)' }
+
+            it { expect(response).to have_http_status(204) }
+          end
+        end
+
+        context "when runner doesn't send version in User-Agent" do
+          let(:user_agent) { 'Go-http-client/1.1' }
+
+          it { expect(response).to have_http_status(404) }
+        end
+
+        context "when runner doesn't have a User-Agent" do
+          let(:user_agent) { nil }
+
+          it { expect(response).to have_http_status(404) }
+        end
+      end
+
+      context 'when no token is provided' do
+        it 'returns 400 error' do
+          post api('/jobs/request')
+
+          expect(response).to have_http_status 400
+        end
+      end
+
+      context 'when invalid token is provided' do
+        it 'returns 403 error' do
+          post api('/jobs/request'), token: 'invalid'
+
+          expect(response).to have_http_status 403
+        end
+      end
+
+      context 'when valid token is provided' do
+        context 'when Runner is not active' do
+          let(:runner) { create(:ci_runner, :inactive) }
+
+          it 'returns 404 error' do
+            request_job
+
+            expect(response).to have_http_status 404
+          end
+        end
+
+        context 'when jobs are finished' do
+          before { job.success }
+
+          it_behaves_like 'no jobs available'
+        end
+
+        context 'when other projects have pending jobs' do
+          before do
+            job.success
+            create(:ci_build, :pending)
+          end
+
+          it_behaves_like 'no jobs available'
+        end
+
+        context 'when shared runner requests job for project without shared_runners_enabled' do
+          let(:runner) { create(:ci_runner, :shared) }
+
+          it_behaves_like 'no jobs available'
+        end
+
+        context 'when there is a pending job' do
+          let(:expected_job_info) do
+            { 'name' => job.name,
+              'stage' => job.stage,
+              'project_id' => job.project.id,
+              'project_name' => job.project.name }
+          end
+
+          let(:expected_git_info) do
+            { 'repo_url' => job.repo_url,
+              'ref' => job.ref,
+              'sha' => job.sha,
+              'before_sha' => job.before_sha,
+              'ref_type' => 'branch' }
+          end
+
+          let(:expected_steps) do
+            [{ 'name' => 'script',
+               'script' => %w(ls date),
+               'timeout' => job.timeout,
+               'when' => 'on_success',
+               'allow_failure' => false },
+             { 'name' => 'after_script',
+               'script' => %w(ls date),
+               'timeout' => job.timeout,
+               'when' => 'always',
+               'allow_failure' => true }]
+          end
+
+          let(:expected_variables) do
+            [{ 'key' => 'CI_BUILD_NAME', 'value' => 'spinach', 'public' => true },
+             { 'key' => 'CI_BUILD_STAGE', 'value' => 'test', 'public' => true },
+             { 'key' => 'DB_NAME', 'value' => 'postgres', 'public' => true }]
+          end
+
+          let(:expected_artifacts) do
+            [{ 'name' => 'artifacts_file',
+               'untracked' => false,
+               'paths' => %w(out/),
+               'when' => 'always',
+               'expire_in' => '7d' }]
+          end
+
+          let(:expected_cache) do
+            [{ 'key' => 'cache_key',
+               'untracked' => false,
+               'paths' => ['vendor/*'] }]
+          end
+
+          it 'picks a job' do
+            request_job info: { platform: :darwin }
+
+            expect(response).to have_http_status(201)
+            expect(response.headers).not_to have_key('X-GitLab-Last-Update')
+            expect(runner.reload.platform).to eq('darwin')
+            expect(json_response['id']).to eq(job.id)
+            expect(json_response['token']).to eq(job.token)
+            expect(json_response['job_info']).to eq(expected_job_info)
+            expect(json_response['git_info']).to eq(expected_git_info)
+            expect(json_response['image']).to eq({ 'name' => 'ruby:2.1' })
+            expect(json_response['services']).to eq([{ 'name' => 'postgres' }])
+            expect(json_response['steps']).to eq(expected_steps)
+            expect(json_response['artifacts']).to eq(expected_artifacts)
+            expect(json_response['cache']).to eq(expected_cache)
+            expect(json_response['variables']).to include(*expected_variables)
+          end
+
+          context 'when job is made for tag' do
+            let!(:job) { create(:ci_build_tag, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0) }
+
+            it 'sets branch as ref_type' do
+              request_job
+
+              expect(response).to have_http_status(201)
+              expect(json_response['git_info']['ref_type']).to eq('tag')
+            end
+          end
+
+          context 'when job is made for branch' do
+            it 'sets tag as ref_type' do
+              request_job
+
+              expect(response).to have_http_status(201)
+              expect(json_response['git_info']['ref_type']).to eq('branch')
+            end
+          end
+
+          it 'updates runner info' do
+            expect { request_job }.to change { runner.reload.contacted_at }
+          end
+
+          %w(name version revision platform architecture).each do |param|
+            context "when info parameter '#{param}' is present" do
+              let(:value) { "#{param}_value" }
+
+              it "updates provided Runner's parameter" do
+                request_job info: { param => value }
+
+                expect(response).to have_http_status(201)
+                expect(runner.reload.read_attribute(param.to_sym)).to eq(value)
+              end
+            end
+          end
+
+          context 'when concurrently updating a job' do
+            before do
+              expect_any_instance_of(Ci::Build).to receive(:run!).
+                  and_raise(ActiveRecord::StaleObjectError.new(nil, nil))
+            end
+
+            it 'returns a conflict' do
+              request_job
+
+              expect(response).to have_http_status(409)
+              expect(response.headers).not_to have_key('X-GitLab-Last-Update')
+            end
+          end
+
+          context 'when project and pipeline have multiple jobs' do
+            let!(:test_job) { create(:ci_build, pipeline: pipeline, name: 'deploy', stage: 'deploy', stage_idx: 1) }
+
+            before { job.success }
+
+            it 'returns dependent jobs' do
+              request_job
+
+              expect(response).to have_http_status(201)
+              expect(json_response['id']).to eq(test_job.id)
+              expect(json_response['dependencies'].count).to eq(1)
+              expect(json_response['dependencies'][0]).to include('id' => job.id, 'name' => 'spinach')
+            end
+          end
+
+          context 'when job has no tags' do
+            before { job.update(tags: []) }
+
+            context 'when runner is allowed to pick untagged jobs' do
+              before { runner.update_column(:run_untagged, true) }
+
+              it 'picks job' do
+                request_job
+
+                expect(response).to have_http_status 201
+              end
+            end
+
+            context 'when runner is not allowed to pick untagged jobs' do
+              before { runner.update_column(:run_untagged, false) }
+
+              it_behaves_like 'no jobs available'
+            end
+          end
+
+          context 'when triggered job is available' do
+            let(:expected_variables) do
+              [{ 'key' => 'CI_BUILD_NAME', 'value' => 'spinach', 'public' => true },
+               { 'key' => 'CI_BUILD_STAGE', 'value' => 'test', 'public' => true },
+               { 'key' => 'CI_BUILD_TRIGGERED', 'value' => 'true', 'public' => true },
+               { 'key' => 'DB_NAME', 'value' => 'postgres', 'public' => true },
+               { 'key' => 'SECRET_KEY', 'value' => 'secret_value', 'public' => false },
+               { 'key' => 'TRIGGER_KEY_1', 'value' => 'TRIGGER_VALUE_1', 'public' => false }]
+            end
+
+            before do
+              trigger = create(:ci_trigger, project: project)
+              create(:ci_trigger_request_with_variables, pipeline: pipeline, builds: [job], trigger: trigger)
+              project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value')
+            end
+
+            it 'returns variables for triggers' do
+              request_job
+
+              expect(response).to have_http_status(201)
+              expect(json_response['variables']).to include(*expected_variables)
+            end
+          end
+
+          describe 'registry credentials support' do
+            let(:registry_url) { 'registry.example.com:5005' }
+            let(:registry_credentials) do
+              { 'type' => 'registry',
+                'url' => registry_url,
+                'username' => 'gitlab-ci-token',
+                'password' => job.token }
+            end
+
+            context 'when registry is enabled' do
+              before { stub_container_registry_config(enabled: true, host_port: registry_url) }
+
+              it 'sends registry credentials key' do
+                request_job
+
+                expect(json_response).to have_key('credentials')
+                expect(json_response['credentials']).to include(registry_credentials)
+              end
+            end
+
+            context 'when registry is disabled' do
+              before { stub_container_registry_config(enabled: false, host_port: registry_url) }
+
+              it 'does not send registry credentials' do
+                request_job
+
+                expect(json_response).to have_key('credentials')
+                expect(json_response['credentials']).not_to include(registry_credentials)
+              end
+            end
+          end
+        end
+
+        def request_job(token = runner.token, **params)
+          new_params = params.merge(token: token, last_update: last_update)
+          post api('/jobs/request'), new_params, { 'User-Agent' => user_agent }
+        end
+      end
+    end
+
+    describe 'PUT /api/v4/jobs/:id' do
+      let(:job) { create(:ci_build, :pending, :trace, pipeline: pipeline, runner_id: runner.id) }
+
+      before { job.run! }
+
+      context 'when status is given' do
+        it 'mark job as succeeded' do
+          update_job(state: 'success')
+
+          expect(job.reload.status).to eq 'success'
+        end
+
+        it 'mark job as failed' do
+          update_job(state: 'failed')
+
+          expect(job.reload.status).to eq 'failed'
+        end
+      end
+
+      context 'when tace is given' do
+        it 'updates a running build' do
+          update_job(trace: 'BUILD TRACE UPDATED')
+
+          expect(response).to have_http_status(200)
+          expect(job.reload.trace).to eq 'BUILD TRACE UPDATED'
+        end
+      end
+
+      context 'when no trace is given' do
+        it 'does not override trace information' do
+          update_job
+
+          expect(job.reload.trace).to eq 'BUILD TRACE'
+        end
+      end
+
+      context 'when job has been erased' do
+        let(:job) { create(:ci_build, runner_id: runner.id, erased_at: Time.now) }
+
+        it 'responds with forbidden' do
+          update_job
+
+          expect(response).to have_http_status(403)
+        end
+      end
+
+      def update_job(token = job.token, **params)
+        new_params = params.merge(token: token)
+        put api("/jobs/#{job.id}"), new_params
+      end
+    end
+
+    describe 'PATCH /api/v4/jobs/:id/trace' do
+      let(:job) { create(:ci_build, :running, :trace, runner_id: runner.id, pipeline: pipeline) }
+      let(:headers) { { API::Helpers::Runner::JOB_TOKEN_HEADER => job.token, 'Content-Type' => 'text/plain' } }
+      let(:headers_with_range) { headers.merge({ 'Content-Range' => '11-20' }) }
+      let(:update_interval) { 10.seconds.to_i }
+
+      before { initial_patch_the_trace }
+
+      context 'when request is valid' do
+        it 'gets correct response' do
+          expect(response.status).to eq 202
+          expect(job.reload.trace).to eq 'BUILD TRACE appended'
+          expect(response.header).to have_key 'Range'
+          expect(response.header).to have_key 'Job-Status'
+        end
+
+        context 'when job has been updated recently' do
+          it { expect{ patch_the_trace }.not_to change { job.updated_at }}
+
+          it "changes the job's trace" do
+            patch_the_trace
+
+            expect(job.reload.trace).to eq 'BUILD TRACE appended appended'
+          end
+
+          context 'when Runner makes a force-patch' do
+            it { expect{ force_patch_the_trace }.not_to change { job.updated_at }}
+
+            it "doesn't change the build.trace" do
+              force_patch_the_trace
+
+              expect(job.reload.trace).to eq 'BUILD TRACE appended'
+            end
+          end
+        end
+
+        context 'when job was not updated recently' do
+          let(:update_interval) { 15.minutes.to_i }
+
+          it { expect { patch_the_trace }.to change { job.updated_at } }
+
+          it 'changes the job.trace' do
+            patch_the_trace
+
+            expect(job.reload.trace).to eq 'BUILD TRACE appended appended'
+          end
+
+          context 'when Runner makes a force-patch' do
+            it { expect { force_patch_the_trace }.to change { job.updated_at } }
+
+            it "doesn't change the job.trace" do
+              force_patch_the_trace
+
+              expect(job.reload.trace).to eq 'BUILD TRACE appended'
+            end
+          end
+        end
+
+        context 'when project for the build has been deleted' do
+          let(:job) do
+            create(:ci_build, :running, :trace, runner_id: runner.id, pipeline: pipeline) do |job|
+              job.project.update(pending_delete: true)
+            end
+          end
+
+          it 'responds with forbidden' do
+            expect(response.status).to eq(403)
+          end
+        end
+      end
+
+      context 'when Runner makes a force-patch' do
+        before do
+          force_patch_the_trace
+        end
+
+        it 'gets correct response' do
+          expect(response.status).to eq 202
+          expect(job.reload.trace).to eq 'BUILD TRACE appended'
+          expect(response.header).to have_key 'Range'
+          expect(response.header).to have_key 'Job-Status'
+        end
+      end
+
+      context 'when content-range start is too big' do
+        let(:headers_with_range) { headers.merge({ 'Content-Range' => '15-20' }) }
+
+        it 'gets 416 error response with range headers' do
+          expect(response.status).to eq 416
+          expect(response.header).to have_key 'Range'
+          expect(response.header['Range']).to eq '0-11'
+        end
+      end
+
+      context 'when content-range start is too small' do
+        let(:headers_with_range) { headers.merge({ 'Content-Range' => '8-20' }) }
+
+        it 'gets 416 error response with range headers' do
+          expect(response.status).to eq 416
+          expect(response.header).to have_key 'Range'
+          expect(response.header['Range']).to eq '0-11'
+        end
+      end
+
+      context 'when Content-Range header is missing' do
+        let(:headers_with_range) { headers }
+
+        it { expect(response.status).to eq 400 }
+      end
+
+      context 'when job has been errased' do
+        let(:job) { create(:ci_build, runner_id: runner.id, erased_at: Time.now) }
+
+        it { expect(response.status).to eq 403 }
+      end
+
+      def patch_the_trace(content = ' appended', request_headers = nil)
+        unless request_headers
+          offset = job.trace_length
+          limit = offset + content.length - 1
+          request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" })
+        end
+
+        Timecop.travel(job.updated_at + update_interval) do
+          patch api("/jobs/#{job.id}/trace"), content, request_headers
+          job.reload
+        end
+      end
+
+      def initial_patch_the_trace
+        patch_the_trace(' appended', headers_with_range)
+      end
+
+      def force_patch_the_trace
+        2.times { patch_the_trace('') }
+      end
+    end
+
+    describe 'artifacts' do
+      let(:job) { create(:ci_build, :pending, pipeline: pipeline, runner_id: runner.id) }
+      let(:jwt_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') }
+      let(:headers) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => jwt_token } }
+      let(:headers_with_token) { headers.merge(API::Helpers::Runner::JOB_TOKEN_HEADER => job.token) }
+      let(:file_upload) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
+      let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') }
+
+      before { job.run! }
+
+      describe 'POST /api/v4/jobs/:id/artifacts/authorize' do
+        context 'when using token as parameter' do
+          it 'authorizes posting artifacts to running job' do
+            authorize_artifacts_with_token_in_params
+
+            expect(response).to have_http_status(200)
+            expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
+            expect(json_response['TempPath']).not_to be_nil
+          end
+
+          it 'fails to post too large artifact' do
+            stub_application_setting(max_artifacts_size: 0)
+
+            authorize_artifacts_with_token_in_params(filesize: 100)
+
+            expect(response).to have_http_status(413)
+          end
+        end
+
+        context 'when using token as header' do
+          it 'authorizes posting artifacts to running job' do
+            authorize_artifacts_with_token_in_headers
+
+            expect(response).to have_http_status(200)
+            expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
+            expect(json_response['TempPath']).not_to be_nil
+          end
+
+          it 'fails to post too large artifact' do
+            stub_application_setting(max_artifacts_size: 0)
+
+            authorize_artifacts_with_token_in_headers(filesize: 100)
+
+            expect(response).to have_http_status(413)
+          end
+        end
+
+        context 'when using runners token' do
+          it 'fails to authorize artifacts posting' do
+            authorize_artifacts(token: job.project.runners_token)
+
+            expect(response).to have_http_status(403)
+          end
+        end
+
+        it 'reject requests that did not go through gitlab-workhorse' do
+          headers.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER)
+
+          authorize_artifacts
+
+          expect(response).to have_http_status(500)
+        end
+
+        context 'authorization token is invalid' do
+          it 'responds with forbidden' do
+            authorize_artifacts(token: 'invalid', filesize: 100 )
+
+            expect(response).to have_http_status(403)
+          end
+        end
+
+        def authorize_artifacts(params = {}, request_headers = headers)
+          post api("/jobs/#{job.id}/artifacts/authorize"), params, request_headers
+        end
+
+        def authorize_artifacts_with_token_in_params(params = {}, request_headers = headers)
+          params = params.merge(token: job.token)
+          authorize_artifacts(params, request_headers)
+        end
+
+        def authorize_artifacts_with_token_in_headers(params = {}, request_headers = headers_with_token)
+          authorize_artifacts(params, request_headers)
+        end
+      end
+
+      describe 'POST /api/v4/jobs/:id/artifacts' do
+        context 'when artifacts are being stored inside of tmp path' do
+          before do
+            # by configuring this path we allow to pass temp file from any path
+            allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return('/')
+          end
+
+          context 'when job has been erased' do
+            let(:job) { create(:ci_build, erased_at: Time.now) }
+
+            before do
+              upload_artifacts(file_upload, headers_with_token)
+            end
+
+            it 'responds with forbidden' do
+              upload_artifacts(file_upload, headers_with_token)
+
+              expect(response).to have_http_status(403)
+            end
+          end
+
+          context 'when job is running' do
+            shared_examples 'successful artifacts upload' do
+              it 'updates successfully' do
+                expect(response).to have_http_status(201)
+              end
+            end
+
+            context 'when uses regular file post' do
+              before { upload_artifacts(file_upload, headers_with_token, false) }
+
+              it_behaves_like 'successful artifacts upload'
+            end
+
+            context 'when uses accelerated file post' do
+              before { upload_artifacts(file_upload, headers_with_token, true) }
+
+              it_behaves_like 'successful artifacts upload'
+            end
+
+            context 'when updates artifact' do
+              before do
+                upload_artifacts(file_upload2, headers_with_token)
+                upload_artifacts(file_upload, headers_with_token)
+              end
+
+              it_behaves_like 'successful artifacts upload'
+            end
+
+            context 'when using runners token' do
+              it 'responds with forbidden' do
+                upload_artifacts(file_upload, headers.merge(API::Helpers::Runner::JOB_TOKEN_HEADER => job.project.runners_token))
+
+                expect(response).to have_http_status(403)
+              end
+            end
+          end
+
+          context 'when artifacts file is too large' do
+            it 'fails to post too large artifact' do
+              stub_application_setting(max_artifacts_size: 0)
+
+              upload_artifacts(file_upload, headers_with_token)
+
+              expect(response).to have_http_status(413)
+            end
+          end
+
+          context 'when artifacts post request does not contain file' do
+            it 'fails to post artifacts without file' do
+              post api("/jobs/#{job.id}/artifacts"), {}, headers_with_token
+
+              expect(response).to have_http_status(400)
+            end
+          end
+
+          context 'GitLab Workhorse is not configured' do
+            it 'fails to post artifacts without GitLab-Workhorse' do
+              post api("/jobs/#{job.id}/artifacts"), { token: job.token }, {}
+
+              expect(response).to have_http_status(403)
+            end
+          end
+
+          context 'when setting an expire date' do
+            let(:default_artifacts_expire_in) {}
+            let(:post_data) do
+              { 'file.path' => file_upload.path,
+                'file.name' => file_upload.original_filename,
+                'expire_in' => expire_in }
+            end
+
+            before do
+              stub_application_setting(default_artifacts_expire_in: default_artifacts_expire_in)
+
+              post(api("/jobs/#{job.id}/artifacts"), post_data, headers_with_token)
+            end
+
+            context 'when an expire_in is given' do
+              let(:expire_in) { '7 days' }
+
+              it 'updates when specified' do
+                expect(response).to have_http_status(201)
+                expect(job.reload.artifacts_expire_at).to be_within(5.minutes).of(7.days.from_now)
+              end
+            end
+
+            context 'when no expire_in is given' do
+              let(:expire_in) { nil }
+
+              it 'ignores if not specified' do
+                expect(response).to have_http_status(201)
+                expect(job.reload.artifacts_expire_at).to be_nil
+              end
+
+              context 'with application default' do
+                context 'when default is 5 days' do
+                  let(:default_artifacts_expire_in) { '5 days' }
+
+                  it 'sets to application default' do
+                    expect(response).to have_http_status(201)
+                    expect(job.reload.artifacts_expire_at).to be_within(5.minutes).of(5.days.from_now)
+                  end
+                end
+
+                context 'when default is 0' do
+                  let(:default_artifacts_expire_in) { '0' }
+
+                  it 'does not set expire_in' do
+                    expect(response).to have_http_status(201)
+                    expect(job.reload.artifacts_expire_at).to be_nil
+                  end
+                end
+              end
+            end
+          end
+
+          context 'posts artifacts file and metadata file' do
+            let!(:artifacts) { file_upload }
+            let!(:metadata) { file_upload2 }
+
+            let(:stored_artifacts_file) { job.reload.artifacts_file.file }
+            let(:stored_metadata_file) { job.reload.artifacts_metadata.file }
+            let(:stored_artifacts_size) { job.reload.artifacts_size }
+
+            before do
+              post(api("/jobs/#{job.id}/artifacts"), post_data, headers_with_token)
+            end
+
+            context 'when posts data accelerated by workhorse is correct' do
+              let(:post_data) do
+                { 'file.path' => artifacts.path,
+                  'file.name' => artifacts.original_filename,
+                  'metadata.path' => metadata.path,
+                  'metadata.name' => metadata.original_filename }
+              end
+
+              it 'stores artifacts and artifacts metadata' do
+                expect(response).to have_http_status(201)
+                expect(stored_artifacts_file.original_filename).to eq(artifacts.original_filename)
+                expect(stored_metadata_file.original_filename).to eq(metadata.original_filename)
+                expect(stored_artifacts_size).to eq(71759)
+              end
+            end
+
+            context 'when there is no artifacts file in post data' do
+              let(:post_data) do
+                { 'metadata' => metadata }
+              end
+
+              it 'is expected to respond with bad request' do
+                expect(response).to have_http_status(400)
+              end
+
+              it 'does not store metadata' do
+                expect(stored_metadata_file).to be_nil
+              end
+            end
+          end
+        end
+
+        context 'when artifacts are being stored outside of tmp path' do
+          before do
+            # by configuring this path we allow to pass file from @tmpdir only
+            # but all temporary files are stored in system tmp directory
+            @tmpdir = Dir.mktmpdir
+            allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir)
+          end
+
+          after { FileUtils.remove_entry @tmpdir }
+
+          it' "fails to post artifacts for outside of tmp path"' do
+            upload_artifacts(file_upload, headers_with_token)
+
+            expect(response).to have_http_status(400)
+          end
+        end
+
+        def upload_artifacts(file, headers = {}, accelerated = true)
+          params = if accelerated
+                     { 'file.path' => file.path, 'file.name' => file.original_filename }
+                   else
+                     { 'file' => file }
+                   end
+          post api("/jobs/#{job.id}/artifacts"), params, headers
+        end
+      end
+
+      describe 'GET /api/v4/jobs/:id/artifacts' do
+        let(:token) { job.token }
+
+        before { download_artifact }
+
+        context 'when job has artifacts' do
+          let(:job) { create(:ci_build, :artifacts) }
+          let(:download_headers) do
+            { 'Content-Transfer-Encoding' => 'binary',
+              'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' }
+          end
+
+          context 'when using job token' do
+            it 'download artifacts' do
+              expect(response).to have_http_status(200)
+              expect(response.headers).to include download_headers
+            end
+          end
+
+          context 'when using runnners token' do
+            let(:token) { job.project.runners_token }
+
+            it 'responds with forbidden' do
+              expect(response).to have_http_status(403)
+            end
+          end
+        end
+
+        context 'when job does not has artifacts' do
+          it 'responds with not found' do
+            expect(response).to have_http_status(404)
+          end
+        end
+
+        def download_artifact(params = {}, request_headers = headers)
+          params = params.merge(token: token)
+          get api("/jobs/#{job.id}/artifacts"), params, request_headers
+        end
+      end
+    end
+  end
 end

spec/services/ci/register_build_service_spec.rb → spec/services/ci/register_job_service_spec.rb

 require 'spec_helper'
 
 module Ci
-  describe RegisterBuildService, services: true do
+  describe RegisterJobService, services: true do
     let!(:project) { FactoryGirl.create :empty_project, shared_runners_enabled: false }
     let!(:pipeline) { FactoryGirl.create :ci_pipeline, project: project }
     let!(:pending_build) { FactoryGirl.create :ci_build, pipeline: pipeline }
@@ -181,7 +181,7 @@ module Ci
           let!(:other_build) { create :ci_build, pipeline: pipeline }
 
           before do
-            allow_any_instance_of(Ci::RegisterBuildService).to receive(:builds_for_specific_runner)
+            allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
               .and_return([pending_build, other_build])
           end
 
@@ -193,7 +193,7 @@ module Ci
 
         context 'when single build is in queue' do
           before do
-            allow_any_instance_of(Ci::RegisterBuildService).to receive(:builds_for_specific_runner)
+            allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
               .and_return([pending_build])
           end
 
@@ -204,7 +204,7 @@ module Ci
 
         context 'when there is no build in queue' do
           before do
-            allow_any_instance_of(Ci::RegisterBuildService).to receive(:builds_for_specific_runner)
+            allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
               .and_return([])
           end