Commit fb4a4866 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'feature/runner-jobs-v4-api' into 'master'

Feature/runner jobs v4 api

Closes #28513

See merge request !9273
parents 7a774d1a 32b09b88
......@@ -517,6 +517,27 @@ module Ci
]
end
def steps
[Gitlab::Ci::Build::Step.from_commands(self),
Gitlab::Ci::Build::Step.from_after_script(self)].compact
end
def image
Gitlab::Ci::Build::Image.from_image(self)
end
def services
Gitlab::Ci::Build::Image.from_services(self)
end
def artifacts
[options[:artifacts]]
end
def cache
[options[:cache]]
end
def credentials
Gitlab::Ci::Build::Credentials::Factory.new(self).create!
end
......
module Ci
# This class responsible for assigning
# proper pending build to runner on runner API request
class RegisterBuildService
class RegisterJobService
include Gitlab::CurrentSettings
attr_reader :runner
......
---
title: Add Runner's jobs v4 API
merge_request: 9273
author:
......@@ -705,5 +705,83 @@ module API
expose :id, :message, :starts_at, :ends_at, :color, :font
expose :active?, as: :active
end
module JobRequest
class JobInfo < Grape::Entity
expose :name, :stage
expose :project_id, :project_name
end
class GitInfo < Grape::Entity
expose :repo_url, :ref, :sha, :before_sha
expose :ref_type do |model|
if model.tag
'tag'
else
'branch'
end
end
end
class RunnerInfo < Grape::Entity
expose :timeout
end
class Step < Grape::Entity
expose :name, :script, :timeout, :when, :allow_failure
end
class Image < Grape::Entity
expose :name
end
class Artifacts < Grape::Entity
expose :name, :untracked, :paths, :when, :expire_in
end
class Cache < Grape::Entity
expose :key, :untracked, :paths
end
class Credentials < Grape::Entity
expose :type, :url, :username, :password
end
class ArtifactFile < Grape::Entity
expose :filename, :size
end
class Dependency < Grape::Entity
expose :id, :name
expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? }
end
class Response < Grape::Entity
expose :id
expose :token
expose :allow_git_fetch
expose :job_info, using: JobInfo do |model|
model
end
expose :git_info, using: GitInfo do |model|
model
end
expose :runner_info, using: RunnerInfo do |model|
model
end
expose :variables
expose :steps, using: Step
expose :image, using: Image
expose :services, using: Image
expose :artifacts, using: Artifacts
expose :cache, using: Cache
expose :credentials, using: Credentials
expose :depends_on_builds, as: :dependencies, using: Dependency
end
end
end
end
module API
module Helpers
module Runner
JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze
JOB_TOKEN_PARAM = :token
UPDATE_RUNNER_EVERY = 10 * 60
def runner_registration_token_valid?
ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token],
current_application_settings.runners_registration_token)
......@@ -18,6 +22,56 @@ module API
def current_runner
@runner ||= ::Ci::Runner.find_by_token(params[:token].to_s)
end
def update_runner_info
return unless update_runner?
current_runner.contacted_at = Time.now
current_runner.assign_attributes(get_runner_version_from_params)
current_runner.save if current_runner.changed?
end
def update_runner?
# Use a random threshold to prevent beating DB updates.
# It generates a distribution between [40m, 80m].
#
contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
current_runner.contacted_at.nil? ||
(Time.now - current_runner.contacted_at) >= contacted_at_max_age
end
def job_not_found!
if headers['User-Agent'].to_s =~ /gitlab(-ci-multi)?-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /
no_content!
else
not_found!
end
end
def validate_job!(job)
not_found! unless job
yield if block_given?
forbidden!('Project has been deleted!') unless job.project
forbidden!('Job has been erased!') if job.erased?
end
def authenticate_job!(job)
validate_job!(job) do
forbidden! unless job_token_valid?(job)
end
end
def job_token_valid?(job)
token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s
token && job.valid_token?(token)
end
def max_artifacts_size
current_application_settings.max_artifacts_size.megabytes.to_i
end
end
end
end
......@@ -48,5 +48,203 @@ module API
Ci::Runner.find_by_token(params[:token]).destroy
end
end
resource :jobs do
desc 'Request a job' do
success Entities::JobRequest::Response
end
params do
requires :token, type: String, desc: %q(Runner's authentication token)
optional :last_update, type: String, desc: %q(Runner's queue last_update token)
optional :info, type: Hash, desc: %q(Runner's metadata)
end
post '/request' do
authenticate_runner!
not_found! unless current_runner.active?
update_runner_info
if current_runner.is_runner_queue_value_latest?(params[:last_update])
header 'X-GitLab-Last-Update', params[:last_update]
Gitlab::Metrics.add_event(:build_not_found_cached)
return job_not_found!
end
new_update = current_runner.ensure_runner_queue_value
result = ::Ci::RegisterJobService.new(current_runner).execute
if result.valid?
if result.build
Gitlab::Metrics.add_event(:build_found,
project: result.build.project.path_with_namespace)
present result.build, with: Entities::JobRequest::Response
else
Gitlab::Metrics.add_event(:build_not_found)
header 'X-GitLab-Last-Update', new_update
job_not_found!
end
else
# We received build that is invalid due to concurrency conflict
Gitlab::Metrics.add_event(:build_invalid)
conflict!
end
end
desc 'Updates a job' do
http_codes [[200, 'Job was updated'], [403, 'Forbidden']]
end
params do
requires :token, type: String, desc: %q(Runners's authentication token)
requires :id, type: Integer, desc: %q(Job's ID)
optional :trace, type: String, desc: %q(Job's full trace)
optional :state, type: String, desc: %q(Job's status: success, failed)
end
put '/:id' do
job = Ci::Build.find_by_id(params[:id])
authenticate_job!(job)
job.update_attributes(trace: params[:trace]) if params[:trace]
Gitlab::Metrics.add_event(:update_build,
project: job.project.path_with_namespace)
case params[:state].to_s
when 'success'
job.success
when 'failed'
job.drop
end
end
desc 'Appends a patch to the job trace' do
http_codes [[202, 'Trace was patched'],
[400, 'Missing Content-Range header'],
[403, 'Forbidden'],
[416, 'Range not satisfiable']]
end
params do
requires :id, type: Integer, desc: %q(Job's ID)
optional :token, type: String, desc: %q(Job's authentication token)
end
patch '/:id/trace' do
job = Ci::Build.find_by_id(params[:id])
authenticate_job!(job)
error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range')
content_range = request.headers['Content-Range']
content_range = content_range.split('-')
current_length = job.trace_length
unless current_length == content_range[0].to_i
return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
end
job.append_trace(request.body.read, content_range[0].to_i)
status 202
header 'Job-Status', job.status
header 'Range', "0-#{job.trace_length}"
end
desc 'Authorize artifacts uploading for job' do
http_codes [[200, 'Upload allowed'],
[403, 'Forbidden'],
[405, 'Artifacts support not enabled'],
[413, 'File too large']]
end
params do
requires :id, type: Integer, desc: %q(Job's ID)
optional :token, type: String, desc: %q(Job's authentication token)
optional :filesize, type: Integer, desc: %q(Artifacts filesize)
end
post '/:id/artifacts/authorize' do
not_allowed! unless Gitlab.config.artifacts.enabled
require_gitlab_workhorse!
Gitlab::Workhorse.verify_api_request!(headers)
job = Ci::Build.find_by_id(params[:id])
authenticate_job!(job)
forbidden!('Job is not running') unless job.running?
if params[:filesize]
file_size = params[:filesize].to_i
file_to_large! unless file_size < max_artifacts_size
end
status 200
content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
Gitlab::Workhorse.artifact_upload_ok
end
desc 'Upload artifacts for job' do
success Entities::JobRequest::Response
http_codes [[201, 'Artifact uploaded'],
[400, 'Bad request'],
[403, 'Forbidden'],
[405, 'Artifacts support not enabled'],
[413, 'File too large']]
end
params do
requires :id, type: Integer, desc: %q(Job's ID)
optional :token, type: String, desc: %q(Job's authentication token)
optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
optional :file, type: File, desc: %q(Artifact's file)
optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
end
post '/:id/artifacts' do
not_allowed! unless Gitlab.config.artifacts.enabled
require_gitlab_workhorse!
job = Ci::Build.find_by_id(params[:id])
authenticate_job!(job)
forbidden!('Job is not running!') unless job.running?
artifacts_upload_path = ArtifactUploader.artifacts_upload_path
artifacts = uploaded_file(:file, artifacts_upload_path)
metadata = uploaded_file(:metadata, artifacts_upload_path)
bad_request!('Missing artifacts file!') unless artifacts
file_to_large! unless artifacts.size < max_artifacts_size
job.artifacts_file = artifacts
job.artifacts_metadata = metadata
job.artifacts_expire_in = params['expire_in'] ||
Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
if job.save
present job, with: Entities::JobRequest::Response
else
render_validation_error!(job)
end
end
desc 'Download the artifacts file for job' do
http_codes [[200, 'Upload allowed'],
[403, 'Forbidden'],
[404, 'Artifact not found']]
end
params do
requires :id, type: Integer, desc: %q(Job's ID)
optional :token, type: String, desc: %q(Job's authentication token)
end
get '/:id/artifacts' do
job = Ci::Build.find_by_id(params[:id])
authenticate_job!(job)
artifacts_file = job.artifacts_file
unless artifacts_file.file_storage?
return redirect_to job.artifacts_file.url
end
unless artifacts_file.exists?
not_found!
end
present_file!(artifacts_file.path, artifacts_file.filename)
end
end
end
end
......@@ -24,7 +24,7 @@ module Ci
new_update = current_runner.ensure_runner_queue_value
result = Ci::RegisterBuildService.new(current_runner).execute
result = Ci::RegisterJobService.new(current_runner).execute
if result.valid?
if result.build
......
module Gitlab
module Ci
module Build
class Image
attr_reader :name
class << self
def from_image(job)
image = Gitlab::Ci::Build::Image.new(job.options[:image])
return unless image.valid?
image
end
def from_services(job)
services = job.options[:services].to_a.map do |service|
Gitlab::Ci::Build::Image.new(service)
end
services.select(&:valid?).compact
end
end
def initialize(image)
@name = image
end
def valid?
@name.present?
end
end
end
end
end
module Gitlab
module Ci
module Build
class Step
WHEN_ON_FAILURE = 'on_failure'.freeze
WHEN_ON_SUCCESS = 'on_success'.freeze
WHEN_ALWAYS = 'always'.freeze
attr_reader :name
attr_writer :script
attr_accessor :timeout, :when, :allow_failure
class << self
def from_commands(job)
self.new(:script).tap do |step|
step.script = job.commands
step.timeout = job.timeout
step.when = WHEN_ON_SUCCESS
end
end
def from_after_script(job)
after_script = job.options[:after_script]
return unless after_script
self.new(:after_script).tap do |step|
step.script = after_script
step.timeout = job.timeout
step.when = WHEN_ALWAYS
step.allow_failure = true
end
end
end
def initialize(name)
@name = name
@allow_failure = false
end
def script
@script.split("\n")
end
end
end
end
end
......@@ -15,8 +15,8 @@ FactoryGirl.define do
options do
{
image: "ruby:2.1",
services: ["postgres"]
image: 'ruby:2.1',
services: ['postgres']
}
end
......@@ -166,5 +166,31 @@ FactoryGirl.define do
allow(build).to receive(:commit).and_return build(:commit)
end
end
trait :extended_options do
options do
{
image: 'ruby:2.1',
services: ['postgres'],
after_script: "ls\ndate",
artifacts: {
name: 'artifacts_file',
untracked: false,
paths: ['out/'],
when: 'always',
expire_in: '7d'
},
cache: {
key: 'cache_key',
untracked: false,
paths: ['vendor/*']
}
}
end
end
trait :no_options do
options { {} }
end
end
end
require 'spec_helper'
describe Gitlab::Ci::Build::Image do
let(:job) { create(:ci_build, :no_options) }
describe '#from_image' do
subject { described_class.from_image(job) }
context 'when image is defined in job' do
let(:image_name) { 'ruby:2.1' }
let(:job) { create(:ci_build, options: { image: image_name } ) }
it 'fabricates an object of the proper class' do
is_expected.to be_kind_of(described_class)
end
it 'populates fabricated object with the proper name attribute' do
expect(subject.name).to eq(image_name)
end
context 'when image name is empty' do
let(:image_name) { '' }
it 'does not fabricate an object' do
is_expected.to be_nil
end
end
end
context 'when image is not defined in job' do
it 'does not fabricate an object' do
is_expected.to be_nil
end
end
end
describe '#from_services' do
subject { described_class.from_services(job) }
context 'when services are defined in job' do
let(:service_image_name) { 'postgres' }
let(:job) { create(:ci_build, options: { services: [service_image_name] }) }
it 'fabricates an non-empty array of objects' do
is_expected.to be_kind_of(Array)
is_expected.not_to be_empty
expect(subject.first.name).to eq(service_image_name)
end
context 'when service image name is empty' do
let(:service_image_name) { '' }
it 'fabricates an empty array' do
is_expected.to be_kind_of(Array)
is_expected.to be_empty
end
end
end
context 'when services are not defined in job' do
it 'fabricates an empty array' do
is_expected.to be_kind_of(Array)
is_expected.to be_empty
end
end
end
end
require 'spec_helper'
describe Gitlab::Ci::Build::Step do
let(:job) { create(:ci_build, :no_options, commands: "ls -la\ndate") }
describe '#from_commands' do
subject { described_class.from_commands(job) }
it 'fabricates an object' do
expect(subject.name).to eq(:script)
expect(subject.script).to eq(['ls -la', 'date'])
expect(subject.timeout).to eq(job.timeout)
expect(subject.when).to eq('on_success')
expect(subject.allow_failure).to be_falsey
end
end
describe '#from_after_script' do
subject { described_class.from_after_script(job) }
context 'when after_script is empty' do
it 'doesn not fabricate an object' do
is_expected.to be_nil
end
end
context 'when after_script is not empty' do
let(:job) { create(:ci_build, options: { after_script: "ls -la\ndate" }) }
it 'fabricates an object' do
expect(subject.name).to eq(:after_script)
expect(subject.script).to eq(['ls -la', 'date'])
expect(subject.timeout).to eq(job.timeout)
expect(subject.when).to eq('always')
expect(subject.allow_failure).to be_truthy
end
end
end
end
This diff is collapsed.
require 'spec_helper'
module Ci
describe RegisterBuildService, services: true do
describe RegisterJobService, services: true do
let!(:project) { FactoryGirl.create :empty_project, shared_runners_enabled: false }
let!(:pipeline) { FactoryGirl.create :ci_pipeline, project: project }
let!(:pending_build) { FactoryGirl.create :ci_build, pipeline: pipeline }
......@@ -181,7 +181,7 @@ module Ci
let!(:other_build) { create :ci_build, pipeline: pipeline }
before do
allow_any_instance_of(Ci::RegisterBuildService).to receive(:builds_for_specific_runner)
allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
.and_return([pending_build, other_build])
end
......@@ -193,7 +193,7 @@ module Ci
context 'when single build is in queue' do
before do
allow_any_instance_of(Ci::RegisterBuildService).to receive(:builds_for_specific_runner)
allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
.and_return([pending_build])
end
......@@ -204,7 +204,7 @@ module Ci
context 'when there is no build in queue' do
before do
allow_any_instance_of(Ci::RegisterBuildService).to receive(:builds_for_specific_runner)
allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
.and_return([])
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment